diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9dc22fd..2ff88ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,11 @@ jobs: - name: Prose run: python3 bin/check_prose.py + # Stdlib only as well. These cover the scripts in bin/, which have + # no dependency manager and no Django to test them with. + - name: The bin/ scripts + run: python3 -m unittest discover -s bin/tests + - name: No migration is missing run: | docker compose -f docker-compose.test.yml run --rm --entrypoint sh pytest \ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d3fb09..d9c9452 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,10 +9,9 @@ repos: rev: v0.42.0 hooks: - id: markdownlint - # Two approved documents have never passed this hook: the design - # spec and the plan it was built from. Reflowing either would be - # a huge diff on prose nobody asked to rewrite. - exclude: ^docs/(specs|superpowers)/ + # Specs and plans have never passed this hook. Reflowing prose + # nobody asked to rewrite would be a huge diff. + exclude: ^docs/(specs|plans)/ - repo: https://github.com/astral-sh/ruff-pre-commit # Must match the ruff in api/uv.lock. On different versions the hook # and `just lint` format differently and undo each other, so a commit diff --git a/AGENTS.md b/AGENTS.md index dd606b3..8fa34f0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ Comments, docstrings, commit messages and documentation follow ASD-STE100. -- One idea per sentence. Under 20 words. +- One idea per sentence. 20 words or fewer. - Active voice. Present tense. - No em dashes. A qualifier goes in brackets: `GitHub (2026-08-30 15:32)`. - A standalone `—` is allowed as an empty-cell marker. It is a placeholder, @@ -27,6 +27,9 @@ Delete a comment when the reason it recorded stops being true. - No sample output. It goes stale and nobody re-runs it. - No section nobody asked for. - A README describes the repository as it is now, not as it changed. +- A spec describes the system, so it moves with the code. A plan + records what somebody intended, so it stays as written. That includes + the places where the plan turned out to be wrong. ## Engineering @@ -54,6 +57,28 @@ edit that worked. Open the page. Read the value back. template that overrides it. An override patches one instance of a problem; the setting fixes the class. +## Layout + +Which file a piece of code goes in, by what it is. + +- An app keeps one `views.py` until it serves more than one resource. + Then it becomes `views/`, one module per resource or per long-lived + operation. `catalog` serves three resources, and `imports.py` is an + operation on `Service`. A sub-resource stays with its parent, so an + event's update log is not a second module. +- A **queryset method** is chainable vocabulary. It lives on the + queryset in `models.py`, because a caller chains it onto a query. +- A **bare expression** is handed to `annotate()`. It lives in + `queries.py`, because nothing chains it. +- `queries.py` is row-scoped and `aggregates.py` is collection-scoped. + One lands in a serialized row. The other lands in the `aggregates` + key beside `results`. +- `common` holds no expression that describes an app. `common/ordering.py` + records this as a scar. The domain subqueries lived there, and the base + layer imported the app it describes. Naming an app's choices to publish + them is another thing, and it is fine: `/meta/` lists every enum, and + one place has to. + ## Settings The split is by who named the setting. @@ -91,11 +116,45 @@ Every behaviour change carries a test. The suite runs with no network: adapters are tested against recorded fixtures. A test's comment says what would break if the assertion failed. +If the answer is "nothing", the test does not earn its place. + +**Assert on behaviour, not arrangement.** Field order, section titles and +`list_display` contents change for cosmetic reasons. A test that pins +them fails on a harmless edit and catches no defect. One pinned the four +component fieldsets in order. Moving two lines then cost a test edit, and +it had never caught a bug. + +**A test fails only when something is broken.** A reasonable refactor +breaks a wrong test and nothing else. + +**Never restate the implementation.** A test that computes its expected +value the way the code does passes while both are wrong. Write the +expected value out. + +**No test-only production code.** No argument, method, flag or branch +exists so that a test can reach something. A path no caller reaches is a +path the test should not reach either. + +**Enter through the front door.** Use the API client, the admin client or +the management command. Call an internal directly only when no pathway +reaches it. + +**Every assertion can fail.** Watch it fail once, before the code exists. +`assert fieldsets[3] is not None` held on every possible input. + +**Test what this project decided.** Django and DRF are already tested. +That a `CharField` stores a string is theirs. That a poller claims an +incident inside the window is ours. + +**In the admin, test the logic, not the form.** Worth pinning: a +read-only field the model maintains, a queryset the page filters, what +`list_editable` writes, which user `created_by` records. Not which +fields appear, or in what order. ```sh -just test # the suite -just test-cov # with the coverage gate +just test # the suite, host and bin/ scripts together just lint # ruff check and format +just check # every check CI runs, coverage gate included ``` ## Database @@ -117,6 +176,31 @@ does not help answer it does not belong there. One theme, dark. `UNFOLD["THEME"]` forces it, so a class with no `dark:` variant still lands on the right ground. +**The change view holds every field.** A model field missing from every +fieldset is invisible and unreachable, and a changelist column with no +place on the form makes opening a row answer less than the list did. +`Service.source` recorded how a service arrived and reached no form at +all. A value computed from related rows lands as a readonly field, not +nowhere. So does provenance: a field recording how a row arrived is +shown and never typed, because an editable one can be made to lie. + +**A column header is the name of its field.** The list and the form show +the same data, so they use the same word. A column headed `State` over +`is_archived` sent a reader looking for a field of that name. When the +two disagree the header gives way: the field name is what the model, the +API, the filters and every query already say. + +**A column that only redraws a field is the field.** Put the field name +in `list_display`. Django draws a boolean as a tick, takes the header +from the model and sorts the column, none of it asked for. Hand-rolling +those three is where the header `State` came from. Keep a method only +where it does what a field cannot: composing several, following a +relation, or building a link. + +**A column the list shows, the list filters.** If it is worth a column, +it is worth narrowing by. The converse does not hold: a filter may +narrow by something no column shows. + ## Commits A lowercase conventional prefix, then a declarative clause. diff --git a/README.md b/README.md index 9ca506b..414d09d 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,14 @@ Each worktree gets its own ports and database. | | | | --- | --- | | `just init` | services, dependencies, migrations, admin, hooks | -| `just dev` | server and poller together | +| `just dev` | postgres, redis, migrations, then the server and poller | | `just test` | the suite, on the host | | `just check` | everything CI runs, in the image CI runs it in | | `just lint` | fix and format | | `just info` | this worktree's ports and database | | `just reset` | drop the database and build it again | -| `just check-pages` | probe every recorded status page | +| `just clean` | stop this checkout's containers | +| `just probe-pages` | probe every recorded status page | | `just release v0.1.0` | tag, build, sign, deploy | | `just deploy` | redeploy the current image | diff --git a/api/api/defaults.py b/api/api/defaults.py index ee6b2f0..7b452dd 100644 --- a/api/api/defaults.py +++ b/api/api/defaults.py @@ -69,24 +69,27 @@ def parse(cls, value): ) from error -# Which deployment this is. We named it, so it is resolved here, and -# `debug` and `secret_key` in `settings.py` read it rather than being -# handed it. +# Which deployment this is. We named it, so `debug` and `secret_key` +# in `settings.py` read it here rather than being handed it. ENVIRONMENT = Environment.parse(os.environ.get("ENVIRONMENT")) class Throttle(StrEnum): """A rate this deployment enforces. - A view names its scope and `settings.py` gives it a rate, so the - string was written twice and had to match. DRF names the first two; - we named the rest. + A view names its scope and `settings.py` gives it a rate. The two + spellings had to match. DRF names the first two; we named the + rest. """ ANONYMOUS = "anon" SIGNED_IN = "user" IMPORT = "import" MAGIC_LINK = "magic-link" + # Its own bucket, not IMPORT's. Add-by-URL tries an import, and on + # a miss sends the same URL here. One scope would let the first + # call spend the second's budget. + SERVICE_REQUEST = "service-request" # Reading the catalog is the point, so the plain rates are wide. Each of @@ -97,6 +100,7 @@ class Throttle(StrEnum): Throttle.SIGNED_IN: "600/min", Throttle.IMPORT: "6/min", Throttle.MAGIC_LINK: "5/hour", + Throttle.SERVICE_REQUEST: "6/min", } @@ -120,6 +124,13 @@ class Throttle(StrEnum): # Long enough to read a week of failures, and no longer. POLL_RUN_RETENTION_DAYS = 30 +# How far back a provider's event may start and still claim one we +# opened. Providers back-date `starts_at` to when an incident really +# began, which is before our poll saw it. +EVENT_CLAIM_WINDOW = timedelta( + seconds=int(os.environ.get("EVENT_CLAIM_WINDOW_SECONDS", "3600")) +) + # Who the system writes as. RFC 2606 reserves the domain. No mail leaves # for it, and nobody can hold the address. SYSTEM_EMAIL = "system@statusboard.invalid" diff --git a/api/api/settings.py b/api/api/settings.py index b528ccd..1ad31cd 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -13,6 +13,7 @@ APP_URL, DEFAULT_PAGE_SIZE, ENVIRONMENT, + EVENT_CLAIM_WINDOW, MAGIC_LINK_TTL, MAX_PAGE_SIZE, POLL_COOLDOWN_SECONDS, @@ -146,9 +147,6 @@ "rest_framework.filters.OrderingFilter", "common.filters.FieldsBackend", ], - # The contract calls it `q`, and DRF's SearchFilter calls it - # `search`. One name, and the schema documents it. - "SEARCH_PARAM": "q", # The contract has always answered 429, and nothing raised one. See # `Throttle` for what each rate is protecting. "DEFAULT_THROTTLE_CLASSES": [ @@ -279,8 +277,10 @@ "COMMAND": {"search_models": True, "show_history": True}, # The sidebar is two levels deep. It cannot nest an item under an # item, so tabs carry the relationship. A component belongs to a - # service, and a status history to a component. All four are one - # page. The sidebar entry stays active on any of its tabs. + # service, and a status history to a component. A service request + # names something the catalog lacks, so it lives beside the catalog + # too. All five are one page. The sidebar entry stays active on any + # of its tabs. "TABS": [ { "models": [ @@ -288,6 +288,7 @@ "catalog.servicecomponent", "status.serviceevent", "status.componentstatus", + "catalog.servicerequest", ], "items": [ { @@ -306,6 +307,10 @@ "title": _("Status history"), "link": reverse_lazy("admin:status_componentstatus_changelist"), }, + { + "title": _("Service requests"), + "link": reverse_lazy("admin:catalog_servicerequest_changelist"), + }, ], }, { @@ -333,6 +338,26 @@ }, ], }, + { + # A magic link belongs to a user. It has no reason to be + # findable on its own. + "models": [ + "authentication.user", + "authentication.magiclinktoken", + ], + "items": [ + { + "title": _("Users"), + "link": reverse_lazy("admin:authentication_user_changelist"), + }, + { + "title": _("Magic links"), + "link": reverse_lazy( + "admin:authentication_magiclinktoken_changelist" + ), + }, + ], + }, ], "SIDEBAR": { "show_search": True, @@ -350,7 +375,9 @@ "link": reverse_lazy("admin:index"), }, { - "title": _("Services"), + # Five models across catalog and status answer + # here now, not only the services themselves. + "title": _("Catalog"), "icon": "lan", "link": reverse_lazy("admin:catalog_service_changelist"), }, @@ -380,11 +407,17 @@ "VERSION": "1.0.0", "SERVE_INCLUDE_SCHEMA": False, "COMPONENT_SPLIT_REQUEST": True, + # EventUpdate.source and ServiceEvent.detected_by share one choice + # set. Without this, the generator names the same enum twice. + "ENUM_NAME_OVERRIDES": { + "EventSourceEnum": "status.choices.EventSource", + }, "TAGS": [ {"name": "meta"}, {"name": "auth"}, {"name": "me"}, {"name": "catalog"}, + {"name": "status"}, {"name": "board"}, ], } diff --git a/api/api/urls.py b/api/api/urls.py index 3057810..6b6d751 100644 --- a/api/api/urls.py +++ b/api/api/urls.py @@ -17,4 +17,5 @@ path("", include("authentication.urls")), path("", include("catalog.urls")), path("", include("dashboards.urls")), + path("", include("status.urls")), ] diff --git a/api/authentication/admin.py b/api/authentication/admin.py index ce00c5e..00174d1 100644 --- a/api/authentication/admin.py +++ b/api/authentication/admin.py @@ -107,10 +107,10 @@ def formfield_for_foreignkey(self, db_field, request, **kwargs): list_display = [ "display_user", "display_default_board", - "display_active", - "display_bot", - "display_staff", - "display_superuser", + "is_active", + "is_bot", + "is_staff", + "is_superuser", "last_login", "last_active_at", ] @@ -137,28 +137,10 @@ def has_delete_permission(self, request, obj=None): return False return super().has_delete_permission(request, obj) - # Django's own labels read "active", "staff status" and "superuser - # status". The three sit side by side, so they read as a set here. @display(description=_("Board"), ordering="default_dashboard__name") def display_default_board(self, obj): return change_link(obj.default_dashboard) - @display(description=_("Bot"), boolean=True, ordering="is_bot") - def display_bot(self, obj): - return obj.is_bot - - @display(description=_("Active"), boolean=True, ordering="is_active") - def display_active(self, obj): - return obj.is_active - - @display(description=_("Staff"), boolean=True, ordering="is_staff") - def display_staff(self, obj): - return obj.is_staff - - @display(description=_("Superuser"), boolean=True, ordering="is_superuser") - def display_superuser(self, obj): - return obj.is_superuser - @display(description=_("User"), header=True, ordering="email") def display_user(self, obj): return [obj.email, str(obj.id), obj.email[:2].upper()] @@ -186,11 +168,12 @@ class MagicLinkTokenAdmin(BaseModelAdmin, ModelAdmin): autocomplete_fields = ["user"] ordering = ["-created_at"] # The token is the credential. It is shown, never typed over: a new - # one by hand is a link nobody was sent. - readonly_fields = ["token"] + # one by hand is a link nobody was sent. The state is read from the + # two dates below it. + readonly_fields = ["token", "display_state"] fieldsets = [ (None, {"fields": ["user", "token"]}), - (_("Validity"), {"fields": ["expires_at", "used_at"]}), + (_("Validity"), {"fields": ["display_state", "expires_at", "used_at"]}), audit_section(), ] @@ -203,6 +186,10 @@ def display_user(self, obj): label={"Usable": "success", "Used": "default", "Expired": "warning"}, ) def display_state(self, obj): + # The add form holds a link with no expiry. A link nobody has + # made yet is in no state. + if obj.expires_at is None: + return "—" if obj.used_at: return "Used" return "Usable" if obj.is_usable else "Expired" diff --git a/api/authentication/views.py b/api/authentication/views.py index 06d5192..364b8cc 100644 --- a/api/authentication/views.py +++ b/api/authentication/views.py @@ -35,6 +35,10 @@ class MagicLinkView(APIView): request=MagicLinkRequestSerializer, responses={ 202: OpenApiResponse(description="Sent, if the address can receive it."), + # Hand rolled, so it is a bare `detail` and carries no + # `code`. A client generated from the contract could not + # see this failure at all. + 400: OpenApiResponse(description="No email address in the body."), 429: OpenApiResponse(description="Too many requests."), }, ) @@ -47,7 +51,9 @@ def post(self, request): user, _ = User.objects.get_or_create(email=email) link = MagicLinkToken.objects.create(user=user) send_magic_link(link) - return Response(status=status.HTTP_204_NO_CONTENT) + # Accepted, not No content. We say nothing about whether the + # address can receive mail, and the contract promises 202. + return Response(status=status.HTTP_202_ACCEPTED) class VerifyView(APIView): diff --git a/api/catalog/admin.py b/api/catalog/admin.py index c5c3351..30ec399 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -18,8 +18,13 @@ from unfold.enums import ActionVariant from unfold.forms import BaseDialogForm -from catalog.models import Service, ServiceComponent, StatusPage -from catalog.queries import OVERALL_SEVERITY, WATCHER_COUNT +from catalog.models import Service, ServiceComponent, ServiceRequest, StatusPage +from catalog.queries import ( + COMPONENT_WATCHER_COUNT, + OVERALL_SEVERITY, + component_count, + is_tracked, +) from common.admin import ( SEVERITY_VARIANTS, BaseModelAdmin, @@ -242,8 +247,8 @@ def has_add_permission(self, request, obj=None): class TrackedFilter(DropdownFilter): """Whether anybody has this service on a board. - The count is worked out when the list is read, so there is no column - to put a range over. This is the question the poller asks too. + There is no column to filter on. `is_tracked` is the question the + poller asks, so the admin asks it the same way. """ title = _("Tracked") @@ -255,9 +260,8 @@ def lookups(self, request, model_admin): def queryset(self, request, queryset): if self.value() is None: return queryset - if self.value() == "1": - return queryset.filter(watcher_count__gt=0) - return queryset.filter(watcher_count=0) + watched = is_tracked() + return queryset.filter(watched if self.value() == "1" else ~watched) class ServiceSeverityFilter(DropdownFilter): @@ -284,6 +288,28 @@ def queryset(self, request, queryset): ) +class WatchedFilter(DropdownFilter): + """Whether anybody holds this component on a board. + + The watcher count is a column and was not a filter. `boards` is the + relation the count reads, so this narrows on the same join. + """ + + title = _("Watched") + parameter_name = "watched" + + def lookups(self, request, model_admin): + return [("1", _("Watched")), ("0", _("Unwatched"))] + + def queryset(self, request, queryset): + if self.value() is None: + return queryset + watched = queryset.filter(boards__isnull=False) + if self.value() == "1": + return watched.distinct() + return queryset.exclude(pk__in=watched.values("pk")) + + class ComponentSeverityFilter(DropdownFilter): """Filter components by the status they are showing now. @@ -310,15 +336,12 @@ class ServiceAdmin(BaseModelAdmin, SimpleHistoryAdmin, ModelAdmin): list_display = [ "display_service", "display_severity", - "is_featured", - "display_watchers", "provider", "display_related", ] search_fields = [ "name", "slug", - "description", "homepage_url", "status_page__url", ] @@ -327,17 +350,22 @@ class ServiceAdmin(BaseModelAdmin, SimpleHistoryAdmin, ModelAdmin): # when. A service belongs to nothing, so its state leads. list_filter = [ ServiceSeverityFilter, - "is_featured", ("status_page__provider", ChoicesDropdownFilter), + ("source", ChoicesDropdownFilter), TrackedFilter, ("created_at", RangeDateTimeFilter), ("updated_at", RangeDateTimeFilter), ] - # Not `ordering`: the checks want a real column there, and this one - # is counted when the list is read. `get_queryset` sorts instead. ordering = ["name"] + # How the row arrived is written by the importer or by the add form. + # A person typing it makes the record claim a history it does not + # have. + readonly_fields = ["source"] + # `source` sits with the name, not in the collapsed trail. It was + # on no form for two releases, and a fold is the next place to lose + # it. fieldsets = [ - (None, {"fields": ["name", "slug", "description", "is_featured"]}), + (None, {"fields": ["name", "slug", "source"]}), (_("Presentation"), {"fields": ["logo", "homepage_url"]}), audit_section(), ] @@ -361,20 +389,14 @@ def get_queryset(self, request): .select_related("status_page", "poller") .annotate( severity_now=OVERALL_SEVERITY, - watcher_count=WATCHER_COUNT, - component_count=related_count(ServiceComponent.objects, "service"), + component_count=component_count(), event_count=related_count(ServiceEvent.objects, "service"), status_count=related_count( ComponentStatus.objects, "component__service" ), ) - .order_by("-watcher_count", "name") ) - @display(description=_("Watchers"), ordering="watcher_count") - def display_watchers(self, obj): - return obj.watcher_count - @display(description=_("Service"), header=True, ordering="name") def display_service(self, obj): # Two lines and the product's own mark, so a long catalog @@ -407,11 +429,18 @@ def display_related(self, obj): return { "title": _("Related"), "items": [ + # The count is the API's, so the admin and a client + # never disagree about how many parts a service has. + # The link carries the same two filters, because a + # count beside a link describes what the link opens. + # Clearing a chip reaches the rollup and the archive. filtered_list( "admin:catalog_servicecomponent_changelist", _("Components"), obj.component_count, service__id__exact=obj.pk, + is_overall__exact=0, + is_archived__exact=0, ), filtered_list( "admin:status_serviceevent_changelist", @@ -458,7 +487,9 @@ def import_from_status_page(self, request, form): url = form.cleaned_data["status_page_url"] try: - service, created = import_from_url(url) + # The staff member who pasted the URL, the same stamp an + # add form leaves. The system account means automated. + service, created = import_from_url(url, author=request.user) except Exception as error: # noqa: BLE001 — the page is a stranger's # Anything can come back from a URL a person pasted. Say what # happened rather than showing a 500. @@ -506,8 +537,13 @@ class ServiceComponentAdmin( "display_service", "display_severity", "is_overall", - "display_archived", + "is_archived", "display_related", + "watchers", + # `suggested` reads this flag on every component. Ticking it on + # a rollup surfaces the service. Ticking it on a leaf surfaces + # that part. + "is_featured", ] search_fields = [ "name", @@ -527,13 +563,20 @@ class ServiceComponentAdmin( "is_archived", ("archived_at", RangeDateTimeFilter), ("created_at", RangeDateTimeFilter), + "is_featured", + WatchedFilter, ] autocomplete_fields = ["service", "parent"] ordering = ["service__name", "status_page_order"] # A parent is one of the same service's components. autocomplete_scope = ("service",) # The date follows the flag, so it is shown and never typed. - readonly_fields = ["archived_at"] + # The count is read from the boards, so it is shown and cannot be + # typed at all. + readonly_fields = ["archived_at", "watchers"] + # A checkbox in the list, not an action. One Save flips any mix of + # rollups and leaves, which a toggle action could never do at once. + list_editable = ["is_featured"] fieldsets = [ ( None, @@ -542,8 +585,8 @@ class ServiceComponentAdmin( "service", "name", "external_id", - "is_archived", - "archived_at", + "is_featured", + "watchers", ] }, ), @@ -551,6 +594,7 @@ class ServiceComponentAdmin( _("Position"), {"fields": ["parent", "status_page_order", "is_overall"]}, ), + (_("Archive"), {"fields": ["is_archived", "archived_at"]}), audit_section(), ] inlines = [ComponentStatusInline] @@ -568,12 +612,17 @@ def formfield_for_foreignkey(self, db_field, request, **kwargs): self.admin_site, scope={"service": editing.service_id if editing else None}, ) + # The rollup is never a parent. Excluding it here, on both + # add and change, keeps the picker from offering the one + # choice the database refuses. + queryset = ServiceComponent.objects.exclude(is_overall=True) if editing is not None: # The server's half: the picker narrows what is offered, # this refuses anything else that is posted. - kwargs["queryset"] = ServiceComponent.objects.filter( - service_id=editing.service_id - ).exclude(pk=editing.pk) + queryset = queryset.filter(service_id=editing.service_id).exclude( + pk=editing.pk + ) + kwargs["queryset"] = queryset return super().formfield_for_foreignkey(db_field, request, **kwargs) def get_queryset(self, request): @@ -585,6 +634,7 @@ def get_queryset(self, request): severity_now=CURRENT_SEVERITY, status_count=related_count(ComponentStatus.objects, "component"), event_count=related_count(ServiceEvent.objects, "affected_components"), + watcher_count=COMPONENT_WATCHER_COUNT, ) ) @@ -602,16 +652,6 @@ def display_component(self, obj): def display_service(self, obj): return change_link(obj.service) - @display( - description=_("State"), - label={"Live": "success", "Archived": "default"}, - ordering="is_archived", - ) - def display_archived(self, obj): - # An archived component still reads on the table. Without this - # it looked like one the provider is still publishing. - return "Archived" if obj.is_archived else "Live" - @display(description=_("View"), dropdown=True) def display_related(self, obj): return { @@ -635,3 +675,33 @@ def display_related(self, obj): @display(description=_("Status"), label=SEVERITY_VARIANTS, ordering="severity_now") def display_severity(self, obj): return severity_label(obj.severity_now) + + @display(description=_("Watchers"), ordering="watcher_count") + def watchers(self, obj): + return obj.watcher_count + + +@admin.register(ServiceRequest) +class ServiceRequestAdmin(BaseModelAdmin, ModelAdmin): + """What the catalog is missing, most asked for first.""" + + list_display = ["url", "request_count", "last_requested_at", "created_by"] + ordering = ["-request_count", "-last_requested_at"] + search_fields = ["url"] + # Who asked, then how many asked, then when the last one did. The + # URL is the row itself, and the search box reaches it. + list_filter = [ + ("created_by", AutocompleteSelectFilter), + ("request_count", RangeNumericFilter), + ("last_requested_at", RangeDateTimeFilter), + ] + # The app writes all three. A hand-typed count is a demand nobody made. + readonly_fields = ["url", "request_count", "last_requested_at"] + fieldsets = [ + (None, {"fields": ["url", "request_count", "last_requested_at"]}), + audit_section(), + ] + + def has_add_permission(self, request): + # A row arrives from the app, never from here. + return False diff --git a/api/catalog/choices.py b/api/catalog/choices.py index 71c4010..5e0b23d 100644 --- a/api/catalog/choices.py +++ b/api/catalog/choices.py @@ -23,3 +23,15 @@ class StatusPageProvider(models.TextChoices): ORACLE = "oracle", "Oracle" GOOGLE_FEED = "google_feed", "Google feed" RSS = "rss", "RSS" + + +class ServiceSource(models.TextChoices): + """How a service reached the catalog. + + Two values, because there are two writers. `import_from_url` makes + one, and the admin's add form makes the other. Anything else, such + as a shell or a migration, reads manual. It was not imported. + """ + + MANUAL = "manual", "Manual" + IMPORT = "import", "Imported from a status page" diff --git a/api/catalog/filters.py b/api/catalog/filters.py index 86618fb..4f71c7e 100644 --- a/api/catalog/filters.py +++ b/api/catalog/filters.py @@ -7,50 +7,52 @@ from django_filters import rest_framework as filters -from catalog.models import Service, ServiceComponent -from status.models import ServiceEvent - - -class ServiceFilter(filters.FilterSet): - # Declared for a second reason. `overall_component` is not a relation - # on Service. The current severity is not a column either. It is the - # open row of a component's status history. So the contract's name is - # kept, pointed at the view's `severity_now` annotation. - overall_component__status__severity = filters.NumberFilter( - field_name="severity_now" - ) - overall_component__status__severity__lte = filters.NumberFilter( - field_name="severity_now", lookup_expr="lte" - ) - - class Meta: - model = Service - fields = { - "status_page__provider": ["exact"], - "is_featured": ["exact"], - } - - -class ComponentFilter(filters.FilterSet): - # `status` is not a relation. A component has a history of statuses - # and the open one is current. Same contract name, same annotation. - status__severity = filters.NumberFilter(field_name="severity_now") - status__severity__lte = filters.NumberFilter( - field_name="severity_now", lookup_expr="lte" - ) +from catalog.models import ServiceComponent +from catalog.queries import descendant_ids +from common.filters import SeverityFilterMixin + + +class ComponentFilter(SeverityFilterMixin): + q = filters.CharFilter(method="filter_q") + service = filters.CharFilter(field_name="service__slug") + # Every descendant, not one level. `parent` would name a query this + # does not run. + ancestor = filters.UUIDFilter(method="filter_ancestor") + event = filters.UUIDFilter(field_name="events__id") + # `for_display` annotates this per viewer. It is not a column, so + # there is nothing to generate the filter from. + is_tracked = filters.BooleanFilter(method="filter_is_tracked") class Meta: model = ServiceComponent fields = {"is_overall": ["exact"]} - -class ServiceEventFilter(filters.FilterSet): - class Meta: - model = ServiceEvent - fields = { - "kind": ["exact"], - "phase": ["exact", "in"], - "ends_at": ["isnull", "gte"], - "starts_at": ["gte", "lte"], - "affected_components": ["exact"], - } + def filter_q(self, queryset, name, value): + """Narrow to what the caller typed, beside the other parameters. + + `ServiceComponentQuerySet.search` is the one definition of what a + word matches. The view read `q` out of the request instead. A + documented parameter then sat outside the filter set. + """ + return queryset.search(value) + + def filter_ancestor(self, queryset, name, value): + """Ask the tree for the ids below, then narrow to them. + + `catalog.queries.descendant_ids` is the one walk down `parent`. + A join written here would be a second answer to the question a + descendant count already asks. + """ + return queryset.filter(pk__in=descendant_ids(value)) + + def filter_is_tracked(self, queryset, name, value): + """Answer it here, because nobody signed out tracks anything. + + Both directions are true for that reader, so neither is an + error. `for_display` annotates `is_tracked` as NULL then, and + SQL reads `NOT NULL` as NULL rather than true. Comparing against + the annotation would drop every row from Untracked. + """ + if self.request is None or self.request.user.is_anonymous: + return queryset.none() if value else queryset + return queryset.filter(is_tracked=value) diff --git a/api/catalog/migrations/0003_component_is_featured.py b/api/catalog/migrations/0003_component_is_featured.py new file mode 100644 index 0000000..7521347 --- /dev/null +++ b/api/catalog/migrations/0003_component_is_featured.py @@ -0,0 +1,22 @@ +# Generated by Django 5.2.17 on 2026-09-03 18:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0002_drop_watcher_count"), + ] + + operations = [ + migrations.AddField( + model_name="historicalservicecomponent", + name="is_featured", + field=models.BooleanField(default=False, verbose_name="Featured"), + ), + migrations.AddField( + model_name="servicecomponent", + name="is_featured", + field=models.BooleanField(default=False, verbose_name="Featured"), + ), + ] diff --git a/api/catalog/migrations/0004_component_ancestor_ids.py b/api/catalog/migrations/0004_component_ancestor_ids.py new file mode 100644 index 0000000..a1537a3 --- /dev/null +++ b/api/catalog/migrations/0004_component_ancestor_ids.py @@ -0,0 +1,70 @@ +# Generated by Django 5.2.17 on 2026-09-03 18:25 + +import django.contrib.postgres.fields +import django.contrib.postgres.indexes +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0003_component_is_featured"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name="historicalservicecomponent", + name="ancestor_ids", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.UUIDField(), + blank=True, + default=list, + editable=False, + size=None, + ), + ), + migrations.AddField( + model_name="servicecomponent", + name="ancestor_ids", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.UUIDField(), + blank=True, + default=list, + editable=False, + size=None, + ), + ), + # The column cannot start empty. `polling.reconcile` fills it, + # and it runs only for a poller in `PollerQuerySet.active()`. A + # service nobody tracks is never polled again, so its rows would + # read as having no ancestor for good. + # + # A cycle terminates the walk on its own. Every row in one has a + # parent, so no cycle is reachable from the roots the base term + # selects. Those rows keep the empty default. + migrations.RunSQL( + sql=""" + WITH RECURSIVE tree AS ( + SELECT id, ARRAY[]::uuid[] AS ancestors + FROM catalog_servicecomponent + WHERE parent_id IS NULL + UNION ALL + SELECT child.id, tree.ancestors || tree.id + FROM catalog_servicecomponent child + JOIN tree ON child.parent_id = tree.id + ) + UPDATE catalog_servicecomponent AS row + SET ancestor_ids = tree.ancestors + FROM tree + WHERE tree.id = row.id; + """, + reverse_sql=migrations.RunSQL.noop, + ), + migrations.AddIndex( + model_name="servicecomponent", + index=django.contrib.postgres.indexes.GinIndex( + fields=["ancestor_ids"], name="components_by_ancestor" + ), + ), + ] diff --git a/api/catalog/migrations/0005_component_search_document.py b/api/catalog/migrations/0005_component_search_document.py new file mode 100644 index 0000000..8c52a53 --- /dev/null +++ b/api/catalog/migrations/0005_component_search_document.py @@ -0,0 +1,59 @@ +# Generated by Django 5.2.17 on 2026-09-03 18:46 + +import django.contrib.postgres.indexes +import django.contrib.postgres.search +from django.conf import settings +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0004_component_ancestor_ids"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name="historicalservicecomponent", + name="search_document", + field=django.contrib.postgres.search.SearchVectorField( + editable=False, null=True + ), + ), + migrations.AddField( + model_name="servicecomponent", + name="search_document", + field=django.contrib.postgres.search.SearchVectorField( + editable=False, null=True + ), + ), + # The column cannot start null. `polling.reconcile` fills it, and + # it runs only for a poller in `PollerQuerySet.active()`. A + # service nobody tracks is never polled again, so its rows would + # stay unfindable by search for good. + # + # `ancestor_ids` is already backfilled by 0004, so the ordered + # walk here only joins the names it names. + migrations.RunSQL( + sql=""" + UPDATE catalog_servicecomponent AS c + SET search_document = + setweight(to_tsvector(coalesce(c.name, '')), 'A') || + setweight(to_tsvector(coalesce(( + SELECT string_agg(a.name, ' ' ORDER BY u.ord) + FROM unnest(c.ancestor_ids) WITH ORDINALITY AS u(id, ord) + JOIN catalog_servicecomponent AS a ON a.id = u.id + ), '')), 'B') || + setweight(to_tsvector(coalesce(s.name, '')), 'C') + FROM catalog_service AS s + WHERE s.id = c.service_id; + """, + reverse_sql=migrations.RunSQL.noop, + ), + migrations.AddIndex( + model_name="servicecomponent", + index=django.contrib.postgres.indexes.GinIndex( + fields=["search_document"], name="components_by_search" + ), + ), + ] diff --git a/api/catalog/migrations/0006_service_request.py b/api/catalog/migrations/0006_service_request.py new file mode 100644 index 0000000..3f08780 --- /dev/null +++ b/api/catalog/migrations/0006_service_request.py @@ -0,0 +1,66 @@ +# Generated by Django 5.2.17 on 2026-09-04 01:02 + +import uuid + +import django.db.models.deletion +import django.utils.timezone +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0005_component_search_document"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="ServiceRequest", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid7, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("url", models.URLField(unique=True)), + ("request_count", models.PositiveIntegerField(default=1)), + ( + "last_requested_at", + models.DateTimeField(default=django.utils.timezone.now), + ), + ( + "created_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "updated_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "ordering": ["-request_count", "-last_requested_at"], + "abstract": False, + }, + ), + ] diff --git a/api/catalog/migrations/0007_drop_service_columns.py b/api/catalog/migrations/0007_drop_service_columns.py new file mode 100644 index 0000000..3a699fd --- /dev/null +++ b/api/catalog/migrations/0007_drop_service_columns.py @@ -0,0 +1,28 @@ +# Generated by Django 5.2.17 on 2026-09-04 01:38 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0006_service_request"), + ] + + operations = [ + migrations.RemoveField( + model_name="historicalservice", + name="description", + ), + migrations.RemoveField( + model_name="historicalservice", + name="is_featured", + ), + migrations.RemoveField( + model_name="service", + name="description", + ), + migrations.RemoveField( + model_name="service", + name="is_featured", + ), + ] diff --git a/api/catalog/migrations/0008_component_ancestor_table.py b/api/catalog/migrations/0008_component_ancestor_table.py new file mode 100644 index 0000000..86542aa --- /dev/null +++ b/api/catalog/migrations/0008_component_ancestor_table.py @@ -0,0 +1,121 @@ +# Generated by Django 5.2.17 on 2026-09-04 06:08 + +import uuid + +import django.db.models.deletion +from django.db import migrations, models + + +def fill_from_parent(apps, schema_editor): + """Fill the table with the walk `rebuild_ancestry` does. + + That walk defines a component's ancestry, so the table starts where + a poll would leave it. The dropped array is not read. It kept the + steps above a deleted component, and that is the defect this table + removes. + + `ServiceComponent.ancestors` holds the rules the walk obeys. + + Python, not SQL, because the key defaults to `uuid.uuid7`. Postgres + 17 has no function for version 7. + """ + ServiceComponent = apps.get_model("catalog", "ServiceComponent") + ComponentAncestor = apps.get_model("catalog", "ComponentAncestor") + rows = { + row_id: (parent_id, service_id) + for row_id, parent_id, service_id in ServiceComponent.objects.values_list( + "id", "parent_id", "service_id" + ) + } + links = [] + for row_id, (parent_id, service_id) in rows.items(): + chain, seen, up = [], {row_id}, parent_id + while up in rows and up not in seen and rows[up][1] == service_id: + seen.add(up) + chain.append(up) + up = rows[up][0] + links += [ + ComponentAncestor(ancestor_id=up, descendant_id=row_id, depth=depth) + # The walk climbs, so the first step is the parent. + for depth, up in enumerate(chain, start=1) + ] + ComponentAncestor.objects.bulk_create(links, batch_size=1000) + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0007_drop_service_columns"), + ] + + operations = [ + migrations.CreateModel( + name="ComponentAncestor", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid7, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("depth", models.PositiveIntegerField()), + ], + options={ + "ordering": ["-depth"], + }, + ), + migrations.AddField( + model_name="componentancestor", + name="ancestor", + field=models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="descendant_links", + to="catalog.servicecomponent", + ), + ), + migrations.AddField( + model_name="componentancestor", + name="descendant", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="ancestor_links", + to="catalog.servicecomponent", + ), + ), + migrations.AddConstraint( + model_name="componentancestor", + constraint=models.UniqueConstraint( + fields=("ancestor", "descendant"), name="one_link_per_component_pair" + ), + ), + migrations.AddConstraint( + model_name="componentancestor", + constraint=models.CheckConstraint( + condition=models.Q(("ancestor", models.F("descendant")), _negated=True), + name="ancestry_excludes_self", + ), + ), + migrations.AddConstraint( + model_name="componentancestor", + constraint=models.CheckConstraint( + condition=models.Q(("depth__gte", 1)), + name="ancestry_depth_starts_at_one", + ), + ), + migrations.RunPython(fill_from_parent, reverse_code=migrations.RunPython.noop), + migrations.RemoveIndex( + model_name="servicecomponent", + name="components_by_ancestor", + ), + migrations.RemoveField( + model_name="historicalservicecomponent", + name="ancestor_ids", + ), + migrations.RemoveField( + model_name="servicecomponent", + name="ancestor_ids", + ), + ] diff --git a/api/catalog/migrations/0009_history_drops_search_document.py b/api/catalog/migrations/0009_history_drops_search_document.py new file mode 100644 index 0000000..f280627 --- /dev/null +++ b/api/catalog/migrations/0009_history_drops_search_document.py @@ -0,0 +1,16 @@ +# Generated by Django 5.2.17 on 2026-09-04 20:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0008_component_ancestor_table"), + ] + + operations = [ + migrations.RemoveField( + model_name="historicalservicecomponent", + name="search_document", + ), + ] diff --git a/api/catalog/migrations/0010_rollup_is_not_a_parent.py b/api/catalog/migrations/0010_rollup_is_not_a_parent.py new file mode 100644 index 0000000..b79bedb --- /dev/null +++ b/api/catalog/migrations/0010_rollup_is_not_a_parent.py @@ -0,0 +1,58 @@ +from django.db import migrations + +# `is_overall` belongs to the parent row, not the row being written. A +# `CheckConstraint` sees one row, so it cannot read a joined column and +# cannot express "my parent is not the rollup". A trigger can, because +# it runs a query rather than evaluating an expression. +# +# The function checks both directions of the same rule. A row cannot be +# inserted or reparented under the rollup. A row cannot gain the rollup +# flag while something already points at it as a parent. Either gap +# would corrupt every descendant count and breadcrumb under it. +CREATE_TRIGGER = """ +CREATE FUNCTION catalog_component_rollup_is_not_a_parent() +RETURNS trigger AS $$ +BEGIN + IF NEW.parent_id IS NOT NULL AND EXISTS ( + SELECT 1 FROM catalog_servicecomponent + WHERE id = NEW.parent_id AND is_overall + ) THEN + RAISE EXCEPTION + 'rollup_is_not_a_parent: % cannot be parented under the overall component', + NEW.id + USING ERRCODE = 'check_violation'; + END IF; + + IF NEW.is_overall AND EXISTS ( + SELECT 1 FROM catalog_servicecomponent WHERE parent_id = NEW.id + ) THEN + RAISE EXCEPTION + 'rollup_is_not_a_parent: % already has children, so it cannot become the overall component', + NEW.id + USING ERRCODE = 'check_violation'; + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER rollup_is_not_a_parent + BEFORE INSERT OR UPDATE OF parent_id, is_overall ON catalog_servicecomponent + FOR EACH ROW + EXECUTE FUNCTION catalog_component_rollup_is_not_a_parent(); +""" + +DROP_TRIGGER = """ +DROP TRIGGER IF EXISTS rollup_is_not_a_parent ON catalog_servicecomponent; +DROP FUNCTION IF EXISTS catalog_component_rollup_is_not_a_parent(); +""" + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0009_history_drops_search_document"), + ] + + operations = [ + migrations.RunSQL(sql=CREATE_TRIGGER, reverse_sql=DROP_TRIGGER), + ] diff --git a/api/catalog/migrations/0011_drop_derived_ancestry_and_search.py b/api/catalog/migrations/0011_drop_derived_ancestry_and_search.py new file mode 100644 index 0000000..8a985f8 --- /dev/null +++ b/api/catalog/migrations/0011_drop_derived_ancestry_and_search.py @@ -0,0 +1,35 @@ +# Generated by Django 5.2.17 on 2026-09-04 21:48 + +from django.db import migrations + +# Ancestry and the search vector were copies of what `parent` and `name` +# already say. A copy goes stale, so each one needed a writer on every +# path that could move a component. A query answers both instead. + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0010_rollup_is_not_a_parent"), + ] + + operations = [ + migrations.RemoveField( + model_name="componentancestor", + name="ancestor", + ), + migrations.RemoveField( + model_name="componentancestor", + name="descendant", + ), + migrations.RemoveIndex( + model_name="servicecomponent", + name="components_by_search", + ), + migrations.RemoveField( + model_name="servicecomponent", + name="search_document", + ), + migrations.DeleteModel( + name="ComponentAncestor", + ), + ] diff --git a/api/catalog/migrations/0012_service_source.py b/api/catalog/migrations/0012_service_source.py new file mode 100644 index 0000000..165029f --- /dev/null +++ b/api/catalog/migrations/0012_service_source.py @@ -0,0 +1,40 @@ +from django.db import migrations, models + +# How a service reached the catalog. `created_by` says who added it, and +# this says how. Every existing row reads manual, which is true of one +# nobody imported. + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0011_drop_derived_ancestry_and_search"), + ] + + operations = [ + migrations.AddField( + model_name="historicalservice", + name="source", + field=models.CharField( + choices=[ + ("manual", "Added by hand"), + ("import", "Imported from a status page"), + ], + db_default="manual", + default="manual", + max_length=32, + ), + ), + migrations.AddField( + model_name="service", + name="source", + field=models.CharField( + choices=[ + ("manual", "Added by hand"), + ("import", "Imported from a status page"), + ], + db_default="manual", + default="manual", + max_length=32, + ), + ), + ] diff --git a/api/catalog/migrations/0013_manual_source_label.py b/api/catalog/migrations/0013_manual_source_label.py new file mode 100644 index 0000000..e484c00 --- /dev/null +++ b/api/catalog/migrations/0013_manual_source_label.py @@ -0,0 +1,38 @@ +# Generated by Django 5.2.17 on 2026-09-04 23:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0012_service_source"), + ] + + operations = [ + migrations.AlterField( + model_name="historicalservice", + name="source", + field=models.CharField( + choices=[ + ("manual", "Manual"), + ("import", "Imported from a status page"), + ], + db_default="manual", + default="manual", + max_length=32, + ), + ), + migrations.AlterField( + model_name="service", + name="source", + field=models.CharField( + choices=[ + ("manual", "Manual"), + ("import", "Imported from a status page"), + ], + db_default="manual", + default="manual", + max_length=32, + ), + ), + ] diff --git a/api/catalog/models.py b/api/catalog/models.py index c467fd6..d84a919 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -6,35 +6,34 @@ from django.utils.text import slugify from simple_history.models import HistoricalRecords -from catalog.choices import StatusPageProvider +from catalog.choices import ServiceSource, StatusPageProvider from common.models import BaseModel from common.queries import related_count class ServiceQuerySet(models.QuerySet): def for_display(self, user=None): - """Everything `ServiceSerializer` reads, without a query a row. + """Everything `ServiceSerializer` reads, in four queries. - Three fields asked per service, and one of them serialized a - component, which asked seven more. + The service page reads one service. Three of its fields are + counted or nested. The nested one is a whole component, which + asks seven more. """ from django.contrib.auth.models import AnonymousUser + from catalog.queries import component_count + tracked = models.Value(0, output_field=models.IntegerField()) if user is not None and not isinstance(user, AnonymousUser): tracked = related_count( - ServiceComponent.objects.filter(boards__owner=user), "service" + ServiceComponent.objects.visible().filter(boards__owner=user), + "service", ) return ( self.select_related("status_page", "poller") .annotate( - _component_count=related_count( - ServiceComponent.objects.filter( - is_overall=False, is_archived=False - ), - "service", - ), - _tracked_component_count=tracked, + component_count=component_count(), + tracked_component_count=tracked, ) .prefetch_related( models.Prefetch( @@ -42,7 +41,7 @@ def for_display(self, user=None): queryset=ServiceComponent.objects.filter( is_overall=True ).for_display(user), - to_attr="_overall_components", + to_attr="overall_components", ) ) ) @@ -55,10 +54,16 @@ class Service(BaseModel): blank=True, help_text="Derived from the name when left blank.", ) - description = models.TextField(blank=True, default="") logo = models.URLField(blank=True, default="") homepage_url = models.URLField(blank=True, default="") - is_featured = models.BooleanField(verbose_name="Featured", default=False) + # How the row arrived. `created_by` says who, and this says how. + # Admin only: no screen reads it, so no serializer carries it. + source = models.CharField( + max_length=32, + choices=ServiceSource.choices, + default=ServiceSource.MANUAL, + db_default=ServiceSource.MANUAL, + ) objects = ServiceQuerySet.as_manager() history = HistoricalRecords() @@ -76,35 +81,17 @@ def save(self, *args, **kwargs): def overall_component(self): """The rollup component, which is the service's own status. - This and the two below read what `for_display` prepared. A row - fetched without it asks instead. The underscore names are that - queryset's, so a rename cannot leave a reader behind. + `for_display` prefetches it. A row fetched without that asks + instead, because a single service nests one. + + The name is singular against the plural prefetch, so the two + never collide. """ - prepared = getattr(self, "_overall_components", None) + prepared = getattr(self, "overall_components", None) if prepared is None: return self.components.filter(is_overall=True).first() return prepared[0] if prepared else None - def component_count(self): - """The parts. The rollup is excluded, because it is the service.""" - prepared = getattr(self, "_component_count", None) - if prepared is not None: - return prepared - return self.components.filter(is_overall=False, is_archived=False).count() - - def tracked_component_count(self, user): - """How many parts this user has on a board.""" - if user is None or not user.is_authenticated: - return 0 - prepared = getattr(self, "_tracked_component_count", None) - if prepared is not None: - return prepared - return ( - ServiceComponent.objects.filter(service=self, boards__owner=user) - .distinct() - .count() - ) - def ensure_poller(self): """Every service is polled, so every service has one. @@ -127,9 +114,10 @@ def ensure_poller(self): # save. This is the smaller of the two. from polling.models import Poller - # Nobody adds this one, so the system account signs it. A blank - # author would read as one that was lost. - author = get_user_model().objects.system() + # A person caused the service, so the poller they spawned + # carries them. The system account signs it only when the + # service has none. A blank author reads as one that was lost. + author = self.created_by or get_user_model().objects.system() poller, _ = Poller.objects.get_or_create( service=self, defaults={"created_by": author, "updated_by": author}, @@ -200,18 +188,86 @@ def normalise_url(url: str) -> str: ) +class ServiceRequest(BaseModel): + """A status page somebody asked for and we could not read. + + One row per URL, not one per request. A row per request would + answer one question twice once a URL was triaged. + + There is no `requested_by`. `BaseModel.created_by` already holds + it, null when the asker was signed out. The v1 spec deleted + `Service.added_by` for the same reason. + + There is no state column either. Nothing closes a request yet, so + it would have no writer. + """ + + url = models.URLField(unique=True) + # The demand signal. Nothing else records how often a URL was + # asked for. This is that record, not a copy of one. + request_count = models.PositiveIntegerField(default=1) + last_requested_at = models.DateTimeField(default=timezone.now) + + class Meta(BaseModel.Meta): + ordering = ["-request_count", "-last_requested_at"] + + def __str__(self): + return self.url + + class ServiceComponentQuerySet(models.QuerySet): + def visible(self): + """The components a caller is served, anywhere. + + The provider stopped publishing an archived one. Every list, + every count, the detail and the track write read this. So no two + of them can answer the same question differently. + + It is not part of `for_display`. That queryset also prefetches a + service's rollup, and a service whose rollup is archived still + needs a header status. + """ + return self.filter(is_archived=False) + + def search(self, q): + """Narrow to what every typed word describes. + + A word matches the component's own name, its service's or its + parent's. So `twilio` reaches Twilio's SMS row, and `messaging` + reaches it through the group above it. Several words are an AND, + so `twilio sms` reaches that row and not its siblings. + + One level up, and no further. A component three deep is not + found by its grandparent's name. `parent__name` is a join read + at query time, so a rename needs nothing rewritten. Storing the + ancestry is what forced a subtree rewrite on every rename. + + The result is unordered. The list view decides what leads a + search. An ordering backend runs after this and would discard + any order set here. + """ + rows = self + for term in q.split(): + rows = rows.filter( + models.Q(name__icontains=term) + | models.Q(service__name__icontains=term) + | models.Q(parent__name__icontains=term) + ) + return rows + def for_display(self, user=None): """Everything `ComponentSerializer` reads, in four queries. It answered seven fields with a query each, per row. A page of fifty cost three hundred and fifty six. - A row that skipped this still serializes, because a single - service nests one. So the fast path has to be asked for. + Every path that serializes a component calls this. A path that + forgets raises `AttributeError` on the first count, rather than + running one query a row in silence. """ from django.contrib.auth.models import AnonymousUser + from catalog.queries import descendant_count from status.models import ComponentStatus, ServiceEvent tracked = models.Value(None, output_field=models.BooleanField()) @@ -227,8 +283,8 @@ def for_display(self, user=None): return ( self.select_related("service", "parent", "parent__parent") .annotate( - _child_count=related_count(self.model.objects, "parent"), - _is_tracked=tracked, + descendant_count=descendant_count(), + is_tracked=tracked, ) .prefetch_related( models.Prefetch( @@ -238,11 +294,14 @@ def for_display(self, user=None): queryset=ComponentStatus.objects.filter( ended_at__isnull=True ).select_related("component__service__poller"), - to_attr="_open_statuses", + to_attr="open_statuses", ), models.Prefetch( "events", queryset=ServiceEvent.objects.live(), + # The one name that keeps its underscore. Its reader + # is `live_events`, so the plain name would shadow + # the method that reads it. to_attr="_live_events", ), ) @@ -266,6 +325,11 @@ class ServiceComponent(BaseModel): status_page_order = models.IntegerField(verbose_name="Page order", default=0) is_overall = models.BooleanField(verbose_name="Overall", default=False) + # Ticked on any component. The suggested sort leads with it. A + # featured rollup surfaces its service, and a featured leaf + # surfaces that part. Severity is the next key, so it breaks the + # tie among the featured rows. + is_featured = models.BooleanField(verbose_name="Featured", default=False) # Archiving is the flag. The date is kept because `updated_at` # moves on every save, so a later rename would erase it. `save` sets # the date from the flag, and a constraint holds them together. @@ -282,6 +346,12 @@ class ServiceComponent(BaseModel): history = HistoricalRecords() + # The rollup is never a parent, but `is_overall` belongs to the + # parent row, not this one. A `CheckConstraint` sees one row, so it + # cannot read a joined column and cannot express this. Migration + # 0010 adds a trigger instead, named `rollup_is_not_a_parent`. It + # checks both directions: a row parented under the rollup, and a + # row with children gaining the flag. class Meta(BaseModel.Meta): constraints = [ models.UniqueConstraint( @@ -302,8 +372,9 @@ class Meta(BaseModel.Meta): def live_events(self, kind): """The live events of one kind. - This and the three below read what `for_display` prepared. A row - fetched without it asks instead. + This and the two below read what `for_display` prepared. Each + picks one row, or one kind, out of a prefetch. So the method + and the attribute cannot share a name. """ prepared = getattr(self, "_live_events", None) if prepared is None: @@ -312,23 +383,16 @@ def live_events(self, kind): def open_status(self): """The status span still running, if there is one.""" - prepared = getattr(self, "_open_statuses", None) + prepared = getattr(self, "open_statuses", None) if prepared is None: return self.statuses.filter(ended_at__isnull=True).first() return prepared[0] if prepared else None - def child_count(self): - """How many components sit under this one.""" - if self.is_overall: - return 0 - prepared = getattr(self, "_child_count", None) - return self.children.count() if prepared is None else prepared - def is_tracked_by(self, user): """Whether this user has it on a board. Null if nobody asked.""" if user is None or not user.is_authenticated: return None - prepared = getattr(self, "_is_tracked", None) + prepared = getattr(self, "is_tracked", None) if prepared is None: return self.boards.filter(owner=user).exists() return prepared @@ -352,9 +416,11 @@ def save(self, *args, **kwargs): def clean(self): """A component sits under one of its own service's components. - Nothing in the column says whose component the parent is. - Without this a service's tree reaches into another service's, - and a page shows one product's components under another. + Nothing in the column says whose component the parent is, or + that the parent is not the rollup. Without the first check a + service's tree reaches into another service's. Without the + second a rollup gains children, and every descendant count and + breadcrumb under it reads wrong. """ super().clean() if self.parent_id is None: @@ -365,22 +431,51 @@ def clean(self): raise ValidationError( {"parent": "That component belongs to another service."} ) + if self.parent.is_overall: + raise ValidationError( + {"parent": "The overall component is never a parent."} + ) @property def ancestors(self): """The components above this one, top down. A provider can nest a component under another. The chain is what - tells you where it sits on the status page. The guard is for bad - data: the column points at its own table, so a loop is possible. + tells you where it sits on the status page. + + Two rules end the walk. A loop is possible, because the column + points at its own table. And the chain stops at the service. + Moving a component leaves its old children pointing across. A + step there names a row this service's lists never hold. + + `catalog.queries` walks the other direction under the same two + rules. The rows here are already loaded, so this stays Python. """ chain, node, seen = [], self.parent, {self.pk} - while node is not None and node.pk not in seen: + while ( + node is not None + and node.pk not in seen + and node.service_id == self.service_id + ): seen.add(node.pk) chain.append(node) node = node.parent return list(reversed(chain)) + @property + def visible_ancestors(self): + """The ancestors a caller is served, top down. + + `ServiceComponentQuerySet.visible` is the same rule in SQL. + `ancestors` has already loaded these rows, so asking the + database again would cost a query a row. + + A dropped step makes the chain shorter, never broken. Every + other reader answers "not served" by leaving the row out. A + marker node would carry an id nothing can fetch. + """ + return [row for row in self.ancestors if not row.is_archived] + @property def path(self): """Where the component sits, read from the service down.""" diff --git a/api/catalog/queries.py b/api/catalog/queries.py index 56a8bd4..e50e7ea 100644 --- a/api/catalog/queries.py +++ b/api/catalog/queries.py @@ -1,15 +1,20 @@ -"""What a caller reads about a service, as SQL. +"""What a caller reads about the catalog, as SQL. -A service's own severity is its overall component's. Who watches it is -counted through the boards its components sit on. Both are questions -about the catalog, so they live here. +Most are a bare expression, handed to `annotate()`. Chainable +vocabulary is a queryset method, and lives on the queryset instead. + +The walk down `parent` is here too. It is raw SQL because a recursive +CTE has no ORM form, and it answers a catalog read. These lived in `common.ordering`, which made the base layer import the app it describes. """ -from django.db.models import Count, Exists, OuterRef, Subquery +from django.db import connection +from django.db.models import Count, Exists, IntegerField, OuterRef, Subquery +from django.db.models.expressions import RawSQL +from common.queries import related_count from status.models import ComponentStatus # A service's own status is the open row of its overall component. That @@ -22,12 +27,11 @@ ).values("severity")[:1] ) -# Who tracks a service, counted when it is read. It was a column that a -# signal kept true, and four write paths never reached the signal. -# -# Two readers, two shapes. Only the suggestion order needs the number, so -# only it pays for a distinct count over three joins. -WATCHER_COUNT = Count("components__boards__owner", distinct=True) +# Who tracks one component, counted when it is read. It was a column +# that a signal kept true, and four write paths never reached the +# signal. `DashboardItem` points straight at a component, so this is one +# join. +COMPONENT_WATCHER_COUNT = Count("boards__owner", distinct=True) def is_tracked(field="pk"): @@ -39,3 +43,101 @@ def is_tracked(field="pk"): from dashboards.models import DashboardItem return Exists(DashboardItem.objects.filter(component__service=OuterRef(field))) + + +def component_count(): + """How many components a service publishes, in one query for the page. + + The rollup is the service itself, and an archived row is served + nowhere. So neither is counted, and the number matches the + provider's own page. The API field and the admin column read this, + because two answers to one question drift apart. + """ + from catalog.models import ServiceComponent + + return related_count( + ServiceComponent.objects.visible().filter(is_overall=False), "service" + ) + + +# Everything under one component, walked down `parent`. `?ancestor=` +# and every descendant count ask this one question, so they read one +# definition of it. +# +# `walked` carries the ids already visited, and the step drops a row +# that is in it. `parent` points at this same table and nothing forbids +# a loop, so an unguarded walk never ends. A depth ceiling would end it +# too, but it would also cut a deep tree that is not a loop. +# +# The step tests `service_id` as well. A component's tree stops at its +# own service, which is the rule `ServiceComponent.ancestors` follows +# climbing the other way. +# +# `steps` is 0 on the component itself. No reader counts a component +# among the ones below it. +_BELOW = """ +WITH RECURSIVE below AS ( + SELECT seed.id, seed.is_archived, seed.service_id, + ARRAY[seed.id] AS walked, 0 AS steps + FROM {table} seed + WHERE seed.id = {root} + UNION ALL + SELECT step.id, step.is_archived, step.service_id, + below.walked || step.id, below.steps + 1 + FROM {table} step + JOIN below ON step.parent_id = below.id + AND step.service_id = below.service_id + WHERE NOT step.id = ANY(below.walked) +) +""" + + +def _table(): + """The component table, quoted. Read late, so no import runs early.""" + from catalog.models import ServiceComponent + + return connection.ops.quote_name(ServiceComponent._meta.db_table) + + +def _below(root, tail): + """The walk, rooted where the caller says, ending as it asks.""" + return _BELOW.format(table=_table(), root=root) + tail + + +class _CountsOwnRow(RawSQL): + """Raw SQL that reads the row it is annotated on. + + Django groups a page by every column it selects, and it cannot see + inside raw SQL. Left alone it groups by the whole subquery, which + Postgres refuses. The row's key is already grouped, and this counts + from that key alone. + """ + + def get_group_by_cols(self): + return [] + + +def descendant_ids(component_id): + """Every component under one, at any depth, as a subquery. + + `?ancestor=` reads this. An archived row is walked through and not + dropped, because the caller narrows the result to what it serves. + """ + return RawSQL(_below("%s", "SELECT id FROM below WHERE steps > 0"), [component_id]) + + +def descendant_count(): + """How many components sit under each row, in one query for the page. + + Archived rows are left out of the total. A caller is not served one + anywhere else, so a count that held them would name components no + list returns. The walk still passes through them, or an archived + group would hide the live rows under it. + """ + # The row being annotated, named by the table itself. Django never + # aliases the base table of a query, and the walk aliases its own. + sql = _below( + f"{_table()}.{connection.ops.quote_name('id')}", + "SELECT count(*) FROM below WHERE steps > 0 AND NOT is_archived", + ) + return _CountsOwnRow(f"({sql})", [], output_field=IntegerField()) diff --git a/api/catalog/serializers.py b/api/catalog/serializers.py index 402c265..36396fe 100644 --- a/api/catalog/serializers.py +++ b/api/catalog/serializers.py @@ -38,7 +38,7 @@ class Meta: fields = ["id", "slug", "name", "logo"] -class PathNodeSerializer(serializers.ModelSerializer): +class PathNodeSerializer(FieldsMixin, serializers.ModelSerializer): """One step above a component, enough to link to it. A joined string named the ancestors and gave a client nothing to @@ -54,7 +54,7 @@ class Meta: class ComponentSerializer(ViewerMixin, FieldsMixin, serializers.ModelSerializer): status = serializers.SerializerMethodField() path = serializers.SerializerMethodField() - child_count = serializers.SerializerMethodField() + descendant_count = serializers.SerializerMethodField() upcoming_maintenance = serializers.SerializerMethodField() upcoming_maintenance_count = serializers.SerializerMethodField() active_incident = serializers.SerializerMethodField() @@ -69,7 +69,7 @@ class Meta: "name", "path", "parent", - "child_count", + "descendant_count", "is_overall", "archived_at", "status", @@ -103,13 +103,21 @@ def get_path(self, row): # `ancestors` walks the chain and guards the loop a self # referencing column allows. This walked it a second time and # did not. + # + # Archived steps are left out. Each node carries an id a client + # links to, and an archived component answers 404. if row.is_overall: return [] - return PathNodeSerializer(row.ancestors, many=True).data + return PathNodeSerializer( + row.visible_ancestors, + many=True, + context=self.context, + fields_tree=self.child_tree("path"), + ).data @extend_schema_field(serializers.IntegerField()) - def get_child_count(self, row): - return row.child_count() + def get_descendant_count(self, row): + return row.descendant_count @extend_schema_field(EventRefSerializer(allow_null=True)) def get_upcoming_maintenance(self, row): @@ -152,7 +160,7 @@ def get_is_tracked(self, row): return row.is_tracked_by(self.viewer) -class ServiceSerializer(ViewerMixin, FieldsMixin, serializers.ModelSerializer): +class ServiceSerializer(FieldsMixin, serializers.ModelSerializer): status_page = StatusPageSerializer(read_only=True) poller = PollerSerializer(read_only=True) overall_component = serializers.SerializerMethodField() @@ -166,7 +174,6 @@ class Meta: "id", "slug", "name", - "description", "homepage_url", "logo", "in_catalog_since", @@ -192,18 +199,18 @@ def get_overall_component(self, service): @extend_schema_field(serializers.IntegerField()) def get_component_count(self, service): - return service.component_count() + return service.component_count @extend_schema_field(serializers.IntegerField()) def get_tracked_component_count(self, service): - return service.tracked_component_count(self.viewer) + return service.tracked_component_count class ImportRequestSerializer(serializers.Serializer): """The body of POST /catalog/import/. - Named `status_page_url`, not `url`, because that is what the contract - documents and the contract is what the client is written against. + Named `status_page_url`, not `url`. The contract names it that way, + and the client follows the contract. """ status_page_url = serializers.URLField() @@ -222,3 +229,9 @@ def validate_status_page_url(self, value): except BlockedAddress as error: raise serializers.ValidationError(str(error)) from error return value + + +class ServiceRequestSerializer(serializers.Serializer): + """The body of POST /catalog/requests/.""" + + url = serializers.URLField() diff --git a/api/catalog/urls.py b/api/catalog/urls.py index 0e6c916..641e318 100644 --- a/api/catalog/urls.py +++ b/api/catalog/urls.py @@ -1,15 +1,31 @@ -from django.urls import include, path -from rest_framework.routers import SimpleRouter +from django.urls import path -from catalog.views import CatalogImportView, ServiceViewSet - -router = SimpleRouter(trailing_slash=True) -# basename "service" gives service-list, service-detail, and one route per -# detail action: service-components and service-events. -router.register("catalog/services", ServiceViewSet, basename="service") +from catalog.views.components import ComponentDetailView, ComponentListView +from catalog.views.imports import CatalogImportView +from catalog.views.requests import ServiceRequestView +from catalog.views.services import ServiceDetailView urlpatterns = [ - # Above the router so its slug route does not shadow it. path("catalog/import/", CatalogImportView.as_view(), name="catalog-import"), - path("", include(router.urls)), + path( + "catalog/requests/", + ServiceRequestView.as_view(), + name="catalog-request", + ), + # Above the service route, so `components` is never read as a slug. + path( + "catalog/components/", + ComponentListView.as_view(), + name="component-list", + ), + path( + "catalog/components//", + ComponentDetailView.as_view(), + name="component-detail", + ), + path( + "catalog/services//", + ServiceDetailView.as_view(), + name="service-detail", + ), ] diff --git a/api/catalog/views.py b/api/catalog/views.py deleted file mode 100644 index 0f2398f..0000000 --- a/api/catalog/views.py +++ /dev/null @@ -1,175 +0,0 @@ -import requests -from django_filters.rest_framework import DjangoFilterBackend -from drf_spectacular.utils import OpenApiResponse, extend_schema -from rest_framework import status as http -from rest_framework.decorators import action -from rest_framework.filters import SearchFilter -from rest_framework.permissions import AllowAny -from rest_framework.response import Response -from rest_framework.views import APIView -from rest_framework.viewsets import ReadOnlyModelViewSet - -from api.defaults import Throttle -from catalog.filters import ComponentFilter, ServiceEventFilter, ServiceFilter -from catalog.models import Service, ServiceComponent -from catalog.queries import OVERALL_SEVERITY, WATCHER_COUNT -from catalog.serializers import ( - ComponentSerializer, - ImportRequestSerializer, - ServiceSerializer, -) -from common.aggregates import EventAggregateSet, StatusAggregateSet -from common.errors import NoStatusPageFound, ProviderUnreachable -from common.filters import FieldsBackend -from common.ordering import MappedOrderingFilter -from polling.importer import import_from_url -from status.models import ServiceEvent -from status.queries import CURRENT_SEVERITY -from status.serializers import ServiceEventSerializer - - -class ServiceAggregateSet(StatusAggregateSet): - """Counts for a list of services. - - A service has no severity of its own. Its severity is the open status - of its overall component, which is the provider's own page-level - reading. So counting services by severity means counting those - components, not the services. - """ - - def components(self): - return ServiceComponent.objects.filter( - service__in=self.queryset, is_overall=True - ) - - -class ServiceViewSet(ReadOnlyModelViewSet): - """Every read of a service, in one place. - - The list and the detail are the same resource. A component list and - an event list belong to a service, so they are detail actions. - - Each action overrides the serializer, filters and ordering it needs. - Those names exist on the class because DRF refuses an initkwarg that - is not already an attribute. - """ - - permission_classes = [AllowAny] - lookup_field = "slug" - serializer_class = ServiceSerializer - aggregate_set = ServiceAggregateSet - filterset_class = ServiceFilter - filter_backends = [ - DjangoFilterBackend, - SearchFilter, - MappedOrderingFilter, - FieldsBackend, - ] - search_fields = ["name", "slug"] - ordering_fields = ["name", "updated_at"] - # `suggested` is not a field. Severity sits behind a related path. - # - # Severity ahead of popularity is deliberate. A middling service - # that is broken now beats a popular one that is fine. - # - # Lower severity is worse, so ascending puts the broken first. A - # service with no reading sorts last, not as healthy. - SUGGESTED = ["-is_featured", "severity_now", "-watcher_count", "name"] - ordering_map = { - "suggested": SUGGESTED, - "overall_component__status__severity": ["severity_now"], - } - ordering = SUGGESTED - # Schema generation calls get_queryset with no URL kwargs. This names - # the model without running it. - queryset = Service.objects.none() - - def get_queryset(self): - # `q` is SearchFilter's, through SEARCH_PARAM. It was a hand - # rolled `name__icontains` beside a `search_fields` that no - # backend read, so the slug was declared searchable and was not. - return Service.objects.for_display(self.request.user).annotate( - watcher_count=WATCHER_COUNT, - severity_now=OVERALL_SEVERITY, - ) - - def _page(self, queryset): - """The same envelope the list actions return.""" - page = self.paginate_queryset(self.filter_queryset(queryset)) - return self.get_paginated_response(self.get_serializer(page, many=True).data) - - @extend_schema(responses={200: ComponentSerializer}) - @action( - detail=True, - url_path="components", - url_name="components", - serializer_class=ComponentSerializer, - aggregate_set=StatusAggregateSet, - filterset_class=ComponentFilter, - ordering_fields=["name", "status_page_order", "updated_at"], - ordering_map={"status__severity": ["severity_now"]}, - ordering=["status_page_order"], - ) - def components(self, request, slug=None): - return self._page( - ServiceComponent.objects.filter(service__slug=slug) - .for_display(self.request.user) - .annotate(severity_now=CURRENT_SEVERITY) - ) - - @extend_schema(responses={200: ServiceEventSerializer}) - @action( - detail=True, - url_path="events", - url_name="events", - serializer_class=ServiceEventSerializer, - aggregate_set=EventAggregateSet, - filterset_class=ServiceEventFilter, - ordering_fields=["starts_at", "ends_at"], - ordering_map={}, - ordering=["-starts_at"], - ) - def events(self, request, slug=None): - return self._page( - ServiceEvent.objects.filter(service__slug=slug).prefetch_related("updates") - ) - - -class CatalogImportView(APIView): - """`POST /catalog/services/` is reserved. - - A future admin create takes a service body. A bulk create takes a - list. So importing by URL gets its own path. It does not overload - the standard collection. - """ - - permission_classes = [AllowAny] - # Every call makes the server fetch a URL somebody else chose. - throttle_scope = Throttle.IMPORT - - @extend_schema( - request=ImportRequestSerializer, - responses={ - 200: ServiceSerializer, - 201: ServiceSerializer, - 400: OpenApiResponse(description="Missing or malformed status_page_url."), - }, - ) - def post(self, request): - body = ImportRequestSerializer(data=request.data) - if not body.is_valid(): - return Response(body.errors, status=400) - url = body.validated_data["status_page_url"] - try: - service, created = import_from_url(url) - except ValueError as error: - # `identify` raises this when no adapter could read the page. - # It is the ordinary outcome of pasting the wrong address, - # and it used to reach the caller as a 500. - raise NoStatusPageFound(str(error)) from error - except requests.RequestException as error: - raise ProviderUnreachable(f"{url} could not be read: {error}") from error - return Response( - ServiceSerializer(service, context={"request": request}).data, - status=http.HTTP_201_CREATED if created else http.HTTP_200_OK, - ) diff --git a/api/catalog/views/__init__.py b/api/catalog/views/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/catalog/views/components.py b/api/catalog/views/components.py new file mode 100644 index 0000000..d016a75 --- /dev/null +++ b/api/catalog/views/components.py @@ -0,0 +1,99 @@ +"""Every read of a component, in one place. + +Discover, a service's Components tab, a component's descendants and an +event's affected list are one collection with different parameters. Four +nested routes would have been four copies of this queryset. +""" + +from django_filters.rest_framework import DjangoFilterBackend +from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_view +from rest_framework import generics +from rest_framework.permissions import AllowAny + +from catalog.filters import ComponentFilter +from catalog.models import ServiceComponent +from catalog.queries import COMPONENT_WATCHER_COUNT +from catalog.serializers import ComponentSerializer +from common.aggregates import StatusAggregateSet +from common.filters import REJECTED_PARAMETER, FieldsBackend +from common.ordering import MappedOrderingFilter +from common.serializers import ErrorSerializer +from status.queries import CURRENT_SEVERITY + + +class ComponentQueryMixin: + permission_classes = [AllowAny] + serializer_class = ComponentSerializer + # get_queryset reads the request, which schema generation has not + # set. This names the model without running it. + queryset = ServiceComponent.objects.none() + + def get_queryset(self): + return ( + ServiceComponent.objects.visible() + .for_display(self.request.user) + .annotate( + severity_now=CURRENT_SEVERITY, + watcher_count=COMPONENT_WATCHER_COUNT, + ) + .distinct() + ) + + +@extend_schema_view( + get=extend_schema( + responses={ + 200: ComponentSerializer, + 400: OpenApiResponse(description=REJECTED_PARAMETER), + } + ) +) +class ComponentListView(ComponentQueryMixin, generics.ListAPIView): + aggregate_set = StatusAggregateSet + filterset_class = ComponentFilter + filter_backends = [DjangoFilterBackend, MappedOrderingFilter, FieldsBackend] + ordering_fields = ["name", "status_page_order", "updated_at"] + # Severity ahead of popularity, the same as the service sort this + # replaces. Lower severity is worse, so ascending puts the broken + # first. + SUGGESTED = ["-is_featured", "severity_now", "-watcher_count", "name"] + ordering_map = { + "suggested": SUGGESTED, + "status__severity": ["severity_now"], + } + ordering = SUGGESTED + + def initial(self, request, *args, **kwargs): + """Pick the default sort from what the caller asked for. + + Both branches set a default, and neither overrides anybody. + `?ordering=` is read after this and wins when it is present. + + DRF reads `ordering` as an attribute, so a method cannot supply + it. It does not check a default against `ordering_fields` + either, so neither branch below needs an entry there. + + There is one sort control and one label, Smart, on every list. A + separate "best match" would name a ranking the search does not + work out. + """ + super().initial(request, *args, **kwargs) + if request.query_params.get("q"): + # The rollup leads, then the suggested keys. Typing a + # service's name matches every part of it. Without this, + # eighty rows bury the one that answers the question. + self.ordering = ["-is_overall", *self.SUGGESTED] + elif request.query_params.get("service"): + # A service tab reads down the provider's own page. + self.ordering = ["status_page_order"] + + +@extend_schema_view( + get=extend_schema(responses={200: ComponentSerializer, 404: ErrorSerializer}) +) +class ComponentDetailView(ComponentQueryMixin, generics.RetrieveAPIView): + filter_backends = [FieldsBackend] + lookup_field = "pk" + # The contract names the path variable `uuid`. DRF renames a `pk` + # one to `id`, which would document a path nothing serves. + lookup_url_kwarg = "uuid" diff --git a/api/catalog/views/imports.py b/api/catalog/views/imports.py new file mode 100644 index 0000000..7702d01 --- /dev/null +++ b/api/catalog/views/imports.py @@ -0,0 +1,76 @@ +import requests +from drf_spectacular.utils import OpenApiResponse, extend_schema +from rest_framework import status as http +from rest_framework.permissions import AllowAny +from rest_framework.response import Response +from rest_framework.views import APIView + +from api.defaults import Throttle +from catalog.models import Service +from catalog.serializers import ImportRequestSerializer, ServiceSerializer +from common.errors import NoStatusPageFound, ProviderUnreachable +from common.serializers import ErrorSerializer +from polling.importer import import_from_url + + +class CatalogImportView(APIView): + """Add a service by pasting the address of its status page. + + Its own path, because the body is a URL rather than a service. A + later create that takes a service body then does not overload this + one. + """ + + permission_classes = [AllowAny] + # Every call makes the server fetch a URL somebody else chose. + throttle_scope = Throttle.IMPORT + + @extend_schema( + request=ImportRequestSerializer, + responses={ + 200: ServiceSerializer, + 201: ServiceSerializer, + # Two shapes under one code. A malformed body answers the + # serializer's field map. A URL no adapter can read answers + # the named error. The contract carries both. + 400: OpenApiResponse( + description=( + "Missing or malformed status_page_url, or no status page " + "found at it." + ) + ), + 429: OpenApiResponse( + response=ErrorSerializer, description="Too many requests." + ), + 502: OpenApiResponse( + response=ErrorSerializer, + description="The provider could not be reached.", + ), + }, + ) + def post(self, request): + body = ImportRequestSerializer(data=request.data) + if not body.is_valid(): + return Response(body.errors, status=400) + url = body.validated_data["status_page_url"] + # Anonymous stays anonymous, as it does on a service request. + # The system account means automated, and a person asked for + # this one. Null says a person we cannot name. + author = request.user if request.user.is_authenticated else None + try: + service, created = import_from_url(url, author=author) + except ValueError as error: + # `identify` raises this when no adapter could read the page. + # It is the ordinary outcome of pasting the wrong address, + # and it used to reach the caller as a 500. + raise NoStatusPageFound(str(error)) from error + except requests.RequestException as error: + raise ProviderUnreachable(f"{url} could not be read: {error}") from error + # Read again, after the import. `ServiceSerializer` reads the + # counts `for_display` annotates, and the import returns a + # plain row. + row = Service.objects.for_display(request.user).get(pk=service.pk) + return Response( + ServiceSerializer(row, context={"request": request}).data, + status=http.HTTP_201_CREATED if created else http.HTTP_200_OK, + ) diff --git a/api/catalog/views/requests.py b/api/catalog/views/requests.py new file mode 100644 index 0000000..9180f93 --- /dev/null +++ b/api/catalog/views/requests.py @@ -0,0 +1,60 @@ +from django.db.models import F +from django.utils import timezone +from drf_spectacular.utils import OpenApiResponse, extend_schema +from rest_framework import status as http +from rest_framework.permissions import AllowAny +from rest_framework.response import Response +from rest_framework.views import APIView + +from api.defaults import Throttle +from catalog.models import ServiceRequest, StatusPage +from catalog.serializers import ServiceRequestSerializer +from common.serializers import ErrorSerializer + + +class ServiceRequestView(APIView): + """ "Send this URL to us", from the Add-by-URL not-found screen. + + An import stores nothing about an attempt that failed, so this is + where a dead end goes. Telling somebody to hunt for a better link + assumes they have not already tried. + """ + + permission_classes = [AllowAny] + # An anonymous write. Without this one person could inflate the + # demand signal the admin list is ordered by. + # + # Its own scope, not IMPORT's. A failed import and the report that + # follows it share one flow. One bucket would let the import spend + # the report's budget. + throttle_scope = Throttle.SERVICE_REQUEST + + @extend_schema( + request=ServiceRequestSerializer, + responses={ + 202: OpenApiResponse(description="Recorded."), + 400: OpenApiResponse(description="Missing or malformed url."), + 429: OpenApiResponse( + response=ErrorSerializer, description="Too many requests." + ), + }, + ) + def post(self, request): + body = ServiceRequestSerializer(data=request.data) + if not body.is_valid(): + return Response(body.errors, status=400) + url = StatusPage.normalise_url(body.validated_data["url"]) + author = request.user if request.user.is_authenticated else None + row, created = ServiceRequest.objects.get_or_create( + url=url, defaults={"created_by": author, "updated_by": author} + ) + if not created: + # F() rather than a read and a write. Two people asking at + # once would otherwise both write 2. + ServiceRequest.objects.filter(pk=row.pk).update( + request_count=F("request_count") + 1, + last_requested_at=timezone.now(), + ) + # Always 202, and always the same body. Anything else would + # tell a stranger which URLs the catalog already holds. + return Response(status=http.HTTP_202_ACCEPTED) diff --git a/api/catalog/views/services.py b/api/catalog/views/services.py new file mode 100644 index 0000000..5736fa3 --- /dev/null +++ b/api/catalog/views/services.py @@ -0,0 +1,31 @@ +from drf_spectacular.utils import extend_schema, extend_schema_view +from rest_framework import generics +from rest_framework.permissions import AllowAny + +from catalog.models import Service +from catalog.serializers import ServiceSerializer +from common.filters import FieldsBackend +from common.serializers import ErrorSerializer + + +@extend_schema_view( + get=extend_schema(responses={200: ServiceSerializer, 404: ErrorSerializer}) +) +class ServiceDetailView(generics.RetrieveAPIView): + """The service page's header and its About tab. + + There is no list. Discover searches components, and the signed-out + board lists overall components. So nothing asked a service + collection a question. + """ + + permission_classes = [AllowAny] + lookup_field = "slug" + serializer_class = ServiceSerializer + filter_backends = [FieldsBackend] + # get_queryset reads the request, which schema generation has not + # set. This names the model without running it. + queryset = Service.objects.none() + + def get_queryset(self): + return Service.objects.for_display(self.request.user) diff --git a/api/common/admin.py b/api/common/admin.py index 5eb78bd..fda6cd1 100644 --- a/api/common/admin.py +++ b/api/common/admin.py @@ -343,7 +343,7 @@ class PollerWrittenAdmin(BaseModelAdmin): Only the three permissions are refused. Django still renders the detail page read-only, so a poll error stays readable. - Seed a database with `just seed-dev`, which polls for real. + Seed a database with `just seed-live`, which polls for real. """ def has_add_permission(self, request, obj=None): diff --git a/api/common/aggregates.py b/api/common/aggregates.py index 5d868f5..65f05d2 100644 --- a/api/common/aggregates.py +++ b/api/common/aggregates.py @@ -1,6 +1,4 @@ -from django.db.models import Count, Exists, Min, OuterRef - -from status.choices import EventKind +from django.db.models import Count, Min class AggregateSet: @@ -22,62 +20,28 @@ def build(self): class StatusAggregateSet(AggregateSet): - """For any list of things carrying a status. Services, or a board.""" + """For any list of components. Discover, a service's parts, a board.""" def build(self): data = super().build() data["by_severity"] = self._by_severity() - data["by_event_kind"] = self._by_event_kind() data["next_refresh_at"] = self._next_refresh_at() data["oldest_refreshed_at"] = self._oldest_refreshed_at() return data - def components(self): - """The components these counts are drawn from. - - Severity and events hang off a component, so every count below - starts here. When the rows are already components this is them. - A list of something else overrides it to say which components - stand for its rows. - """ - return self.queryset - def _by_severity(self): rows = ( - self.components() - .filter(statuses__ended_at__isnull=True) + self.queryset.filter(statuses__ended_at__isnull=True) .values("statuses__severity") .annotate(n=Count("id", distinct=True)) ) return {str(r["statuses__severity"]): r["n"] for r in rows} - def _by_event_kind(self): - """How many components have a live event of each kind. - - Through `ServiceEvent.objects.live`, the same rule the component - beside it counts by. This had its own copy, and it left out the - end of a maintenance window. A chip and a row could disagree - about one that was over. - """ - from status.models import ServiceEvent - - counts = {} - for kind in EventKind: - live = ServiceEvent.objects.live(kind).filter( - affected_components=OuterRef("pk") - ) - counts[str(kind)] = ( - self.components().filter(Exists(live)).distinct().count() - ) - return counts - def _next_refresh_at(self): - return self.components().aggregate(v=Min("service__poller__next_at"))["v"] + return self.queryset.aggregate(v=Min("service__poller__next_at"))["v"] def _oldest_refreshed_at(self): - return self.components().aggregate(v=Min("service__poller__last_success_at"))[ - "v" - ] + return self.queryset.aggregate(v=Min("service__poller__last_success_at"))["v"] class EventAggregateSet(AggregateSet): diff --git a/api/common/errors.py b/api/common/errors.py index b27cdae..68c152a 100644 --- a/api/common/errors.py +++ b/api/common/errors.py @@ -2,7 +2,7 @@ `ErrorSerializer` and the codes were declared and documented from the start, and nothing raised them. Three different shapes came back -instead: DRF's `detail`, a field map from a serializer, and an +instead. DRF's `detail`, a field map from a serializer, and an uncaught exception as a 500. A client is generated from the contract, so it looks for `code` and @@ -49,8 +49,8 @@ def _code(exception): return exception.code if isinstance(exception, Throttled): return ErrorCode.THROTTLED - # Django raises its own 404 and DRF translates it, so the exception - # reaching here is `Http404` rather than DRF's `NotFound`. + # Django raises its own 404, and DRF translates it. The exception + # reaching here is `Http404`, not DRF's `NotFound`. if isinstance(exception, NotFound | Http404): return ErrorCode.NOT_FOUND return None diff --git a/api/common/filters.py b/api/common/filters.py index 29ae67d..c349282 100644 --- a/api/common/filters.py +++ b/api/common/filters.py @@ -1,3 +1,4 @@ +from django_filters import rest_framework as filters from rest_framework.filters import BaseFilterBackend FIELDS_DESCRIPTION = ( @@ -5,6 +6,14 @@ "nested object rather than dropping it." ) +# One description for the one 400 a list answers. `fields`, `ordering` +# and every filter refuse the same way, so a client branches once. +REJECTED_PARAMETER = ( + "A query parameter names something this list does not serve, or " + "carries a value it cannot read. The body is a field-errors object, " + "keyed by the parameter." +) + class FieldsBackend(BaseFilterBackend): """Declare `?fields=` for drf-spectacular. @@ -27,3 +36,28 @@ def get_schema_operation_parameters(self, view): "schema": {"type": "string"}, } ] + + +class SeverityFilterMixin(filters.FilterSet): + """How a caller narrows a list of components by severity. + + Declared, not generated. A component has a history of statuses and + the open one is current. So there is no `status` relation to + generate from. The contract's name points at the `severity_now` + annotation each view supplies instead. + + A `FilterSet` rather than a plain mixin. django-filter collects + declared filters only from a base that already carries them. It + names no model, so `common` still imports no app. A subclass brings + its own `Meta`. + """ + + status__severity = filters.NumberFilter(field_name="severity_now") + status__severity__lte = filters.NumberFilter( + field_name="severity_now", lookup_expr="lte" + ) + # The Severity filter offers all six values at once. One exact match + # cannot serve it. + status__severity__in = filters.BaseInFilter( + field_name="severity_now", lookup_expr="in" + ) diff --git a/api/common/mixins.py b/api/common/mixins.py index ea9b8af..edac56a 100644 --- a/api/common/mixins.py +++ b/api/common/mixins.py @@ -1,3 +1,5 @@ +from rest_framework.serializers import ValidationError + _UNSET = object() @@ -51,7 +53,30 @@ def _parse(raw): branch.add(tail) return tree + def reject_unknown(self, names, known): + """Refuse a path that names nothing, for a reader that prunes itself. + + A name nothing serves used to pop every field and answer + `200 {}`. A typo, or a field a later change renames, then read + as a server with nothing to say. + + `_prune` recurses into a nested serializer. A plain dict and a + method field hold no `fields`, so the recursion stops before + them. Such a reader filters by hand and calls this. One rule + then answers for every place a dotted path is accepted. + """ + unknown = sorted(set(names) - set(known)) + if unknown: + raise ValidationError( + { + self.fields_param or "fields": [ + f"Unknown field: {name}." for name in unknown + ] + } + ) + def _prune(self, tree): + self.reject_unknown(tree, self.fields) for name in set(self.fields) - set(tree): self.fields.pop(name) for name, children in tree.items(): diff --git a/api/common/ordering.py b/api/common/ordering.py index 78d3fbb..5283d23 100644 --- a/api/common/ordering.py +++ b/api/common/ordering.py @@ -5,6 +5,7 @@ importing `catalog` and `status` back. """ +from rest_framework.exceptions import ValidationError from rest_framework.filters import OrderingFilter @@ -13,6 +14,10 @@ class MappedOrderingFilter(OrderingFilter): A cursor cannot order on a related path or on a name that is not a field. A view declares `ordering_map` and annotates the flat field it names. + + A value the view does not declare answers 400, the way `?fields=` + refuses an unknown name. Dropping it left the caller reading a list + sorted by something else, with nothing to say so. """ def remove_invalid_fields(self, queryset, fields, view, request): @@ -20,9 +25,17 @@ def remove_invalid_fields(self, queryset, fields, view, request): allowed = set(getattr(view, "ordering_fields", ())) | set(mapping) out = [] for term in fields: - name = term.lstrip("-") + name = term.removeprefix("-") if name not in allowed: - continue - for mapped in mapping.get(name, [name]): - out.append(f"-{mapped}" if term.startswith("-") else mapped) + raise ValidationError({"ordering": [f"Unknown ordering: {term}."]}) + mapped = mapping.get(name, [name]) + if not term.startswith("-"): + out.extend(mapped) + elif len(mapped) == 1 and not mapped[0].startswith("-"): + out.append(f"-{mapped[0]}") + else: + # An editorial order already names its own directions. + # A second prefix built `--is_featured`, which reached + # the ORM as a 500. + raise ValidationError({"ordering": [f"Cannot reverse: {term}."]}) return out diff --git a/api/common/pagination.py b/api/common/pagination.py index acd116f..aa76b78 100644 --- a/api/common/pagination.py +++ b/api/common/pagination.py @@ -32,3 +32,14 @@ def get_paginated_response(self, data): "results": data, } ) + + +class TimelinePagination(EnvelopePagination): + """One event's log, oldest first. + + `posted_at` is not unique on its own, so `created_at` breaks a tie + between two updates posted in the same second. + """ + + ordering = "posted_at" + tiebreak = "created_at" diff --git a/api/common/queries.py b/api/common/queries.py index 59ab896..ddc88a1 100644 --- a/api/common/queries.py +++ b/api/common/queries.py @@ -14,12 +14,16 @@ def related_count(queryset, group_by, ref="pk"): Several counts on one row multiply into one another. Each is a join, so a service is read once per component per event. Each count asks its own question instead. + + Distinct, because the caller may hand in a queryset that already + joins. A component on two of one person's boards is one component, + and it arrived twice. """ counted = ( queryset.filter(**{group_by: OuterRef(ref)}) .order_by() .values(group_by) - .annotate(total=Count("pk")) + .annotate(total=Count("pk", distinct=True)) .values("total") ) return Coalesce(Subquery(counted, output_field=IntegerField()), 0) diff --git a/api/common/serializers.py b/api/common/serializers.py index 43cfb22..e034254 100644 --- a/api/common/serializers.py +++ b/api/common/serializers.py @@ -1,7 +1,10 @@ from enum import StrEnum +from drf_spectacular.utils import extend_schema_field from rest_framework import serializers +from common.mixins import FieldsMixin + class ErrorCode(StrEnum): """What a failure calls itself. @@ -25,13 +28,35 @@ class ErrorSerializer(serializers.Serializer): detail = serializers.CharField() -class MetaSerializer(serializers.Serializer): +class MetaSerializer(FieldsMixin, serializers.Serializer): """Deployment-wide configuration. Nothing here is a user preference.""" poll_interval_seconds = serializers.IntegerField() poll_cooldown_seconds = serializers.IntegerField() default_page_size = serializers.IntegerField() max_page_size = serializers.IntegerField() - enums = serializers.DictField( - help_text="Label maps for every enum, keyed by enum name." + enums = serializers.SerializerMethodField() + + @extend_schema_field( + serializers.DictField( + help_text="Label maps for every enum, keyed by enum name." + ) ) + def get_enums(self, obj): + """Keep only the enums a dotted `?fields=enums.` names. + + `enums` is a plain dict, not a nested serializer, so + `FieldsMixin` cannot prune inside it on its own. This reads the + branch under "enums" and filters by hand instead. `MetaView` + still names every enum in the one place it always has. + + `reject_unknown` is the same refusal `_prune` makes. Without it + a typed enum name answered `200 {"enums": {}}`, which is the + empty payload the 400 exists to prevent. + """ + enums = obj["enums"] + wanted = self.child_tree("enums") + if not wanted: + return enums + self.reject_unknown(wanted, enums) + return {name: labels for name, labels in enums.items() if name in wanted} diff --git a/api/common/views.py b/api/common/views.py index 9f59509..fcfd7f2 100644 --- a/api/common/views.py +++ b/api/common/views.py @@ -9,7 +9,14 @@ from catalog.choices import StatusPageProvider from common.serializers import MetaSerializer -from status.choices import EVENT_PHASES_BY_KIND, EventKind, Severity, StatusSource +from status.choices import ( + EVENT_PHASES_BY_KIND, + EventKind, + EventPhaseState, + EventSource, + Severity, + StatusSource, +) def _labels(choices): @@ -41,24 +48,27 @@ class MetaView(APIView): @extend_schema(responses={200: MetaSerializer}) def get(self, request): - return Response( - { - "poll_interval_seconds": settings.POLL_INTERVAL_SECONDS, - "poll_cooldown_seconds": settings.POLL_COOLDOWN_SECONDS, - "default_page_size": settings.DEFAULT_PAGE_SIZE, - "max_page_size": settings.MAX_PAGE_SIZE, - "enums": { - "severity": _labels(Severity), - "status_source": _labels(StatusSource), - "status_page_provider": _labels(StatusPageProvider), - "event_kind": _labels(EventKind), - "event_phase": { - str(kind): _labels(phases) - for kind, phases in EVENT_PHASES_BY_KIND.items() - }, + data = { + "poll_interval_seconds": settings.POLL_INTERVAL_SECONDS, + "poll_cooldown_seconds": settings.POLL_COOLDOWN_SECONDS, + "default_page_size": settings.DEFAULT_PAGE_SIZE, + "max_page_size": settings.MAX_PAGE_SIZE, + "enums": { + "severity": _labels(Severity), + "status_source": _labels(StatusSource), + "status_page_provider": _labels(StatusPageProvider), + "event_kind": _labels(EventKind), + "event_source": _labels(EventSource), + "event_phase": { + str(kind): _labels(phases) + for kind, phases in EVENT_PHASES_BY_KIND.items() }, - } - ) + "event_phase_state": _labels(EventPhaseState), + }, + } + # `Response(data)` skips `MetaSerializer`, so `?fields=` never + # pruned. `FieldsMixin` prunes at construction, on the instance. + return Response(MetaSerializer(data, context={"request": request}).data) class ApiDocsView(TemplateView): diff --git a/api/dashboards/admin.py b/api/dashboards/admin.py index 791a355..6d59387 100644 --- a/api/dashboards/admin.py +++ b/api/dashboards/admin.py @@ -31,6 +31,25 @@ class DashboardItemInline(TabularInline): ) +class HoldsItemsFilter(admin.SimpleListFilter): + """Whether a board holds anything. + + The count is a column and was not a filter. An empty board is what + a person opens the list to find. + """ + + title = _("Tracked") + parameter_name = "holds" + + def lookups(self, request, model_admin): + return [("1", _("Holds components")), ("0", _("Empty"))] + + def queryset(self, request, queryset): + if self.value() is None: + return queryset + return queryset.filter(items__isnull=self.value() == "0").distinct() + + class DefaultBoardFilter(admin.SimpleListFilter): """Whether a board is the one its owner opens. @@ -54,17 +73,24 @@ def queryset(self, request, queryset): class DashboardAdmin(ScopedAutocompleteMixin, BaseModelAdmin, ModelAdmin): # A person picking their default board is offered their own. autocomplete_scope = ("owner",) - list_display = ["display_board", "display_owner", "display_default", "item_count"] + # No `default` column. The pointer lives on the user, as + # `default_dashboard`. It is a fact about the owner, not the board. + # It stays on the form, and its filter stays. + list_display = ["display_board", "display_owner", "item_count"] search_fields = ["name", "owner__email"] list_filter = [ ("owner", AutocompleteSelectFilter), DefaultBoardFilter, + HoldsItemsFilter, ("created_at", RangeDateTimeFilter), ] autocomplete_fields = ["owner"] inlines = [DashboardItemInline] + # Both are read from elsewhere. The default is a pointer on the + # owner, and the count is the rows below. + readonly_fields = ["display_default", "item_count"] fieldsets = [ - (None, {"fields": ["owner", "name"]}), + (None, {"fields": ["owner", "name", "display_default", "item_count"]}), audit_section(), ] diff --git a/api/dashboards/filters.py b/api/dashboards/filters.py index d5271a9..0a6dc9f 100644 --- a/api/dashboards/filters.py +++ b/api/dashboards/filters.py @@ -1,37 +1,8 @@ -from django.db.models import Exists, OuterRef -from django_filters import rest_framework as filters - from catalog.models import ServiceComponent -from status.choices import CLOSED_PHASES, EventKind -from status.models import ServiceEvent - - -class BoardComponentFilter(filters.FilterSet): - """`event` is declared, not generated. +from common.filters import SeverityFilterMixin - `ServiceEvent` and `ServiceComponent` are many-to-many. - django-filter applies each parameter as its own `.filter()` call. - On a many-to-many, each call joins again and can match a different event. - Both conditions must sit in one call, so this is a single subquery. - """ - - event = filters.ChoiceFilter(choices=EventKind.choices, method="filter_event") - - # Declared for the same reason as the catalog's. A component has a - # history of `statuses`. The open one is current. So there is no - # `status` relation to generate from. The contract's name points at - # the annotation instead. - status__severity = filters.NumberFilter(field_name="severity_now") - status__severity__lte = filters.NumberFilter( - field_name="severity_now", lookup_expr="lte" - ) +class BoardComponentFilter(SeverityFilterMixin): class Meta: model = ServiceComponent fields = [] - - def filter_event(self, queryset, name, value): - live = ServiceEvent.objects.filter( - kind=value, affected_components=OuterRef("pk") - ).exclude(phase__in=CLOSED_PHASES) - return queryset.filter(Exists(live)) diff --git a/api/dashboards/views.py b/api/dashboards/views.py index 8275d2a..5e11736 100644 --- a/api/dashboards/views.py +++ b/api/dashboards/views.py @@ -1,6 +1,6 @@ from django.shortcuts import get_object_or_404 from django_filters.rest_framework import DjangoFilterBackend -from drf_spectacular.utils import OpenApiResponse, extend_schema +from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_view from rest_framework import generics, status from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response @@ -9,7 +9,7 @@ from catalog.models import ServiceComponent from catalog.serializers import ComponentSerializer from common.aggregates import StatusAggregateSet -from common.filters import FieldsBackend +from common.filters import REJECTED_PARAMETER, FieldsBackend from common.ordering import MappedOrderingFilter from dashboards.filters import BoardComponentFilter from dashboards.models import Dashboard, DashboardItem @@ -22,9 +22,31 @@ def _board(request, uuid): return get_object_or_404(Dashboard, id=uuid, owner=request.user) +# On `post`, not on `create`. The generator reads the handler the router +# dispatches to. An annotation on `create` was dropped in silence, so +# the documented 200 and 404 reached no schema. +@extend_schema_view( + get=extend_schema( + responses={ + 200: ComponentSerializer, + 400: OpenApiResponse(description=REJECTED_PARAMETER), + } + ), + post=extend_schema( + request=TrackComponentSerializer, + responses={ + 201: ComponentSerializer, + 200: OpenApiResponse( + response=ComponentSerializer, + description="Already tracked. Tracking twice is not an error.", + ), + 404: OpenApiResponse(description="No such board, or no such component."), + }, + ), +) class BoardComponentListView(generics.ListCreateAPIView): - # get_queryset reads self.kwargs, which schema generation has - # not set. This names the model without running it. + # Schema generation reads the model off this before it calls + # get_queryset. It names the model without running one. queryset = ServiceComponent.objects.none() permission_classes = [IsAuthenticated] serializer_class = ComponentSerializer @@ -40,37 +62,43 @@ class BoardComponentListView(generics.ListCreateAPIView): ordering = ["severity_now"] # worst first: lower severity is worse def get_queryset(self): + rows = ServiceComponent.objects.visible().annotate( + severity_now=CURRENT_SEVERITY, next_transition=NEXT_TRANSITION + ) + if "uuid" not in self.kwargs: + # Schema generation reaches here with no URL. It reads the + # annotations, because the filters and the sorts name them. + return rows.none() board = _board(self.request, self.kwargs["uuid"]) return ( - ServiceComponent.objects.filter(tracked_by__dashboard=board) + rows.filter(tracked_by__dashboard=board) .for_display(self.request.user) - .annotate(severity_now=CURRENT_SEVERITY, next_transition=NEXT_TRANSITION) .distinct() ) - @extend_schema( - request=TrackComponentSerializer, - responses={ - 201: ComponentSerializer, - 200: OpenApiResponse( - response=ComponentSerializer, - description="Already tracked. Tracking twice is not an error.", - ), - }, - ) def create(self, request, *args, **kwargs): board = _board(request, self.kwargs["uuid"]) + # `visible`, so a write agrees with the reads. An archived id + # already answers 404 on the component detail. Accepting it here + # made a board row that no list renders. component = get_object_or_404( - ServiceComponent, id=request.data.get("component_id") + ServiceComponent.objects.visible(), id=request.data.get("component_id") ) - # DashboardItem.save keeps the service's watcher count true. _, created = DashboardItem.objects.get_or_create( dashboard=board, component=component, defaults={"created_by": request.user, "updated_by": request.user}, ) + # Read again, after the write. `ComponentSerializer` reads the + # counts `for_display` annotates, and `is_tracked` is false + # until the item exists. + row = ( + ServiceComponent.objects.visible() + .for_display(request.user) + .get(pk=component.pk) + ) return Response( - ComponentSerializer(component, context={"request": request}).data, + ComponentSerializer(row, context={"request": request}).data, status=status.HTTP_201_CREATED if created else status.HTTP_200_OK, ) diff --git a/api/polling/adapters/services/cstate.py b/api/polling/adapters/services/cstate.py index 6f1b1d1..71d7c8a 100644 --- a/api/polling/adapters/services/cstate.py +++ b/api/polling/adapters/services/cstate.py @@ -75,7 +75,4 @@ def fetch_incidents(self): def fetch_service_metadata(self): payload = self._payload() - return { - "name": payload.get("title", ""), - "description": payload.get("description", ""), - } + return {"name": payload.get("title", "")} diff --git a/api/polling/admin.py b/api/polling/admin.py index 1b6a92c..526722b 100644 --- a/api/polling/admin.py +++ b/api/polling/admin.py @@ -59,22 +59,6 @@ def humanise_seconds(seconds): class PollRunColumns: """How a run reads. The same on its own table and under a poller.""" - @display( - description=_("Result"), - label={"OK": "success", "Failed": "danger"}, - ordering="ok", - ) - def display_ok(self, obj): - return "OK" if obj.ok else "Failed" - - @display( - description=_("Error type"), - ordering="error_type", - label={"Unrecorded": "warning"}, - ) - def display_error_type(self, obj): - return obj.error_type or "—" - @display(description=_("Error")) def display_error(self, obj): return (obj.error or "—")[:90] @@ -105,11 +89,11 @@ class PollRunInline(PollRunColumns, TabularInline): show_change_link = True per_page = 20 fields = [ - "display_ok", + "ok", "provider", "started_at", "finished_at", - "display_error_type", + "error_type", "display_error", ] readonly_fields = fields @@ -153,6 +137,13 @@ class PollerAdmin( autocomplete_fields = ["service"] ordering = ["-consecutive_failure_count", "next_at"] inlines = [PollRunInline] + # Read from the pause flag and the failure count, so it is shown + # and never typed. + readonly_fields = ["display_health"] + # A checkbox in the list, not an action. A provider going down means + # pausing several pollers, and one Save covers any mix of pausing + # and resuming. + list_editable = ["is_paused"] fieldsets = [ (None, {"fields": ["service", "is_paused", "note"]}), ( @@ -167,7 +158,14 @@ class PollerAdmin( ), ( _("Health"), - {"fields": ["next_at", "last_success_at", "consecutive_failure_count"]}, + { + "fields": [ + "display_health", + "next_at", + "last_success_at", + "consecutive_failure_count", + ] + }, ), audit_section(), ] @@ -275,19 +273,19 @@ def has_manage_permission(self, request, obj=None): class PollRunAdmin(PollRunColumns, PollerWrittenAdmin, ModelAdmin): """We are the thing that tells you when services break. - So we cannot quietly break ourselves. A failing poll is a labelled row - here, not a number to go looking for. + So we cannot quietly break ourselves. A failing poll is marked on its + own row here, not a number to go looking for. """ list_display = [ "display_run", "display_poller", "display_service", - "display_ok", + "ok", "provider", "started_at", "display_duration", - "display_error_type", + "error_type", "display_related", ] date_hierarchy = "started_at" diff --git a/api/polling/importer.py b/api/polling/importer.py index 983c0aa..545e85d 100644 --- a/api/polling/importer.py +++ b/api/polling/importer.py @@ -17,24 +17,36 @@ from django.db import transaction from django.utils import timezone +from catalog.choices import ServiceSource from catalog.models import Service, StatusPage from polling.adapters.registry import identify from polling.models import PollRun from polling.reconcile import apply_fetch +# What `author` means when a caller passes nothing. A scheduled job +# and a management command sign their rows with the system account. +# `None` is a different answer: a person we cannot name. +AUTOMATED = object() -def import_from_url(url: str) -> tuple[Service, bool]: + +def import_from_url(url: str, author=AUTOMATED) -> tuple[Service, bool]: """Build a whole service from a status page URL. The provider, the name, the components and the event history all come from the page. Returns (service, created); importing a page that is already in the catalog returns the existing service. + `author` signs the service and its status page. A request path + passes the caller, so the admin who imported Twilio is on the row. + It falls back to the system account, which is true of a command. + Reading and writing are two steps, and only the write is a transaction. A provider takes seconds to answer, and a database connection must not spend them waiting on one. """ + if author is AUTOMATED: + author = get_user_model().objects.system() key = StatusPage.normalise_url(url) already = _imported(key) if already is not None: @@ -51,7 +63,7 @@ def import_from_url(url: str) -> tuple[Service, bool]: "source": adapter.status_source, "started": started, } - return _import(key, adapter_class, fetch_url, fetched) + return _import(key, adapter_class, fetch_url, fetched, author) def _imported(key): @@ -61,7 +73,7 @@ def _imported(key): @transaction.atomic -def _import(key, adapter_class, fetch_url, fetched): +def _import(key, adapter_class, fetch_url, fetched, author): """Write what the page said, in one transaction.""" # Somebody may have imported the same page while we read it. The # fetch is outside the transaction now, so that window is @@ -71,14 +83,11 @@ def _import(key, adapter_class, fetch_url, fetched): return already, False metadata = fetched["metadata"] - # Nobody typed these in, so they are signed by the system account - # rather than left blank. - author = get_user_model().objects.system() service = Service.objects.create( name=metadata["name"], - description=metadata.get("description", ""), homepage_url=metadata.get("homepage_url", ""), logo=fetched["logo"], + source=ServiceSource.IMPORT, created_by=author, updated_by=author, ) @@ -96,9 +105,8 @@ def _import(key, adapter_class, fetch_url, fetched): # the first reading has no provenance, and the log is missing # the request that made the rows. # - # Nothing above makes the poller. `create_poller` in - # polling.signals does, on the Service save. Making one here - # would trip the one-to-one column. + # Nothing above makes the poller. `Service.ensure_poller` does, on + # the save. Making one here would trip the one-to-one column. run = PollRun.open( service.poller, key, adapter_class.provider, at=fetched["started"] ) diff --git a/api/polling/reconcile.py b/api/polling/reconcile.py index 604c665..30e4dae 100644 --- a/api/polling/reconcile.py +++ b/api/polling/reconcile.py @@ -3,11 +3,12 @@ from django.utils import timezone from catalog.models import ServiceComponent +from polling.system_events import claim, reconcile_system_events from status.models import ComponentStatus, EventUpdate, ServiceEvent @transaction.atomic -def apply_fetch(service, components, events, source, run=None): +def apply_fetch(service, components, events, source, run): """Write one adapter fetch to the database. A poll is a reconciliation, not a status read. @@ -15,7 +16,8 @@ def apply_fetch(service, components, events, source, run=None): An unchanged severity leaves the open status row alone. `run` is the PollRun that produced this data. Stamping it makes a - reading traceable back to the fetch that wrote it. + reading traceable back to the fetch that wrote it. Every fetch has + one, so nothing here writes a reading with no provenance. Nobody types any of this, so the system account signs every row. A blank author reads the same as one that was lost, and a component @@ -24,7 +26,7 @@ def apply_fetch(service, components, events, source, run=None): # A poll writes to the service it read. Nothing in the arguments # ties them together. A run from another poller would file one # service's readings under another. - if run is not None and run.poller.service_id != service.pk: + if run.poller.service_id != service.pk: raise ValueError( f"{run} polled {run.poller.service}, not {service}. " "A poll writes to the service it read." @@ -34,6 +36,9 @@ def apply_fetch(service, components, events, source, run=None): _archive_vanished(service, components, author) _write_statuses(components, rows, source, author, run) _upsert_events(service, events, rows, author, run) + # After the provider's events, so a component they explained does + # not also get one of ours in the same pass. + reconcile_system_events(service, author) def _signed(author, **fields): @@ -65,7 +70,7 @@ def _upsert_components(service, components, author): ) rows[incoming.external_id] = row - # Parents in a second pass: a child may arrive before its parent exists. + # Parents second: a child may arrive before its parent exists. for incoming in components: parent = ( rows.get(incoming.parent_external_id) @@ -89,7 +94,7 @@ def _archive_vanished(service, components, author): ).update(is_archived=True, archived_at=timezone.now(), updated_by=author) -def _write_statuses(components, rows, source, author, run=None): +def _write_statuses(components, rows, source, author, run): now = timezone.now() for incoming in components: row = rows[incoming.external_id] @@ -135,7 +140,7 @@ def _affected(service, rows, external_ids): return named -def _upsert_events(service, events, rows, author, run=None): +def _upsert_events(service, events, rows, author, run): for incoming in events: event, _ = ServiceEvent.objects.update_or_create( service=service, @@ -164,3 +169,6 @@ def _upsert_events(service, events, rows, author, run=None): "updated_by": author, }, ) + # After the updates, so they move with the row if this event + # takes over one we opened. + claim(event, author) diff --git a/api/polling/system_events.py b/api/polling/system_events.py new file mode 100644 index 0000000..c7f2617 --- /dev/null +++ b/api/polling/system_events.py @@ -0,0 +1,282 @@ +"""Events we open when a provider explains nothing. + +A provider can move a component to Degraded and never write an +incident. An event-only feed would hide that outage, and the closed +`ComponentStatus` span is exposed nowhere else. + +A provider often posts after our poll saw the change. `claim` folds +their event into ours, so one outage stays one card. + +`ComponentStatus` stays the truth. An event here is a projection of it, +written by this module alone and rebuildable from nothing. That is what +keeps it from being a second answer to the same question. +""" + +from django.conf import settings +from django.db import transaction +from django.utils import timezone + +from status.choices import EventKind, EventSource, IncidentPhase, Severity +from status.models import ComponentStatus, EventUpdate, ServiceEvent + +# Worse than this and nothing is wrong. Severity 3 is our own poll +# failing to read their page. Severity 4 is a window a provider always +# announces. Neither is an outage of theirs to report. +SYSTEM_EVENT_MAX_SEVERITY = Severity.DEGRADED + +# Which severities end one of ours. Not the inverse of the rule above, +# on purpose. Unknown is absent because it is our own poll failing. A +# page we cannot read is no account of a recovery. Maintenance closes +# because a window a provider announces supersedes what we saw. +SYSTEM_EVENT_CLOSING_SEVERITIES = frozenset( + {Severity.OPERATIONAL, Severity.MAINTENANCE} +) + + +def reconcile_system_events(service, author): + """Open, extend and close the events this deployment owns.""" + for span in _bad_statuses(service): + _open_or_extend(span, author) + _close_recovered(service, author) + + +@transaction.atomic +def claim(provider_event, author): + """Fold a provider's event into the one we opened for the same outage. + + A provider often posts minutes after our poll saw the change. Two + rows would be two cards for one outage. + + The provider's row is deleted and ours takes its id, so the fact + that we found it first survives. Deleting theirs rather than ours + is what keeps `detected_by` meaningful. + + Atomic on its own. A failure between the delete and the save loses + their row. Ours is then left without the id. The next poll writes + their event again, and moves duplicate updates onto ours. + """ + if provider_event.detected_by != EventSource.PROVIDER: + return False + # Only an incident claims. A claim copies their phase onto ours, + # and a maintenance phase on an incident is invalid. A planned + # window is also no account of the outage we found. + if provider_event.kind != EventKind.INCIDENT: + return False + ours = _claimable(provider_event) + if ours is None: + return False + ours.external_id = provider_event.external_id + ours.title = provider_event.title + ours.phase = provider_event.phase + ours.ends_at = provider_event.ends_at + ours.updated_by = author + # Their timeline moves first. A delete cascades to their updates, + # so the other order loses every post they made. + provider_event.updates.update(event=ours) + ours.affected_components.add(*provider_event.affected_components.all()) + # The unique key is the service and the provider id. Ours cannot + # hold their id while their row still does, so the save is last. + provider_event.delete() + ours.save(update_fields=["external_id", "title", "phase", "ends_at", "updated_by"]) + return True + + +def _claimable(provider_event): + """Our open event for the same outage, or nothing. + + A candidate is on the same service and names one of their + components. It began no later than their start plus the window. + The candidate whose start is nearest theirs wins when several match. + """ + components = list(provider_event.affected_components.all()) + if not components: + return None + # Read as "ours began no later than theirs plus the window". That + # is the same rule as "theirs began no earlier than ours less it". + floor = provider_event.starts_at + settings.EVENT_CLAIM_WINDOW + candidates = ( + ServiceEvent.objects.live() + .filter( + service=provider_event.service, + detected_by=EventSource.SYSTEM, + external_id__isnull=True, + affected_components__in=components, + starts_at__lte=floor, + ) + .distinct() + ) + return min( + candidates, + key=lambda e: abs(e.starts_at - provider_event.starts_at), + default=None, + ) + + +def _bad_statuses(service): + """Open spans bad enough to be somebody's outage. + + An archived component is left out. Its last span stays open for + good, so each poll would open a card that closes in the same pass. + """ + return ComponentStatus.objects.filter( + component__service=service, + component__is_archived=False, + ended_at__isnull=True, + severity__lte=SYSTEM_EVENT_MAX_SEVERITY, + ).select_related("component") + + +def _explained(component): + """Whether a live provider event already accounts for this component. + + An incident that names no component covers its whole service. A + provider publishes one for an outage that reaches everything. One + outage is one card, so ours does not open beside it. + + A maintenance window naming nothing covers nothing. Planned work is + no account of an outage. Read as one, it hides every outage that + runs beside a window. + """ + live = ServiceEvent.objects.live().filter(detected_by=EventSource.PROVIDER) + if live.filter(affected_components=component).exists(): + return True + return live.filter( + service=component.service, + kind=EventKind.INCIDENT, + affected_components__isnull=True, + ).exists() + + +def _ours(component): + """The open event we opened for this component, if there is one.""" + return ( + ServiceEvent.objects.live() + .filter(affected_components=component, detected_by=EventSource.SYSTEM) + .first() + ) + + +def _open_or_extend(span, author): + component = span.component + if _explained(component): + return + event = _ours(component) + if event is None: + event = ServiceEvent.objects.create( + service=component.service, + external_id=None, + detected_by=EventSource.SYSTEM, + kind=EventKind.INCIDENT, + title=_title(component, span.severity), + phase=IncidentPhase.DETECTED, + starts_at=span.started_at, + created_by=author, + updated_by=author, + ) + event.affected_components.set([component]) + _record(event, span, author) + + +def _record(event, span, author): + """One update per severity span, keyed on when the span began. + + A poll that changes nothing runs this again. `get_or_create` on the + span's start is what stops a duplicate post per beat. + """ + EventUpdate.objects.get_or_create( + event=event, + posted_at=span.started_at, + defaults={ + "phase": event.phase, + "source": EventSource.SYSTEM, + "body": _title(span.component, span.severity), + "created_by": author, + "updated_by": author, + }, + ) + + +def _close_recovered(service, author): + """Close ours once every component it names is no longer bad.""" + open_events = ServiceEvent.objects.live().filter( + service=service, detected_by=EventSource.SYSTEM + ) + for event in open_events: + closing = _closing(event) + if closing is not None: + _close(event, author, *closing) + + +def _closing(event): + """When and why an event ends, or nothing while a component is bad. + + A claim merges the provider's components onto ours, and they name + several. One recovery is no account of the others. + + The component that settles last decides both. Holding the card open + until then is the conservative error. Closing early leaves a + component down with nothing covering it. The next poll then opens a + second card for the same outage. + """ + now = timezone.now() + # The primary key sorts by creation, so this is a total order. The + # relation has none, and the pair below has to be repeatable. + components = list(event.affected_components.order_by("pk")) + if not components: + return None + settled = [] + for component in components: + ended = _settled(component, now) + if ended is None: + return None + settled.append(ended) + return max(settled, key=lambda pair: pair[0]) + + +def _settled(component, now): + """When and why one component stopped being bad, or nothing.""" + # A provider stopped publishing an archived component. Nothing can + # report its recovery, so the event would never close. + if component.is_archived: + return component.archived_at, f"{component.name} is no longer published" + current = ComponentStatus.objects.filter( + component=component, ended_at__isnull=True + ).first() + if not _recovered(current): + return None + # A component with no open span has nothing wrong with it. + severity = current.severity if current is not None else Severity.OPERATIONAL + return now, _title(component, severity) + + +def _recovered(current): + """Whether the open span says the outage is over. + + Closing needs a severity that means recovery, not merely one above + the threshold that opened the event. No open span closes it too. + """ + if current is None: + return True + return current.severity in SYSTEM_EVENT_CLOSING_SEVERITIES + + +def _close(event, author, ends_at, body): + """Resolve one of ours, and post the update that says why.""" + EventUpdate.objects.create( + event=event, + posted_at=ends_at, + phase=IncidentPhase.RESOLVED, + source=EventSource.SYSTEM, + body=body, + created_by=author, + updated_by=author, + ) + event.phase = IncidentPhase.RESOLVED + event.ends_at = ends_at + event.updated_by = author + event.save(update_fields=["phase", "ends_at", "updated_by"]) + + +def _title(component, severity): + """What the card says before a provider gives it a better name.""" + return f"{component.name} {Severity(severity).label.lower()}" diff --git a/api/polling/tasks.py b/api/polling/tasks.py index 19b1376..def78ed 100644 --- a/api/polling/tasks.py +++ b/api/polling/tasks.py @@ -75,7 +75,7 @@ def _refresh_metadata(service, metadata, adapter=None): if logo: service.logo = logo changed.append("logo") - for field in ("name", "description", "homepage_url"): + for field in ("name", "homepage_url"): value = metadata.get(field) if value and getattr(service, field) != value: setattr(service, field, value) diff --git a/api/pyproject.toml b/api/pyproject.toml index 369c1bd..e38992b 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -44,4 +44,4 @@ ignore = ["RUF012"] DJANGO_SETTINGS_MODULE = "api.settings" python_files = ["test_*.py"] addopts = "--cov=. --cov-report=term-missing" -# The gate is in `just test-cov`. Here it fails every run until the last task. +# The gate is in `just check`. Here it would fail on every partial run. diff --git a/api/status/admin.py b/api/status/admin.py index c175e4f..239d29d 100644 --- a/api/status/admin.py +++ b/api/status/admin.py @@ -26,7 +26,7 @@ severity_label, ) from common.queries import related_count -from status.choices import CLOSED_PHASES, EVENT_PHASES_BY_KIND, EventKind +from status.choices import EVENT_PHASES_BY_KIND, EventKind from status.models import ComponentStatus, EventUpdate, ServiceEvent @@ -59,7 +59,7 @@ class ComponentStatusAdmin(PollerWrittenAdmin, ModelAdmin): "display_service", "display_severity", "display_span", - "display_source", + "source", "display_poll_run", ] date_hierarchy = "started_at" @@ -104,19 +104,6 @@ def display_span(self, obj): # standing. return date_span(obj.started_at, obj.ended_at) - @display(description=_("Source")) - def display_source(self, obj): - # How the severity was arrived at, which is not the same question - # as what it is. - return obj.get_source_display() - - def get_readonly_fields(self, request, obj=None): - # The run is not an editable column, so it reaches the record - # only this way. Without it you could see where a reading came - # from on the table and not on the reading itself. - fields = list(super().get_readonly_fields(request, obj)) - return fields + ["display_poll_run"] - @display(description=_("Severity"), label=SEVERITY_VARIANTS, ordering="severity") def display_severity(self, obj): return severity_label(obj.severity) @@ -131,10 +118,11 @@ def display_service(self, obj): class EventUpdateInline(TabularInline): - """The provider's update log, read only. + """The event's update log, read only. - These are the provider's words, written by a poll. Editing them here - would make the admin disagree with the status page it mirrors. + A poll writes these. Most are the provider's words. A claimed event + also holds ours. Editing any of them here would make the admin + disagree with the status page it mirrors. """ model = EventUpdate @@ -145,7 +133,9 @@ class EventUpdateInline(TabularInline): # Read only, but still worth opening: the row is a summary. show_change_link = True per_page = 10 - fields = ["phase", "body", "posted_at"] + # A claimed event interleaves our updates with the provider's. + # Without the source the timeline reads as one author. + fields = ["phase", "source", "body", "posted_at"] readonly_fields = fields ordering = ["-posted_at"] @@ -161,6 +151,9 @@ class ServiceEventAdmin(PollerWrittenAdmin, ModelAdmin): "display_when", "display_kind", "display_phase", + # `external_id IS NULL` cannot answer this. A claim fills the + # column in, which destroys the fact that we found it first. + "detected_by", "display_related", ] date_hierarchy = "starts_at" @@ -172,6 +165,7 @@ class ServiceEventAdmin(PollerWrittenAdmin, ModelAdmin): ("affected_components", AutocompleteSelectFilter), ("poll_run", AutocompleteSelectFilter), ("kind", ChoicesDropdownFilter), + ("detected_by", ChoicesDropdownFilter), PhaseFilter, ("starts_at", RangeDateTimeFilter), ("ends_at", RangeDateTimeFilter), @@ -254,7 +248,7 @@ def get_fieldsets(self, request, obj=None): ] }, ), - (_("Classification"), {"fields": ["kind", "phase"]}), + (_("Classification"), {"fields": ["kind", "phase", "detected_by"]}), (_("Span"), {"fields": ["starts_at", "ends_at"]}), audit_section(), ] @@ -326,10 +320,6 @@ def display_phase(self, obj): # A closed phase means the event is over. return phase_label(obj) - @display(description=_("Open"), boolean=True) - def display_open(self, obj): - return obj.phase not in CLOSED_PHASES - @admin.register(EventUpdate) class EventUpdateAdmin(PollerWrittenAdmin, ModelAdmin): @@ -338,6 +328,7 @@ class EventUpdateAdmin(PollerWrittenAdmin, ModelAdmin): "display_event", "display_service", "phase", + "source", "posted_at", ] display_update = record_column(_("Update")) @@ -352,11 +343,13 @@ class EventUpdateAdmin(PollerWrittenAdmin, ModelAdmin): list_filter = [ ("event__service", AutocompleteSelectFilter), ("event", AutocompleteSelectFilter), + PhaseFilter, + ("source", ChoicesDropdownFilter), ("posted_at", RangeDateTimeFilter), ("created_at", RangeDateTimeFilter), ] fieldsets = [ - (None, {"fields": ["event", "phase", "posted_at", "body"]}), + (None, {"fields": ["event", "phase", "source", "posted_at", "body"]}), audit_section(), ] diff --git a/api/status/choices.py b/api/status/choices.py index 53dae77..50746ac 100644 --- a/api/status/choices.py +++ b/api/status/choices.py @@ -28,6 +28,9 @@ class EventKind(models.TextChoices): class IncidentPhase(models.TextChoices): + # First, because it precedes anything a provider posts. We write it + # when a severity drops with nothing explaining it. + DETECTED = "detected", "Detected" INVESTIGATING = "investigating", "Investigating" IDENTIFIED = "identified", "Identified" MONITORING = "monitoring", "Monitoring" @@ -41,6 +44,30 @@ class MaintenancePhase(models.TextChoices): COMPLETED = "completed", "Completed" +class EventSource(models.TextChoices): + """Who produced a row: the provider's page, or this deployment. + + An event records who opened it. An update records who wrote it. A + claimed event has both, one per update. + """ + + PROVIDER = "provider", "Provider" + SYSTEM = "system", "Statusboard" + + +class EventPhaseState(models.TextChoices): + """Whether an event is still running, or over. + + The `phase=` filter takes one of these. `CLOSED_PHASES` below draws + the line and stays on the server, so a client never restates which + phases are terminal. These two labels are what a client shows, so + `/meta/` publishes them beside every other fixed set. + """ + + OPEN = "open", "Open" + CLOSED = "closed", "Closed" + + EVENT_PHASES_BY_KIND = { EventKind.INCIDENT: IncidentPhase, EventKind.MAINTENANCE: MaintenancePhase, diff --git a/api/status/filters.py b/api/status/filters.py new file mode 100644 index 0000000..b5ab0bc --- /dev/null +++ b/api/status/filters.py @@ -0,0 +1,62 @@ +"""What a caller may narrow the event feed by. + +Beside the views, and apart from them. A filter is the query contract, +and it is read on its own more often than the view around it. +""" + +from django.shortcuts import get_object_or_404 +from django_filters import rest_framework as filters +from rest_framework.exceptions import NotAuthenticated + +from dashboards.models import Dashboard +from status.choices import CLOSED_PHASES, EventPhaseState +from status.models import ServiceEvent + + +class EventFilter(filters.FilterSet): + # Declared, not generated. `CLOSED_PHASES` draws the line between + # an open phase and a closed one. A client restating it is a + # second copy of one rule. + phase = filters.ChoiceFilter(choices=EventPhaseState.choices, method="filter_phase") + service = filters.CharFilter(field_name="service__slug") + component = filters.UUIDFilter(field_name="affected_components__id") + dashboard = filters.UUIDFilter(method="filter_dashboard") + + class Meta: + model = ServiceEvent + fields = {"kind": ["exact"]} + + def filter_phase(self, queryset, name, value): + """Narrow to the open phases, or to the terminal ones. + + An absent `phase` returns both. django-filter skips a method + when the value is empty, so nothing here runs and no default + applies. + + `ChoiceFilter` admits `open` and `closed` only. The last return + is the closed case, not a fallback. + """ + if value == EventPhaseState.OPEN: + return queryset.exclude(phase__in=CLOSED_PHASES) + return queryset.filter(phase__in=CLOSED_PHASES) + + def filter_dashboard(self, queryset, name, value): + """Everything posted across the services on one board. + + The board's rows are components, and an event names components. + + Who owns the board is decided here too, because one parameter + belongs to one place. The view read the id again and parsed the + UUID by hand. django-filter has parsed it before this runs, and + a malformed one never arrives. + """ + # An anonymous caller owns no board. Refuse before comparing + # against AnonymousUser, which get_object_or_404 cannot do. + if not self.request.user.is_authenticated: + raise NotAuthenticated + # 404 rather than 403: someone else's board id should not be + # confirmable. + get_object_or_404(Dashboard, id=value, owner=self.request.user) + return queryset.filter( + affected_components__tracked_by__dashboard_id=value + ).distinct() diff --git a/api/status/migrations/0003_event_detected_by.py b/api/status/migrations/0003_event_detected_by.py new file mode 100644 index 0000000..b9266d0 --- /dev/null +++ b/api/status/migrations/0003_event_detected_by.py @@ -0,0 +1,55 @@ +# Generated by Django 5.2.17 on 2026-09-03 19:35 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0005_component_search_document"), + ("polling", "0002_every_service_has_a_poller"), + ("status", "0002_events_by_service_and_kind"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.RemoveConstraint( + model_name="serviceevent", + name="one_event_per_provider_id", + ), + migrations.AddField( + model_name="eventupdate", + name="source", + field=models.CharField( + choices=[("provider", "Provider"), ("system", "Statusboard")], + db_default="provider", + default="provider", + max_length=32, + ), + ), + migrations.AddField( + model_name="serviceevent", + name="detected_by", + field=models.CharField( + choices=[("provider", "Provider"), ("system", "Statusboard")], + db_default="provider", + default="provider", + max_length=32, + ), + ), + migrations.AlterField( + model_name="serviceevent", + name="external_id", + field=models.CharField( + blank=True, max_length=200, null=True, verbose_name="Provider ID" + ), + ), + migrations.AddConstraint( + model_name="serviceevent", + constraint=models.UniqueConstraint( + condition=models.Q(("external_id__isnull", False)), + fields=("service", "external_id"), + name="one_event_per_provider_id", + ), + ), + ] diff --git a/api/status/models.py b/api/status/models.py index f9595ea..a07b3e3 100644 --- a/api/status/models.py +++ b/api/status/models.py @@ -9,6 +9,7 @@ CLOSED_PHASES, EVENT_PHASES_BY_KIND, EventKind, + EventSource, Severity, StatusSource, ) @@ -124,7 +125,20 @@ class ServiceEvent(BaseModel): service = models.ForeignKey( Service, on_delete=models.CASCADE, related_name="events" ) - external_id = models.CharField(verbose_name="Provider ID", max_length=200) + # Null until a provider claims this event. We open events a + # provider never published, and they have no id of theirs. + external_id = models.CharField( + verbose_name="Provider ID", max_length=200, null=True, blank=True + ) + # Who opened it, never rewritten by a claim. `external_id IS NULL` + # cannot answer this: claiming fills the column in, which destroys + # the fact that we found the outage first. + detected_by = models.CharField( + max_length=32, + choices=EventSource.choices, + default=EventSource.PROVIDER, + db_default=EventSource.PROVIDER, + ) kind = models.CharField(max_length=32, choices=EventKind.choices) title = models.CharField(max_length=500) phase = models.CharField(max_length=32) @@ -154,12 +168,17 @@ class ServiceEvent(BaseModel): class Meta(BaseModel.Meta): constraints = [ + # Partial. Two events we opened both hold null, and null + # does not collide with null in Postgres anyway. Stating + # the condition says the exemption is deliberate. models.UniqueConstraint( - fields=["service", "external_id"], name="one_event_per_provider_id" + fields=["service", "external_id"], + condition=models.Q(external_id__isnull=False), + name="one_event_per_provider_id", ) ] indexes = [ - # `live` reads this for every component on a page, and the + # `live` reads this for every component on a page. The # only index was the unique key, which serves neither # column. Ordered as the filter narrows: the service first, # then the kind, then the phase it excludes. @@ -202,6 +221,14 @@ class EventUpdate(BaseModel): phase = models.CharField(max_length=32) body = models.TextField() posted_at = models.DateTimeField(verbose_name="Posted") + # Who wrote this post. A claimed event holds both: our detection + # first, then the provider's log. + source = models.CharField( + max_length=32, + choices=EventSource.choices, + default=EventSource.PROVIDER, + db_default=EventSource.PROVIDER, + ) def clean(self): """A phase belongs to one kind, the same rule the event follows. diff --git a/api/status/serializers.py b/api/status/serializers.py index f62c1b1..80a4ca4 100644 --- a/api/status/serializers.py +++ b/api/status/serializers.py @@ -5,6 +5,14 @@ from status.models import ComponentStatus, EventUpdate, ServiceEvent +def _service_ref_serializer(): + # Imported lazily. catalog imports status, so a module-level import + # here would be a cycle. + from catalog.serializers import ServiceRefSerializer + + return ServiceRefSerializer + + class StatusSerializer(FieldsMixin, serializers.ModelSerializer): last_refreshed_at = serializers.SerializerMethodField() @@ -32,12 +40,63 @@ class Meta: class EventUpdateSerializer(FieldsMixin, serializers.ModelSerializer): class Meta: model = EventUpdate - fields = ["phase", "body", "posted_at"] + fields = ["phase", "body", "posted_at", "source"] class ServiceEventSerializer(FieldsMixin, serializers.ModelSerializer): - updates = EventUpdateSerializer(many=True, read_only=True) + """One row of the feed. The log is `/events/{uuid}/updates/`.""" class Meta: model = ServiceEvent - fields = ["id", "kind", "title", "phase", "starts_at", "ends_at", "updates"] + fields = [ + "id", + "kind", + "title", + "phase", + "starts_at", + "ends_at", + "detected_by", + "service", + ] + + def get_fields(self): + # `catalog.serializers` imports this module for `EventRefSerializer`, + # so the reverse import has to wait until both have finished loading. + fields = super().get_fields() + fields["service"] = _service_ref_serializer()(read_only=True) + return fields + + +class ServiceEventDetailSerializer(ServiceEventSerializer): + """The event screen's header and its About tab. + + Both counts are here because the header draws both tab badges + before either tab has made a request. + """ + + update_count = serializers.SerializerMethodField() + affected_count = serializers.SerializerMethodField() + last_update_at = serializers.SerializerMethodField() + + class Meta(ServiceEventSerializer.Meta): + fields = [ + *ServiceEventSerializer.Meta.fields, + "update_count", + "affected_count", + "last_update_at", + ] + + @extend_schema_field(serializers.IntegerField()) + def get_update_count(self, event): + return event.updates.count() + + @extend_schema_field(serializers.IntegerField()) + def get_affected_count(self, event): + # `visible`, because the Affects tab reads `?event=` and counts + # the same rows. The badge cannot say 2 above a list of one. + return event.affected_components.visible().count() + + @extend_schema_field(serializers.DateTimeField(allow_null=True)) + def get_last_update_at(self, event): + newest = event.updates.order_by("-posted_at").first() + return newest.posted_at if newest else None diff --git a/api/status/urls.py b/api/status/urls.py new file mode 100644 index 0000000..05495a0 --- /dev/null +++ b/api/status/urls.py @@ -0,0 +1,13 @@ +from django.urls import path + +from status.views import EventDetailView, EventListView, EventUpdateListView + +urlpatterns = [ + path("events/", EventListView.as_view(), name="event-list"), + path("events//", EventDetailView.as_view(), name="event-detail"), + path( + "events//updates/", + EventUpdateListView.as_view(), + name="event-updates", + ), +] diff --git a/api/status/views.py b/api/status/views.py new file mode 100644 index 0000000..3e8d8ce --- /dev/null +++ b/api/status/views.py @@ -0,0 +1,86 @@ +"""Every read of an event, in one place. + +Home, a service and a component show the same feed at different scopes. +The screens call it Updates. This names the model, and `/meta/` +publishes the labels. +""" + +from django.shortcuts import get_object_or_404 +from django_filters.rest_framework import DjangoFilterBackend +from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_view +from rest_framework import generics +from rest_framework.permissions import AllowAny + +from common.aggregates import EventAggregateSet +from common.filters import REJECTED_PARAMETER, FieldsBackend +from common.ordering import MappedOrderingFilter +from common.pagination import TimelinePagination +from common.serializers import ErrorSerializer +from status.filters import EventFilter +from status.models import EventUpdate, ServiceEvent +from status.serializers import ( + EventUpdateSerializer, + ServiceEventDetailSerializer, + ServiceEventSerializer, +) + + +@extend_schema_view( + get=extend_schema( + responses={ + 200: ServiceEventSerializer, + 400: OpenApiResponse(description=REJECTED_PARAMETER), + } + ) +) +class EventListView(generics.ListAPIView): + permission_classes = [AllowAny] + serializer_class = ServiceEventSerializer + aggregate_set = EventAggregateSet + filterset_class = EventFilter + filter_backends = [DjangoFilterBackend, MappedOrderingFilter, FieldsBackend] + ordering_fields = ["starts_at", "ends_at"] + ordering_map = {} + ordering = ["-starts_at"] + # `EventFilter.filter_dashboard` narrows this to one board, and + # decides whether the caller owns it. + queryset = ServiceEvent.objects.select_related("service") + + +@extend_schema_view( + get=extend_schema( + responses={200: ServiceEventDetailSerializer, 404: ErrorSerializer} + ) +) +class EventDetailView(generics.RetrieveAPIView): + permission_classes = [AllowAny] + serializer_class = ServiceEventDetailSerializer + filter_backends = [FieldsBackend] + lookup_field = "pk" + # The contract names the path variable `uuid`. DRF renames a `pk` + # one to `id`, which would document a path nothing serves. + lookup_url_kwarg = "uuid" + queryset = ServiceEvent.objects.select_related("service") + + +class EventUpdateListView(generics.ListAPIView): + """One event's log, oldest first. + + The feed is newest first because it is a feed. This is a + narrative, and a narrative is read forwards. + """ + + permission_classes = [AllowAny] + serializer_class = EventUpdateSerializer + filter_backends = [FieldsBackend] + pagination_class = TimelinePagination + # get_queryset reads self.kwargs, which schema generation has not + # set. This names the model without running it. + queryset = EventUpdate.objects.none() + + def get_queryset(self): + event = get_object_or_404(ServiceEvent, pk=self.kwargs["uuid"]) + # `TimelinePagination.ordering` is what actually orders the + # response: CursorPagination always re-sorts by it. This is + # the correct order regardless, in case pagination is ever off. + return event.updates.order_by("posted_at") diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 1bee08b..7bd36c2 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -1,4 +1,38 @@ import pytest +from rest_framework.test import APIClient +from rest_framework_simplejwt.tokens import RefreshToken + +from authentication.models import User + + +@pytest.fixture +def board(db): + user = User.objects.create(email="a@b.com") + return user.default_dashboard + + +def jwt_client(user): + """An `APIClient` bearing a fresh access token for this user. + + `DEFAULT_AUTHENTICATION_CLASSES` is JWT only. `force_login` leaves + a DRF view answering 401, because there is no session backend. + """ + api = APIClient() + api.credentials( + HTTP_AUTHORIZATION=f"Bearer {RefreshToken.for_user(user).access_token}" + ) + return api + + +@pytest.fixture +def authenticated_client(board): + """A board owner's own client. Most tests want the anonymous one.""" + return jwt_client(board.owner) + + +@pytest.fixture +def user(db): + return User.objects.create(email="asker@example.test") @pytest.fixture(autouse=True) diff --git a/api/tests/factories.py b/api/tests/factories.py index 41ccee5..7b161f3 100644 --- a/api/tests/factories.py +++ b/api/tests/factories.py @@ -1,9 +1,17 @@ import factory +from django.contrib.auth import get_user_model from catalog.models import Service, ServiceComponent, StatusPage from polling.models import Poller +class UserFactory(factory.django.DjangoModelFactory): + class Meta: + model = get_user_model() + + email = factory.Sequence(lambda n: f"watcher{n}@example.test") + + class ServiceFactory(factory.django.DjangoModelFactory): class Meta: model = Service @@ -65,22 +73,70 @@ class Meta: name = factory.Sequence(lambda n: f"Component {n}") +def poll_run(service, **kwargs): + """The fetch that a test says wrote its rows. + + `apply_fetch` takes one, and refuses a run from another service's + poller. The run has to belong to the service being reconciled. + + The status page supplies the url and the provider. A test about the + reconciliation does not have to make one. + """ + from polling.models import PollRun + + page = getattr(service, "status_page", None) or StatusPageFactory(service=service) + return PollRun.open(service.poller, page.url, page.provider, **kwargs) + + def track(component, user=None): - """Track a component, the way somebody using the app would.""" - from django.contrib.auth import get_user_model + """Track a component, the way somebody using the app would. + The user comes from `UserFactory`, which owns the one counter behind + these addresses. A second counter here reset per test and collided + with the factory's, on a column the database holds unique. + """ from dashboards.models import Dashboard, DashboardItem if user is None: - user = get_user_model().objects.create( - email=f"watcher{DashboardItem.objects.count()}@example.test" - ) + user = UserFactory() board = user.dashboards.first() or Dashboard.objects.create(owner=user) return DashboardItem.objects.create(dashboard=board, component=component) -def watchers(service): - """How many people track it, counted the way the app counts.""" - from catalog.queries import WATCHER_COUNT +def ancestry(component): + """The ids above it, root first, worked out from `parent`. + + `ServiceComponent.ancestors` is the one walk up the tree, and a + breadcrumb reads the same order. + """ + return [row.pk for row in component.ancestors] + + +def descendants(component): + """The ids under it, at any depth, the way `?ancestor=` asks.""" + from catalog.queries import descendant_ids - return Service.objects.annotate(n=WATCHER_COUNT).get(pk=service.pk).n + return set( + ServiceComponent.objects.filter( + pk__in=descendant_ids(component.pk) + ).values_list("pk", flat=True) + ) + + +def watchers(component): + """What the app counts, checked against a count walked in Python. + + The app's expression is the thing under test, so it cannot also be + the answer. Borrowing it made every caller assert that it equalled + itself, and dropping `distinct` would have stayed green. + """ + from catalog.queries import COMPONENT_WATCHER_COUNT + + counted = ( + ServiceComponent.objects.annotate(n=COMPONENT_WATCHER_COUNT) + .get(pk=component.pk) + .n + ) + walked = len({board.owner_id for board in component.boards.all()}) + assert counted == walked, f"the app counted {counted}, the tree holds {walked}" + return counted diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 2cc5dec..1eaa9ba 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -1,10 +1,77 @@ +import re + import pytest +from django.conf import settings from django.contrib import admin from django.urls import reverse from django.utils import timezone from authentication.models import User +# The apps this project owns. A third-party admin, such as +# auth.Group or a simplejwt token table, is not ours to place. +# It sits outside what this test requires reachable. +OUR_APPS = {"catalog", "status", "authentication", "dashboards", "polling"} + +CHANGELIST_PATH = re.compile(r"^/admin/(?P[^/]+)/(?P[^/]+)/$") + + +def _changelist_target(link): + """The registered model a changelist link opens, or None. + + A link to something other than a changelist, such as the + dashboard, names no model. It is not a gap to report. + """ + match = CHANGELIST_PATH.match(str(link)) + if not match: + return None + app_label, model_name = match.group("app_label"), match.group("model_name") + for model in admin.site._registry: + if model._meta.app_label == app_label and model._meta.model_name == model_name: + return model._meta.label + return None + + +def _sidebar_reachable(): + return { + label + for group in settings.UNFOLD["SIDEBAR"]["navigation"] + for item in group["items"] + if (label := _changelist_target(item["link"])) + } + + +def _tab_reachable(): + labels = set() + for group in settings.UNFOLD["TABS"]: + for dotted in group["models"]: + app_label, model_name = dotted.split(".") + for model in admin.site._registry: + if ( + model._meta.app_label == app_label + and model._meta.model_name == model_name + ): + labels.add(model._meta.label) + return labels + + +def _inline_reachable(top_level): + """A model reachable only as an inline on an already-reachable page. + + An inline on a page nothing else reaches does not count. Reachable + describes a page a person can open, not a class attribute nobody + links to. + """ + labels = set() + for model, site_admin in admin.site._registry.items(): + if model._meta.label not in top_level: + continue + for inline in getattr(site_admin, "inlines", []): + inline_model = getattr(inline, "model", None) + if inline_model is not None: + labels.add(inline_model._meta.label) + return labels + @pytest.fixture def staff_client(client): @@ -27,7 +94,6 @@ def service_form_data(staff_client, **fields): data = { "name": "", "slug": "", - "description": "", "logo": "", "homepage_url": "", **fields, @@ -42,16 +108,10 @@ def service_form_data(staff_client, **fields): def test_the_dashboard_leads_with_poller_health(staff_client): # The landing page answers "is polling healthy?". A stalled poller # shows every board a stale green, which is worse than showing nothing. - body = staff_client.get(reverse("admin:index")).content.decode() - for card in ( - "Services tracked", - "Behind schedule", - "Poll success", - "Stalest service", - ): - assert card in body, f"{card} missing from the dashboard" - for panel in ("Polls, last 24 hours", "Services tracked, last 30 days"): - assert panel in body, f"{panel} missing from the dashboard" + response = staff_client.get(reverse("admin:index")) + # `dashboard_callback` builds the cards. Unwired, the page is bare. + assert response.context["cards"] + body = response.content.decode() # Neither the catalog of every model nor a log of admin edits says # anything about whether polling works. assert "app-list" not in body @@ -65,13 +125,6 @@ def test_the_environment_is_named_on_every_page(staff_client): assert "Development" in staff_client.get(reverse("admin:index")).content.decode() -@pytest.mark.django_db -def test_the_brand_marks_are_wired(staff_client): - body = staff_client.get(reverse("admin:index")).content.decode() - assert "statusboard/logo-light.svg" in body - assert "statusboard/favicon.svg" in body - - @pytest.mark.django_db @pytest.mark.parametrize( "model", @@ -162,6 +215,21 @@ def test_the_admin_stamps_who_created_a_row(staff_client): assert service.updated_by.email == "admin@example.com" +@pytest.mark.django_db +def test_a_service_added_on_the_admin_form_says_it_was_added_by_hand(staff_client): + # The add form is the one path that is not an import. A row saying + # import here claims a status page named the service. + from catalog.choices import ServiceSource + from catalog.models import Service + + staff_client.post( + reverse("admin:catalog_service_add"), + service_form_data(staff_client, name="By hand"), + ) + + assert Service.objects.get(name="By hand").source == ServiceSource.MANUAL + + @pytest.mark.django_db @pytest.mark.parametrize("field", ["created_by", "updated_by"]) def test_derived_fields_are_not_on_the_form(staff_client, field): @@ -171,21 +239,13 @@ def test_derived_fields_are_not_on_the_form(staff_client, field): assert f'name="{field}"' not in body -@pytest.mark.django_db -def test_the_changelist_offers_an_import_button(staff_client): - body = staff_client.get( - reverse("admin:catalog_service_changelist") - ).content.decode() - assert "Import from URL" in body - - @pytest.mark.django_db def test_importing_from_the_admin_creates_the_service(staff_client, monkeypatch): from catalog.models import Service monkeypatch.setattr( "catalog.admin.import_from_url", - lambda url: (Service.objects.create(name="Imported"), True), + lambda url, author=None: (Service.objects.create(name="Imported"), True), ) response = staff_client.post( "/admin/catalog/service/import-from-url/", @@ -195,11 +255,33 @@ def test_importing_from_the_admin_creates_the_service(staff_client, monkeypatch) assert Service.objects.filter(name="Imported").exists() +@pytest.mark.django_db +def test_importing_from_the_admin_records_the_admin_who_asked( + staff_client, monkeypatch +): + # The action has a request, so the row names the person. Dropping + # the caller signs the bot for work an admin did. + from catalog.models import Service + + def recording_import(url, author=None): + return Service.objects.create(name="Imported", created_by=author), True + + monkeypatch.setattr("catalog.admin.import_from_url", recording_import) + staff_client.post( + "/admin/catalog/service/import-from-url/", + {"status_page_url": "https://status.example.com/", "_form_submitted": "1"}, + ) + + assert Service.objects.get().created_by == User.objects.get( + email="admin@example.com" + ) + + @pytest.mark.django_db def test_an_unreadable_page_is_reported_not_a_500(staff_client, monkeypatch): # The URL came from a person pasting a stranger's page. Anything can # come back from it. - def boom(url): + def boom(url, author=None): raise ValueError("not a status page") monkeypatch.setattr("catalog.admin.import_from_url", boom) @@ -218,6 +300,9 @@ def test_events_can_be_filtered_by_phase(staff_client): An incident and a maintenance window move through different phases, which is why a plain ChoicesDropdownFilter cannot do this. + + The two option labels are what prove the filter is declared. `phase` + is a local field, so the queryset narrows with or without it. """ from django.utils import timezone @@ -273,7 +358,6 @@ def test_every_table_links_somewhere(staff_client, url_name): from django.utils import timezone - from catalog.models import Service from polling.models import PollRun from status.choices import EventKind, IncidentPhase, Severity, StatusSource from status.models import ComponentStatus, ServiceEvent @@ -304,20 +388,19 @@ def test_every_table_links_somewhere(staff_client, url_name): starts_at=timezone.now(), poll_run=run, ) - assert Service.objects.filter(pk=service.pk).exists() body = staff_client.get(reverse(url_name)).content.decode() - assert re.search(r'href="/admin/\w+/\w+/[0-9a-f-]{36}/change/"', body), ( + assert re.search(r'href="/admin/\w+/\w+/[^"]+/change/"', body), ( f"{url_name} has no row that leads anywhere" ) @pytest.mark.django_db -def test_a_service_shows_its_logo_and_falls_back_to_an_initial(staff_client): - """A missing logo looks incomplete; a wrong one names the wrong thing. +def test_a_service_shows_its_logo(staff_client): + """A wrong logo names the wrong thing, so the column carries its own. - Unfold drops the initials when there is an image, which is the - fallback the spec asks for. + The row with no logo is here because the column renders both ways. + Unfold, not this project, decides what stands in for the image. """ from tests.factories import ServiceFactory @@ -328,7 +411,6 @@ def test_a_service_shows_its_logo_and_falls_back_to_an_initial(staff_client): reverse("admin:catalog_service_changelist") ).content.decode() assert "https://cdn.example/mark.png" in body - assert "NO" in body # the initial standing in for the missing one @pytest.mark.django_db @@ -345,7 +427,7 @@ def test_a_blank_interval_says_what_it_will_do(staff_client, settings): body = staff_client.get( reverse("admin:polling_poller_change", args=[service.poller.pk]) ).content.decode() - assert "deployment default of 900 seconds" in body + assert "900 seconds" in body @pytest.mark.django_db @@ -432,21 +514,25 @@ def test_a_poll_run_reaches_what_it_wrote(staff_client): @pytest.mark.django_db def test_a_reading_says_which_poll_wrote_it(staff_client): - # It is not an editable column, so the record is the only place left - # to say it. Without this you could read it on the table alone. + # The run is not an editable column. A fieldset naming it is what + # carries the link onto the record, and a link is the useful form. from status.choices import Severity, StatusSource from status.models import ComponentStatus - from tests.factories import ComponentFactory + from tests.factories import ComponentFactory, poll_run + component = ComponentFactory() + run = poll_run(component.service) reading = ComponentStatus.objects.create( - component=ComponentFactory(), + component=component, severity=Severity.OPERATIONAL, source=StatusSource.PROVIDER, started_at=timezone.now(), + poll_run=run, ) url = reverse("admin:status_componentstatus_change", args=[reading.pk]) - assert "Written by" in staff_client.get(url).content.decode() + body = staff_client.get(url).content.decode() + assert reverse("admin:polling_pollrun_change", args=[run.pk]) in body PROJECT_APPS = {"authentication", "catalog", "dashboards", "polling", "status"} @@ -456,32 +542,14 @@ def test_a_reading_says_which_poll_wrote_it(staff_client): ] -@pytest.mark.parametrize("model", PROJECT_ADMINS, ids=lambda m: m._meta.label) -def test_every_filter_names_a_real_path(model): - """A filter naming a path that does not exist fails only when used. - - Django answers an unrecognised lookup with a redirect. A broken one - reads as an empty page, not an error. A typo stays invisible until - somebody picks that filter. - """ - for entry in admin.site._registry[model].list_filter or []: - path = entry[0] if isinstance(entry, tuple) else entry - if not isinstance(path, str): - continue # A filter class brings its own queryset. - target = model - for part in path.split("__"): - field = target._meta.get_field(part) - if field.is_relation: - target = field.related_model - - @pytest.mark.django_db @pytest.mark.parametrize("model", PROJECT_ADMINS, ids=lambda m: m._meta.label) def test_every_filter_choice_opens(staff_client, model): """Every option a filter offers is one somebody can click. - The paths are checked above. This is the other half: the option the - page renders has to be one the changelist will accept back. + Django's own `admin.E116` proves each path resolves. This is the + other half: the option the page renders has to be one the + changelist will accept back. """ from django.test import RequestFactory @@ -517,16 +585,72 @@ def test_every_search_field_names_a_real_path(model): target = field.related_model -@pytest.mark.django_db -@pytest.mark.parametrize("model", PROJECT_ADMINS, ids=lambda m: m._meta.label) -def test_every_table_can_be_searched(staff_client, model): - # Every row hangs off a service somewhere, so the service's name is - # the one term that should reach every table. - opts = model._meta - url = reverse(f"admin:{opts.app_label}_{opts.model_name}_changelist") +# Every row on these tables hangs off a service. A user and a board do +# not, so no service name reaches them. +SERVICE_SEARCHABLE = [ + "admin:catalog_service_changelist", + "admin:catalog_servicecomponent_changelist", + "admin:dashboards_dashboarditem_changelist", + "admin:polling_poller_changelist", + "admin:polling_pollrun_changelist", + "admin:status_componentstatus_changelist", + "admin:status_eventupdate_changelist", + "admin:status_serviceevent_changelist", +] - assert staff_client.get(url, {"q": "nothing matches this"}).status_code == 200 - assert admin.site._registry[model].search_fields, opts.label + +@pytest.fixture +def two_services(db): + """One row under each of two services, on every table above. + + The second service is what proves a search narrows. A table + holding one row is found by a search that matches everything. + """ + from status.choices import ( + EventKind, + EventSource, + IncidentPhase, + Severity, + StatusSource, + ) + from status.models import ComponentStatus, EventUpdate, ServiceEvent + from tests.factories import ServiceFactory, poll_run + + for name in ("Twilio", "Stripe"): + service = ServiceFactory(name=name, tracked=1) + run = poll_run(service) + ComponentStatus.objects.create( + component=service.components.get(), + severity=Severity.OPERATIONAL, + source=StatusSource.PROVIDER, + started_at=timezone.now(), + poll_run=run, + ) + event = ServiceEvent.objects.create( + service=service, + external_id="1", + kind=EventKind.INCIDENT, + title="An outage", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + poll_run=run, + ) + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.INVESTIGATING, + body="Looking into it", + posted_at=timezone.now(), + source=EventSource.PROVIDER, + ) + + +@pytest.mark.django_db +@pytest.mark.parametrize("url_name", SERVICE_SEARCHABLE) +def test_a_service_name_reaches_the_rows_under_it(staff_client, two_services, url_name): + # A table holds every service's rows. Typing the name is how a + # person reads one service's, instead of building a filter by hand. + response = staff_client.get(reverse(url_name), {"q": "Twilio"}) + assert response.context["cl"].result_count == 1 @pytest.mark.django_db @@ -626,6 +750,209 @@ def test_a_person_cannot_open_somebody_elses_board(): mine.full_clean() +@pytest.mark.django_db +def test_the_component_list_shows_watchers(admin_client, db): + # The count is an annotation. A column that forgets to annotate it + # raises rather than showing a wrong number. A column with no + # ordering field cannot be sorted at all. + from catalog.models import ServiceComponent + from tests.factories import ComponentFactory, track + + ComponentFactory(name="Quiet") + loud = ComponentFactory(name="Loud") + track(loud) + track(loud) + + model_admin = admin.site._registry[ServiceComponent] + index = model_admin.list_display.index("watchers") + 1 + response = admin_client.get( + reverse("admin:catalog_servicecomponent_changelist"), {"o": f"-{index}"} + ) + assert response.status_code == 200 + assert b">2<" in response.content + assert response.content.index(b"Loud") < response.content.index(b"Quiet") + + +@pytest.mark.django_db +def test_service_requests_are_listed_by_demand(admin_client, db): + # The list is the demand signal for what the catalog is missing, + # so the most-asked-for URL is the first row. + from catalog.models import ServiceRequest + + ServiceRequest.objects.create(url="https://a.example", request_count=1) + ServiceRequest.objects.create(url="https://b.example", request_count=9) + response = admin_client.get(reverse("admin:catalog_servicerequest_changelist")) + assert response.status_code == 200 + assert response.content.index(b"b.example") < response.content.index(b"a.example") + + +def test_every_registered_admin_is_reachable(): + # A registered admin nothing links to is a page nobody finds, the + # way magic links once shipped. Reachable means named in the + # sidebar, named in a tab group, or shown as a parent's inline. + # A hardcoded title list cannot tell that apart from an admin + # reachable by nothing at all. + registered = { + model._meta.label + for model in admin.site._registry + if model._meta.app_label in OUR_APPS + } + top_level = _sidebar_reachable() | _tab_reachable() + reachable = top_level | _inline_reachable(top_level) + missing = registered - reachable + assert not missing, f"registered but unreachable: {sorted(missing)}" + + +def _change_view_names(model_admin, request): + """Every name the change view puts on the page. + + An inline is part of the form. A component's severity history is a + table on it, so the column that summarises it has a place there. + """ + from django.contrib.admin.utils import flatten_fieldsets + + names = set(flatten_fieldsets(model_admin.get_fieldsets(request))) + for inline in model_admin.inlines: + names.update(inline.fields or []) + names.update(field.name for field in inline.model._meta.concrete_fields) + return names + + +@pytest.mark.django_db +@pytest.mark.parametrize("model", PROJECT_ADMINS, ids=lambda m: m._meta.label) +def test_every_column_has_a_place_on_the_form(staff_client, model): + """A column with no counterpart on the change view was found twice. + + Opening the record then answers less than the list did. A list of + the known gaps would go stale, so the pairing is read off the + registry. + + A column counts as answered by its own name on the form, or by the + field it sorts on. It cannot see a counterpart under another name. + """ + model_admin = admin.site._registry[model] + request = staff_client.request().wsgi_request + covered = _change_view_names(model_admin, request) + list_display = model_admin.get_list_display(request) + linked = model_admin.get_list_display_links(request, list_display) or [] + + missing = [] + for entry in list_display: + column = getattr(model_admin, entry, None) + # The linked column opens the record, and a dropdown holds links + # to other tables. Neither carries a value of its own. + if entry in linked or getattr(column, "dropdown", False): + continue + ordering = getattr(column, "admin_order_field", "") + root = ordering.split("__")[0] if isinstance(ordering, str) else "" + if {entry, root} & covered: + continue + missing.append(entry) + assert not missing, f"{model._meta.label}: no place on the form for {missing}" + + +def _tab_strip(body): + """The tab strip's own markup, not the whole page. + + The sidebar links to a service or a poller too, on every admin + page. Searching the full body would pass on that link alone and + never prove a tab exists. + """ + start = body.index('id="tabs-items"') + return body[start : body.index("", start)] + + +@pytest.mark.django_db +def test_the_catalog_tabs_carry_service_requests(staff_client): + # A sidebar link once pointed here on its own. A thing that belongs + # to the catalog belongs on the catalog's tabs instead. + body = staff_client.get( + reverse("admin:catalog_service_changelist") + ).content.decode() + tabs = _tab_strip(body) + assert 'href="/admin/catalog/servicerequest/"' in tabs + + +@pytest.mark.django_db +def test_the_user_tabs_carry_magic_links(staff_client): + # MagicLinkTokenAdmin answered but nothing pointed at it. A token + # belongs to a user, so its tab sits on the user's page. + body = staff_client.get( + reverse("admin:authentication_user_changelist") + ).content.decode() + tabs = _tab_strip(body) + assert 'href="/admin/authentication/magiclinktoken/"' in tabs + + +@pytest.mark.django_db +def test_the_component_list_formset_flips_the_flag(admin_client, db): + # A mixed selection defeats the two actions in one pass. One row is + # featured, the other cleared, one Save. + from tests.factories import ComponentFactory, ServiceFactory + + service = ServiceFactory() + to_feature = ComponentFactory(service=service, is_overall=True, is_featured=False) + to_clear = ComponentFactory(service=service, is_overall=False, is_featured=True) + admin_client.post( + reverse("admin:catalog_servicecomponent_changelist"), + { + "form-TOTAL_FORMS": "2", + "form-INITIAL_FORMS": "2", + "form-MIN_NUM_FORMS": "0", + "form-MAX_NUM_FORMS": "1000", + "form-0-id": str(to_feature.pk), + "form-0-is_featured": "on", + "form-1-id": str(to_clear.pk), + "_save": "Save", + }, + ) + to_feature.refresh_from_db() + to_clear.refresh_from_db() + assert to_feature.is_featured is True + assert to_clear.is_featured is False + + +@pytest.mark.django_db +def test_the_poller_list_formset_flips_the_pause(admin_client, db): + # A provider going down means pausing several pollers at once. + # One Save pauses one and resumes another. + from tests.factories import PollerFactory, ServiceFactory + + to_pause = PollerFactory(service=ServiceFactory(), is_paused=False) + to_resume = PollerFactory(service=ServiceFactory(), is_paused=True) + admin_client.post( + reverse("admin:polling_poller_changelist"), + { + "form-TOTAL_FORMS": "2", + "form-INITIAL_FORMS": "2", + "form-MIN_NUM_FORMS": "0", + "form-MAX_NUM_FORMS": "1000", + "form-0-id": str(to_pause.pk), + "form-0-is_paused": "on", + "form-1-id": str(to_resume.pk), + "_save": "Save", + }, + ) + to_pause.refresh_from_db() + to_resume.refresh_from_db() + assert to_pause.is_paused is True + assert to_resume.is_paused is False + + +@pytest.mark.django_db +def test_the_component_form_features_a_leaf(staff_client): + # The change page is where an admin already is when they decide to + # feature something. Without the field the form cannot set it. + from tests.factories import ComponentFactory + + leaf = ComponentFactory(is_overall=False, is_featured=False) + url, data = component_form_data(staff_client, leaf, is_featured="on") + staff_client.post(url, data) + + leaf.refresh_from_db() + assert leaf.is_featured is True + + @pytest.mark.django_db def test_the_polling_schedule_page_opens(staff_client): """Readonly is not free: Django drops a readonly field from the form. @@ -643,3 +970,244 @@ def test_the_polling_schedule_page_opens(staff_client): url = reverse("admin:django_celery_beat_periodictask_change", args=[task.pk]) assert staff_client.get(url).status_code == 200 + + +@pytest.fixture +def claimed_event(): + """An outage we found first, later claimed by the provider. + + Both updates hang off one event. That interleaving is what the + claim mechanism produces, and what the admin has to tell apart. + """ + from status.choices import EventKind, EventSource, IncidentPhase + from status.models import EventUpdate, ServiceEvent + from tests.factories import ServiceFactory + + service = ServiceFactory(name="Twilio") + ours = ServiceEvent.objects.create( + service=service, + kind=EventKind.INCIDENT, + title="We saw it first", + phase=IncidentPhase.DETECTED, + starts_at=timezone.now(), + detected_by=EventSource.SYSTEM, + ) + theirs = ServiceEvent.objects.create( + service=service, + external_id="their-1", + kind=EventKind.INCIDENT, + title="They saw it first", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + detected_by=EventSource.PROVIDER, + ) + EventUpdate.objects.create( + event=ours, + phase=IncidentPhase.DETECTED, + body="Our detection", + posted_at=timezone.now(), + source=EventSource.SYSTEM, + ) + EventUpdate.objects.create( + event=ours, + phase=IncidentPhase.INVESTIGATING, + body="Their first post", + posted_at=timezone.now(), + source=EventSource.PROVIDER, + ) + return ours, theirs + + +@pytest.mark.django_db +def test_the_event_list_says_who_found_the_outage(staff_client, claimed_event): + # `detected_by` answers "did we find this first". It was on no + # changelist, so the table could not be read for it. Pin the cell, + # not the word. The filter beside it offers the same one. + body = staff_client.get( + reverse("admin:status_serviceevent_changelist") + ).content.decode() + assert ">Statusboard" in body + + +@pytest.mark.django_db +def test_events_can_be_filtered_by_who_detected_them(staff_client, claimed_event): + # Two halves. The option has to be offered, and picking it has to + # narrow. Django allows a local-field lookup whether or not a filter + # declares it, so the queryset alone proves nothing. + from status.choices import EventSource + + ours, _theirs = claimed_event + url = reverse("admin:status_serviceevent_changelist") + assert "detected_by__exact" in staff_client.get(url).content.decode() + + response = staff_client.get(url, {"detected_by__exact": EventSource.SYSTEM}) + assert list(response.context["cl"].queryset) == [ours] + + +@pytest.mark.django_db +def test_the_event_page_says_who_found_the_outage(staff_client, claimed_event): + # A claim fills `external_id` in, so the record alone cannot say who + # opened the event. Only this field can. Pin the label, not the + # value. The timeline below says Statusboard too. + ours, _theirs = claimed_event + url = reverse("admin:status_serviceevent_change", args=[ours.pk]) + assert "Detected by" in staff_client.get(url).content.decode() + + +@pytest.mark.django_db +def test_an_events_timeline_says_who_wrote_each_update(staff_client, claimed_event): + # We opened this event, so the only Provider on the page is an + # update the provider wrote. Without the source the timeline reads + # as one author. + ours, _theirs = claimed_event + url = reverse("admin:status_serviceevent_change", args=[ours.pk]) + assert ">Provider<" in staff_client.get(url).content.decode() + + +@pytest.mark.django_db +def test_the_update_list_says_who_wrote_each_post(staff_client, claimed_event): + # The same question on the updates table, where a person arrives + # from a filtered link rather than from the event. + body = staff_client.get( + reverse("admin:status_eventupdate_changelist") + ).content.decode() + assert ">Statusboard" in body + assert ">Provider" in body + + +@pytest.mark.django_db +def test_updates_can_be_filtered_by_who_wrote_them(staff_client, claimed_event): + # Offered and narrowing, for the same reason as the event filter. + from status.choices import EventSource + + ours, _theirs = claimed_event + url = reverse("admin:status_eventupdate_changelist") + assert "source__exact" in staff_client.get(url).content.decode() + + response = staff_client.get(url, {"source__exact": EventSource.SYSTEM}) + assert list(response.context["cl"].queryset) == [ + ours.updates.get(body="Our detection") + ] + + +def component_form_data(staff_client, component, **fields): + """The component change form, with a management form per inline. + + The prefixes are read off the rendered page. A browser posts what + the page asks for, and a missing management form is a 200 that + saved nothing. + """ + import re + + url = reverse("admin:catalog_servicecomponent_change", args=[component.pk]) + body = staff_client.get(url).content.decode() + data = { + "service": str(component.service_id), + "name": component.name, + "external_id": component.external_id, + "is_archived": "", + "parent": "", + "status_page_order": str(component.status_page_order), + "is_overall": "", + **fields, + } + for prefix in set(re.findall(r'name="([\w-]+)-TOTAL_FORMS"', body)): + for key in ("TOTAL_FORMS", "INITIAL_FORMS", "MIN_NUM_FORMS", "MAX_NUM_FORMS"): + data[f"{prefix}-{key}"] = "0" + return url, data + + +@pytest.mark.django_db +def test_reparenting_in_the_admin_moves_the_component(staff_client): + """The admin posts a form, so it reaches the model by another road. + + A component reparented here has to answer `?ancestor=` under its + new parent at once. An untracked service is never polled, so + nothing else would come along and correct it. + """ + from tests.factories import ComponentFactory, ServiceFactory, ancestry, descendants + + service = ServiceFactory(name="Twilio") + parent = ComponentFactory(service=service, name="Programmable Messaging") + child = ComponentFactory(service=service, name="SMS") + + url, data = component_form_data(staff_client, child, parent=str(parent.pk)) + assert staff_client.post(url, data).status_code == 302 + + child.refresh_from_db() + assert ancestry(child) == [parent.pk] + assert descendants(parent) == {child.pk} + + +@pytest.mark.django_db +def test_the_parent_picker_refuses_the_rollup(staff_client): + """The queryset, not just `clean`, is what a posted form meets first. + + A `ModelChoiceField` rejects a posted pk outside its queryset before + the instance is ever built, so `clean` never runs. Narrowing the + queryset is what turns a stray click into a redisplayed form + instead of a trigger's `IntegrityError`. + """ + from tests.factories import ComponentFactory, ServiceFactory + + service = ServiceFactory(name="Twilio") + rollup = ComponentFactory(service=service, is_overall=True) + child = ComponentFactory(service=service, name="SMS") + + url, data = component_form_data(staff_client, child, parent=str(rollup.pk)) + response = staff_client.post(url, data) + + assert response.status_code == 200 + assert b"valid choice" in response.content + child.refresh_from_db() + assert child.parent_id is None + + +def related_item(admin_user, service, label): + """One entry of a service row's View dropdown, count and link.""" + from django.test import RequestFactory + + from catalog.models import Service + + request = RequestFactory().get("/") + request.user = admin_user + model_admin = admin.site._registry[Service] + row = model_admin.get_queryset(request).get(pk=service.pk) + items = model_admin.display_related(row)["items"] + return next(item for item in items if str(item["title"]).startswith(label)) + + +@pytest.mark.django_db +def test_the_service_row_counts_the_components_the_api_counts(admin_client): + # `Service.component_count` leaves out the rollup and the archived + # rows. A second number under the same name is how the admin and a + # client come to disagree about one service. + from tests.factories import ComponentFactory, ServiceFactory + + admin_user = User.objects.get(email="admin@example.com") + service = ServiceFactory() + ComponentFactory(service=service, is_overall=True) + ComponentFactory(service=service) + gone = ComponentFactory(service=service) + gone.is_archived = True + gone.save(update_fields=["is_archived"]) + + assert related_item(admin_user, service, "Components")["title"] == "Components (1)" + + +@pytest.mark.django_db +def test_the_components_link_opens_exactly_what_it_counts(admin_client): + # The count sits on the label of a link. A number that does not + # describe the list it opens is worse than no number. + from tests.factories import ComponentFactory, ServiceFactory + + admin_user = User.objects.get(email="admin@example.com") + service = ServiceFactory() + ComponentFactory(service=service, is_overall=True) + ComponentFactory(service=service) + gone = ComponentFactory(service=service) + gone.is_archived = True + gone.save(update_fields=["is_archived"]) + + link = related_item(admin_user, service, "Components")["link"] + assert admin_client.get(link).context["cl"].result_count == 1 diff --git a/api/tests/test_archived.py b/api/tests/test_archived.py new file mode 100644 index 0000000..3d58723 --- /dev/null +++ b/api/tests/test_archived.py @@ -0,0 +1,253 @@ +"""The archived rule, against every reader that serves components. + +A provider stops publishing a component. The row is archived rather +than deleted, because somebody may be tracking it. +`ServiceComponentQuerySet.visible` is the one definition of which rows +a caller is served. + +Three readers have forgotten to call it: the Affects badge, +`tracked_component_count` and the board write. Each was caught by hand +in review. Remembering has failed three times, so the suite asks +instead. + +The last test derives its list of paths from the contract. So it fails +on an operation that serves a component and has no probe. It says +nothing about a new reader on a path a probe already covers. Another +query parameter, or a nested component on a schema that path returns, +adds no path and trips nothing. +""" + +from pathlib import Path +from unittest import mock + +import pytest +import yaml +from django.urls import reverse +from django.utils import timezone +from rest_framework.test import APIClient + +from authentication.models import User +from dashboards.models import DashboardItem +from status.choices import EventKind, IncidentPhase +from status.models import ServiceEvent +from tests.conftest import jwt_client +from tests.factories import ComponentFactory, ServiceFactory, StatusPageFactory + +CONTRACT = Path(__file__).resolve().parents[2] / "docs" / "api" / "openapi.yaml" + +pytestmark = pytest.mark.django_db + + +def archived(service=None, **fields): + """A component the provider stopped publishing.""" + row = ComponentFactory(service=service or ServiceFactory(), **fields) + row.is_archived = True + row.save(update_fields=["is_archived"]) + return row + + +def signed_in(): + return jwt_client(User.objects.create(email="archived-probe@example.test")) + + +def incident(service): + return ServiceEvent.objects.create( + service=service, + external_id="probe", + kind=EventKind.INCIDENT, + title="x", + phase=IncidentPhase.DETECTED, + starts_at=timezone.now(), + ) + + +# Each probe archives one component and returns what a caller can still +# see of it. Anything truthy is a leak. + + +def component_list(): + gone = archived() + body = APIClient().get(reverse("component-list"), {"service": gone.service.slug}) + return body.json()["results"] or body.json()["aggregates"]["total"] + + +def descendants_of_a_component(): + service = ServiceFactory() + top = ComponentFactory(service=service) + archived(service=service, parent=top) + url = reverse("component-list") + return APIClient().get(url, {"ancestor": str(top.id)}).json()["results"] + + +def an_events_affected_list(): + gone = archived() + event = incident(gone.service) + event.affected_components.add(gone) + url = reverse("component-list") + return APIClient().get(url, {"event": str(event.id)}).json()["results"] + + +def component_detail(): + gone = archived() + response = APIClient().get(reverse("component-detail", args=[gone.id])) + return response.status_code != 404 + + +def a_components_path(): + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = archived(service=service, parent=top) + leaf = ComponentFactory(service=service, parent=middle) + body = APIClient().get(reverse("component-detail", args=[leaf.id])).json() + return [node for node in body["path"] if node["id"] == str(middle.id)] + + +def a_components_descendant_count(): + service = ServiceFactory() + top = ComponentFactory(service=service) + archived(service=service, parent=top) + body = APIClient().get(reverse("component-detail", args=[top.id])).json() + return body["descendant_count"] + + +def a_services_component_count(): + service = ServiceFactory() + StatusPageFactory(service=service) + archived(service=service) + url = reverse("service-detail", kwargs={"slug": service.slug}) + return APIClient().get(url).json()["component_count"] + + +def a_services_tracked_count(): + service = ServiceFactory() + StatusPageFactory(service=service) + gone = archived(service=service) + caller = signed_in() + DashboardItem.objects.create( + dashboard=User.objects.get( + email="archived-probe@example.test" + ).default_dashboard, + component=gone, + ) + url = reverse("service-detail", kwargs={"slug": service.slug}) + return caller.get(url).json()["tracked_component_count"] + + +def an_imported_services_counts(): + service = ServiceFactory() + StatusPageFactory(service=service) + archived(service=service) + with ( + mock.patch("polling.fetch.check", lambda url: url), + mock.patch( + "catalog.views.imports.import_from_url", return_value=(service, False) + ), + ): + body = APIClient().post( + reverse("catalog-import"), + {"status_page_url": "https://status.example.com/"}, + format="json", + ) + return body.json()["component_count"] + + +def the_board_list(): + caller = signed_in() + board = User.objects.get(email="archived-probe@example.test").default_dashboard + gone = archived() + DashboardItem.objects.create(dashboard=board, component=gone) + url = reverse("board-components", kwargs={"uuid": board.id}) + body = caller.get(url).json() + return body["results"] or body["aggregates"]["total"] + + +def the_track_write(): + caller = signed_in() + board = User.objects.get(email="archived-probe@example.test").default_dashboard + gone = archived() + url = reverse("board-components", kwargs={"uuid": board.id}) + response = caller.post(url, {"component_id": str(gone.id)}, format="json") + return response.status_code != 404 + + +def the_affects_badge(): + gone = archived() + event = incident(gone.service) + event.affected_components.add(gone) + url = reverse("event-detail", kwargs={"uuid": event.id}) + return APIClient().get(url).json()["affected_count"] + + +# One entry per reader, against the contract path it answers on. The +# path is what the coverage check below reads. +ARCHIVED_PROBES = { + "the component list": ("/catalog/components/", component_list), + "a component's descendants": ("/catalog/components/", descendants_of_a_component), + "an event's affected list": ("/catalog/components/", an_events_affected_list), + "the component detail": ("/catalog/components/{uuid}/", component_detail), + "a component's path": ("/catalog/components/{uuid}/", a_components_path), + "a component's descendant count": ( + "/catalog/components/{uuid}/", + a_components_descendant_count, + ), + "a service's component count": ( + "/catalog/services/{slug}/", + a_services_component_count, + ), + "a service's tracked count": ( + "/catalog/services/{slug}/", + a_services_tracked_count, + ), + "an imported service's counts": ("/catalog/import/", an_imported_services_counts), + "the board list": ("/dashboards/{uuid}/components/", the_board_list), + "the track write": ("/dashboards/{uuid}/components/", the_track_write), + "the Affects badge": ("/events/{uuid}/", the_affects_badge), +} + + +@pytest.mark.parametrize("reader", sorted(ARCHIVED_PROBES)) +def test_no_reader_serves_an_archived_component(reader): + # The one exception is `Service.overall_component`. A service whose + # rollup is archived still needs a status in its header, and + # `test_catalog_api` covers that. + _path, probe = ARCHIVED_PROBES[reader] + leaked = probe() + assert not leaked, f"{reader} served an archived component: {leaked}" + + +def _reachable(node, schemas, found=None): + """Every schema an operation can produce, following each `$ref`.""" + found = set() if found is None else found + if isinstance(node, list): + for item in node: + _reachable(item, schemas, found) + return found + if not isinstance(node, dict): + return found + for key, value in node.items(): + if key == "$ref" and value.startswith("#/components/schemas/"): + name = value.rsplit("/", 1)[1] + if name not in found: + found.add(name) + _reachable(schemas.get(name, {}), schemas, found) + else: + _reachable(value, schemas, found) + return found + + +def test_the_probes_cover_every_operation_that_can_serve_a_component(): + # A reader added later that forgets `visible()` serves a row the + # provider stopped publishing, and nothing says so. This fails by + # path, the way the `?fields=` probes do. + contract = yaml.safe_load(CONTRACT.read_text()) + schemas = contract["components"]["schemas"] + serves = { + path + for path, operations in contract["paths"].items() + for operation in operations.values() + if "Component" in _reachable(operation.get("responses", {}), schemas) + } + probed = {path for path, _probe in ARCHIVED_PROBES.values()} + assert not serves - probed, ( + f"serves components, unprobed: {sorted(serves - probed)}" + ) diff --git a/api/tests/test_auth.py b/api/tests/test_auth.py index 243677d..2abdf51 100644 --- a/api/tests/test_auth.py +++ b/api/tests/test_auth.py @@ -22,7 +22,9 @@ def test_requesting_a_link_sends_one_and_creates_a_token(): response = APIClient().post( reverse("magic-link"), {"email": "a@b.com"}, format="json" ) - assert response.status_code == 204 + # 202, not 204. We accept the address and say nothing about whether + # it can receive mail, which is what the contract promises. + assert response.status_code == 202 assert len(mail.outbox) == 1 assert MagicLinkToken.objects.filter(user__email="a@b.com").count() == 1 @@ -36,7 +38,7 @@ def test_requesting_a_link_for_an_unknown_email_looks_identical(): second = APIClient().post( reverse("magic-link"), {"email": "new@b.com"}, format="json" ) - assert first.status_code == second.status_code == 204 + assert first.status_code == second.status_code == 202 @pytest.mark.django_db @@ -152,9 +154,7 @@ def test_the_subject_is_one_line(): # A header cannot hold a newline, and a template file ends with one. APIClient().post(reverse("magic-link"), {"email": "a@b.com"}, format="json") - subject = mail.outbox[0].subject - assert "\n" not in subject - assert subject == "Your sign-in link for Statusboard" + assert "\n" not in mail.outbox[0].subject def test_no_app_means_no_link_rather_than_a_link_to_nowhere(settings): @@ -188,3 +188,28 @@ def test_the_sign_in_page_is_not_the_endpoint_it_calls(): from django.urls import reverse assert settings.APP_MAGIC_LINK_PATH != reverse("verify") + + +@pytest.mark.django_db +def test_a_link_request_with_no_email_says_which_shape_it_refuses_in(): + # Hand rolled, so it is a bare `detail` with no `code`. The contract + # documents that shape rather than claiming the `Error` one. + response = APIClient().post(reverse("magic-link"), {}, format="json") + assert response.status_code == 400 + assert response.json() == {"detail": "email is required"} + + +@pytest.mark.django_db +def test_logging_out_answers_with_an_empty_object(): + # simplejwt's view answers 200 and a body, not 204. The contract + # documents what ships, because a client parses the response. + APIClient().post(reverse("magic-link"), {"email": "a@b.com"}, format="json") + token = MagicLinkToken.objects.get().token + pair = APIClient().post(reverse("verify"), {"token": token}, format="json").json() + + response = APIClient().post( + reverse("logout"), {"refresh": pair["refresh"]}, format="json" + ) + + assert response.status_code == 200 + assert response.json() == {} diff --git a/api/tests/test_board_api.py b/api/tests/test_board_api.py index 14e3866..ab8deb4 100644 --- a/api/tests/test_board_api.py +++ b/api/tests/test_board_api.py @@ -8,7 +8,6 @@ from status.choices import ( EventKind, IncidentPhase, - MaintenancePhase, Severity, StatusSource, ) @@ -22,20 +21,11 @@ @pytest.fixture -def board(db): - user = User.objects.create(email="a@b.com") - return user.default_dashboard - - -@pytest.fixture -def client(board): - from rest_framework_simplejwt.tokens import RefreshToken - - api = APIClient() - api.credentials( - HTTP_AUTHORIZATION=f"Bearer {RefreshToken.for_user(board.owner).access_token}" - ) - return api +def client(authenticated_client): + # Every test in this module reads or writes one board as its + # owner. The plain `client` fixture stays anonymous everywhere + # else, so this override is local to this module. + return authenticated_client def _track(board, severity=Severity.OPERATIONAL, slug=None): @@ -81,85 +71,66 @@ def test_all_returns_every_tracked_component(client, board): @pytest.mark.django_db -def test_the_incidents_tab_filters_on_event_kind(client, board): - with_incident = _track(board) - _event(with_incident, EventKind.INCIDENT, IncidentPhase.INVESTIGATING) +def test_the_board_leaves_out_an_archived_component(client, board): + # The provider stopped publishing it, so no screen shows it. A board + # row that no other list holds would have nowhere to link to. _track(board) - url = reverse("board-components", kwargs={"uuid": board.id}) + "?event=incident" - assert len(client.get(url).json()["results"]) == 1 - - -@pytest.mark.django_db -def test_a_resolved_incident_does_not_put_a_row_on_the_incidents_tab(client, board): - component = _track(board) - _event( - component, EventKind.INCIDENT, IncidentPhase.RESOLVED, ends_at=timezone.now() - ) - url = reverse("board-components", kwargs={"uuid": board.id}) + "?event=incident" - assert client.get(url).json()["results"] == [] + gone = _track(board) + gone.is_archived = True + gone.save(update_fields=["is_archived"]) - -@pytest.mark.django_db -def test_the_event_filter_binds_both_conditions_to_the_same_event(client, board): - # `ServiceEvent` and `ServiceComponent` are many-to-many. - # Two parameters filter twice and can match two different events. - component = _track(board) - _event( - component, EventKind.INCIDENT, IncidentPhase.RESOLVED, ends_at=timezone.now() - ) - _event(component, EventKind.MAINTENANCE, MaintenancePhase.IN_PROGRESS) - url = reverse("board-components", kwargs={"uuid": board.id}) + "?event=incident" - assert client.get(url).json()["results"] == [] - - -@pytest.mark.django_db -def test_scheduled_maintenance_appears_on_the_maintenance_tab_while_reading_operational( - client, board -): - # A window three days out leaves severity 5. - # Severity cannot select it, so the tab filters on the event. - component = _track(board, severity=Severity.OPERATIONAL) - _event( - component, - EventKind.MAINTENANCE, - MaintenancePhase.SCHEDULED, - starts_at=timezone.now() + timezone.timedelta(days=3), - ) - url = reverse("board-components", kwargs={"uuid": board.id}) + "?event=maintenance" - rows = client.get(url).json()["results"] - assert len(rows) == 1 - assert rows[0]["status"]["severity"] == Severity.OPERATIONAL + url = reverse("board-components", kwargs={"uuid": board.id}) + body = client.get(url).json() + assert body["aggregates"]["total"] == 1 + assert str(gone.id) not in [r["id"] for r in body["results"]] @pytest.mark.django_db def test_one_fetch_fills_every_chip(client, board): - # One response must fill all three chips. + # One response must fill every chip on the board. # Otherwise the client makes a request per chip. - broken = _track(board, severity=Severity.MAJOR_OUTAGE) - _event(broken, EventKind.INCIDENT, IncidentPhase.INVESTIGATING) - planned = _track(board) - _event(planned, EventKind.MAINTENANCE, MaintenancePhase.SCHEDULED) + _track(board, severity=Severity.MAJOR_OUTAGE) + _track(board, severity=Severity.OPERATIONAL) aggregates = client.get( reverse("board-components", kwargs={"uuid": board.id}) ).json()["aggregates"] assert aggregates["total"] == 2 - assert aggregates["by_event_kind"]["incident"] == 1 - assert aggregates["by_event_kind"]["maintenance"] == 1 + assert aggregates["by_severity"][str(Severity.MAJOR_OUTAGE)] == 1 + assert aggregates["by_severity"][str(Severity.OPERATIONAL)] == 1 assert "next_refresh_at" in aggregates assert "oldest_refreshed_at" in aggregates @pytest.mark.django_db -def test_severity_composes_with_a_tab(client, board): - down = _track(board, severity=Severity.MAJOR_OUTAGE) - _event(down, EventKind.INCIDENT, IncidentPhase.INVESTIGATING) - up = _track(board, severity=Severity.OPERATIONAL) - _event(up, EventKind.INCIDENT, IncidentPhase.MONITORING) - url = ( - reverse("board-components", kwargs={"uuid": board.id}) - + "?event=incident&status__severity__lte=3" +def test_the_board_takes_several_severities_at_once(client, board): + # The Severity filter offers all six values. A single exact match + # could not express "everything that needs attention". + _track(board, severity=Severity.MAJOR_OUTAGE) + _track(board, severity=Severity.PARTIAL_OUTAGE) + _track(board, severity=Severity.OPERATIONAL) + response = client.get( + reverse("board-components", kwargs={"uuid": board.id}), + {"status__severity__in": "0,1,2"}, ) - assert len(client.get(url).json()["results"]) == 1 + assert response.status_code == 200 + severities = {r["status"]["severity"] for r in response.json()["results"]} + assert severities == {Severity.MAJOR_OUTAGE, Severity.PARTIAL_OUTAGE} + + +@pytest.mark.django_db +def test_the_event_parameter_is_gone(client, board): + # It named the Home Incidents and Maintenance tabs. Home is Board + # and Updates now, and Updates is `/events/?dashboard=`. + with_incident = _track(board) + _event(with_incident, EventKind.INCIDENT, IncidentPhase.INVESTIGATING) + _track(board) + body = client.get( + reverse("board-components", kwargs={"uuid": board.id}), {"event": "incident"} + ).json() + # django-filter ignores an unknown parameter rather than failing, + # so the proof is that it no longer narrows anything. + assert body["aggregates"]["total"] == 2 + assert "by_event_kind" not in body["aggregates"] @pytest.mark.django_db @@ -179,7 +150,21 @@ def test_tracking_a_component_adds_it_and_bumps_the_watcher_count(client, board) response = client.post(url, {"component_id": str(component.id)}, format="json") assert response.status_code == 201 assert DashboardItem.objects.filter(dashboard=board, component=component).exists() - assert watchers(component.service) == 1 + assert watchers(component) == 1 + + +@pytest.mark.django_db +def test_the_tracked_component_comes_back_prepared(client, board): + # The write returns a component row. Its counts are annotations, so + # a fetch that skips `for_display` cannot fill them. + parent = ComponentFactory(service=ServiceFactory()) + ComponentFactory(service=parent.service, parent=parent) + url = reverse("board-components", kwargs={"uuid": board.id}) + body = client.post(url, {"component_id": str(parent.id)}, format="json").json() + assert body["descendant_count"] == 1 + # Read after the write. The annotation is false until the item is + # there, so the order of the two matters. + assert body["is_tracked"] is True @pytest.mark.django_db @@ -192,6 +177,18 @@ def test_tracking_the_same_component_twice_is_not_an_error(client, board): assert DashboardItem.objects.filter(dashboard=board).count() == 1 +@pytest.mark.django_db +def test_tracking_an_archived_component_is_refused(client, board): + # `Service.overall_component` still hands a client an archived + # rollup, so this needs no stale id. Accepting it made a board row + # that no list renders and nothing can offer to untrack. + component = ComponentFactory(service=ServiceFactory(), is_archived=True) + url = reverse("board-components", kwargs={"uuid": board.id}) + response = client.post(url, {"component_id": str(component.id)}, format="json") + assert response.status_code == 404 + assert not DashboardItem.objects.filter(dashboard=board).exists() + + @pytest.mark.django_db def test_stopping_tracking_removes_the_item(client, board): component = _track(board) @@ -212,19 +209,19 @@ def test_a_board_belonging_to_someone_else_is_not_readable(client): @pytest.mark.django_db def test_watcher_count_is_distinct_users_not_tracked_items(client, board): - # Someone tracking five Twilio components is one watcher. Counting - # items would let one person outrank a crowd in the suggestion order. - service = ServiceFactory() - StatusPageFactory(service=service) - url = reverse("board-components", kwargs={"uuid": board.id}) - for external_id in ("a", "b", "c"): - component = ComponentFactory(service=service, external_id=external_id) + # One person tracking a component from two boards is one watcher. + # Counting items would let one person outrank a crowd in the + # suggestion order. + component = ComponentFactory() + second_board = Dashboard.objects.create(owner=board.owner, name="Second") + for target in (board, second_board): + url = reverse("board-components", kwargs={"uuid": target.id}) client.post(url, {"component_id": str(component.id)}, format="json") - assert watchers(service) == 1 + assert watchers(component) == 1 @pytest.mark.django_db -def test_two_people_tracking_one_service_count_twice(client, board): +def test_two_people_tracking_one_component_count_twice(client, board): service = ServiceFactory() StatusPageFactory(service=service) component = ComponentFactory(service=service) @@ -233,21 +230,20 @@ def test_two_people_tracking_one_service_count_twice(client, board): other = Dashboard.objects.get(owner=User.objects.create(email="second@b.com")) DashboardItem.objects.create(dashboard=other, component=component) - assert watchers(service) == 2 + assert watchers(component) == 2 @pytest.mark.django_db def test_untracking_the_last_component_drops_the_watcher(client, board): component = _track(board) - service = component.service - assert watchers(service) == 1 + assert watchers(component) == 1 client.delete( reverse( "board-component-detail", kwargs={"uuid": board.id, "component_id": component.id}, ) ) - assert watchers(service) == 0 + assert watchers(component) == 0 @pytest.mark.django_db @@ -271,7 +267,7 @@ def test_the_watcher_count_is_right_however_a_row_arrives_or_goes(how): user = get_user_model().objects.create(email="watcher@b.com") board = Dashboard.objects.create(owner=user, name="B") DashboardItem.objects.create(dashboard=board, component=component) - assert watchers(service) == 1 + assert watchers(component) == 1 if how == "the admin": DashboardItem.objects.get(dashboard=board, component=component).delete() @@ -283,4 +279,4 @@ def test_the_watcher_count_is_right_however_a_row_arrives_or_goes(how): else: DashboardItem.objects.all().delete() - assert watchers(service) == 0 + assert watchers(component) == 0 diff --git a/api/tests/test_catalog_api.py b/api/tests/test_catalog_api.py index 88224bf..eae9d4e 100644 --- a/api/tests/test_catalog_api.py +++ b/api/tests/test_catalog_api.py @@ -1,5 +1,5 @@ import pytest -from django.urls import reverse +from django.urls import Resolver404, resolve, reverse from django.utils import timezone from rest_framework.test import APIClient @@ -11,7 +11,20 @@ StatusSource, ) from status.models import ComponentStatus, ServiceEvent -from tests.factories import ComponentFactory, ServiceFactory, StatusPageFactory +from tests.factories import ( + ComponentFactory, + ServiceFactory, + StatusPageFactory, + track, +) + + +def resolve_or_none(path): + """The view a path reaches, or None when nothing serves it.""" + try: + return resolve(path) + except Resolver404: + return None def _with_status(service, severity, is_overall=False, external_id=None): @@ -29,14 +42,48 @@ def _with_status(service, severity, is_overall=False, external_id=None): return component +def _service_detail(service): + return APIClient().get(reverse("service-detail", args=[service.slug])).json() + + +def test_the_service_list_is_gone(): + # Discover searches components, and the signed-out board lists + # overall components. Nothing asked a service list a question. + assert resolve_or_none("/catalog/services/") is None + + +@pytest.mark.django_db +def test_the_nested_component_route_is_gone(): + # `/catalog/components/?service=` serves it, and three other + # screens besides. + service = ServiceFactory() + response = APIClient().get(f"/catalog/services/{service.slug}/components/") + assert response.status_code == 404 + + +@pytest.mark.django_db +def test_the_nested_event_route_is_gone(): + # `/events/?service=` serves it, beside every other event list. + service = ServiceFactory() + response = APIClient().get(f"/catalog/services/{service.slug}/events/") + assert response.status_code == 404 + + @pytest.mark.django_db -def test_the_list_is_public(): - assert APIClient().get(reverse("service-list")).status_code == 200 +def test_a_service_still_answers_by_slug(): + # The service page reads it for the header and the About tab. + service = ServiceFactory(name="Twilio") + StatusPageFactory(service=service) + response = APIClient().get(reverse("service-detail", args=[service.slug])) + assert response.status_code == 200 + assert response.json()["name"] == "Twilio" + # Nothing renders a description, so it is not in the shape. + assert "description" not in response.json() @pytest.mark.django_db def test_every_list_response_uses_the_same_envelope(): - body = APIClient().get(reverse("service-list")).json() + body = APIClient().get(reverse("component-list")).json() assert set(body) == {"aggregates", "next", "results"} assert "total" in body["aggregates"] @@ -46,7 +93,7 @@ def test_a_service_nests_its_overall_component_rather_than_copying_its_fields(): service = ServiceFactory(slug="twilio") StatusPageFactory(service=service) _with_status(service, Severity.MAJOR_OUTAGE, is_overall=True) - body = APIClient().get(reverse("service-list")).json()["results"][0] + body = _service_detail(service) assert body["overall_component"]["status"]["severity"] == Severity.MAJOR_OUTAGE assert "severity" not in body @@ -57,17 +104,13 @@ def test_the_overall_component_is_a_plain_component(): service = ServiceFactory() StatusPageFactory(service=service) _with_status(service, Severity.OPERATIONAL, is_overall=True) - overall = ( - APIClient() - .get(reverse("service-list")) - .json()["results"][0]["overall_component"] - ) + overall = _service_detail(service)["overall_component"] for field in ( "id", "name", "path", "parent", - "child_count", + "descendant_count", "is_overall", "archived_at", "status", @@ -81,15 +124,38 @@ def test_the_overall_component_is_a_plain_component(): assert field in overall, f"{field} missing from the overall component" +@pytest.mark.django_db +def test_a_service_whose_rollup_is_archived_still_has_a_header_status(): + # The one documented exception to the archived rule. Moving + # `visible` into `for_display` is the obvious simplification, and it + # would leave the service page with no status at all. + from catalog.models import Service + + service = ServiceFactory() + StatusPageFactory(service=service) + rollup = _with_status(service, Severity.MAJOR_OUTAGE, is_overall=True) + rollup.is_archived = True + rollup.save(update_fields=["is_archived"]) + + body = _service_detail(service) + assert body["overall_component"]["id"] == str(rollup.id) + assert body["overall_component"]["archived_at"] is not None + assert Service.objects.get(pk=service.pk).overall_component() == rollup + + @pytest.mark.django_db def test_severity_filters_use_the_orm_path(): + # The contract's name is not a column. It points at an annotation, + # and a filter that missed it would answer every row. service = ServiceFactory() StatusPageFactory(service=service) _with_status(service, Severity.MAJOR_OUTAGE, is_overall=True) - url = reverse("service-list") + "?overall_component__status__severity__lte=3" - assert len(APIClient().get(url).json()["results"]) == 1 - url = reverse("service-list") + "?overall_component__status__severity__lte=0" - assert len(APIClient().get(url).json()["results"]) == 1 + _with_status(service, Severity.OPERATIONAL) + url = reverse("component-list") + assert len(APIClient().get(url).json()["results"]) == 2 + assert ( + len(APIClient().get(url, {"status__severity__lte": 3}).json()["results"]) == 1 + ) @pytest.mark.django_db @@ -97,11 +163,8 @@ def test_fields_prunes_the_response(): service = ServiceFactory() StatusPageFactory(service=service) _with_status(service, Severity.OPERATIONAL, is_overall=True) - body = ( - APIClient() - .get(reverse("service-list") + "?fields=id,name") - .json()["results"][0] - ) + url = reverse("service-detail", args=[service.slug]) + body = APIClient().get(url, {"fields": "id,name"}).json() assert set(body) == {"id", "name"} @@ -110,8 +173,10 @@ def test_a_dotted_field_path_prunes_inside_a_nested_object(): service = ServiceFactory() StatusPageFactory(service=service) _with_status(service, Severity.OPERATIONAL, is_overall=True) - url = reverse("service-list") + "?fields=id,overall_component.status.severity" - body = APIClient().get(url).json()["results"][0] + url = reverse("service-detail", args=[service.slug]) + body = ( + APIClient().get(url, {"fields": "id,overall_component.status.severity"}).json() + ) assert set(body) == {"id", "overall_component"} assert set(body["overall_component"]) == {"status"} assert set(body["overall_component"]["status"]) == {"severity"} @@ -133,11 +198,7 @@ def test_active_incident_excludes_resolved_ones(): ends_at=timezone.now(), ) resolved.affected_components.add(component) - overall = ( - APIClient() - .get(reverse("service-list")) - .json()["results"][0]["overall_component"] - ) + overall = _service_detail(service)["overall_component"] assert overall["active_incident"] is None assert overall["active_incident_count"] == 0 @@ -157,127 +218,89 @@ def test_upcoming_maintenance_excludes_finished_windows(): ends_at=timezone.now() - timezone.timedelta(days=29), ) past.affected_components.add(component) - overall = ( - APIClient() - .get(reverse("service-list")) - .json()["results"][0]["overall_component"] - ) + overall = _service_detail(service)["overall_component"] assert overall["upcoming_maintenance"] is None assert overall["upcoming_maintenance_count"] == 0 -@pytest.mark.django_db -def test_the_events_endpoint_returns_both_kinds_and_filters_on_kind(): - service = ServiceFactory(slug="twilio") - StatusPageFactory(service=service) - ServiceEvent.objects.create( - service=service, - external_id="1", - kind=EventKind.INCIDENT, - title="I", - phase=IncidentPhase.INVESTIGATING, - starts_at=timezone.now(), - ) - ServiceEvent.objects.create( - service=service, - external_id="2", - kind=EventKind.MAINTENANCE, - title="M", - phase=MaintenancePhase.SCHEDULED, - starts_at=timezone.now(), - ) - url = reverse("service-events", kwargs={"slug": "twilio"}) - assert len(APIClient().get(url).json()["results"]) == 2 - assert len(APIClient().get(url + "?kind=incident").json()["results"]) == 1 - - -@pytest.mark.django_db -def test_the_events_aggregate_is_by_phase_only(): - service = ServiceFactory(slug="twilio") - StatusPageFactory(service=service) - url = reverse("service-events", kwargs={"slug": "twilio"}) - aggregates = APIClient().get(url).json()["aggregates"] - assert set(aggregates) == {"total", "by_phase"} - - -@pytest.mark.django_db -def test_an_event_carries_no_component_ids(): - # The projection runs one way: a component carries its event, not the reverse. - service = ServiceFactory(slug="twilio") - StatusPageFactory(service=service) - ServiceEvent.objects.create( - service=service, - external_id="1", - kind=EventKind.INCIDENT, - title="I", - phase=IncidentPhase.INVESTIGATING, - starts_at=timezone.now(), - ) - url = reverse("service-events", kwargs={"slug": "twilio"}) - row = APIClient().get(url).json()["results"][0] - assert set(row) == { - "id", - "kind", - "title", - "phase", - "starts_at", - "ends_at", - "updates", - } - - @pytest.mark.django_db def test_suggestions_put_what_is_broken_ahead_of_what_is_popular(): """Severity sits ahead of popularity on purpose. - A middling service that is broken now beats a popular one that is + A middling component that is broken now beats a popular one that is fine. That is the premise of the public view. """ - popular = ServiceFactory(name="Popular", tracked=500) + popular = ServiceFactory(name="Popular") StatusPageFactory(service=popular) - _with_status(popular, Severity.OPERATIONAL, is_overall=True) + watched = _with_status(popular, Severity.OPERATIONAL, is_overall=True) + for _ in range(5): + track(watched) - broken = ServiceFactory(name="Broken", tracked=1) + broken = ServiceFactory(name="Broken") StatusPageFactory(service=broken) - _with_status(broken, Severity.MAJOR_OUTAGE, is_overall=True) + track(_with_status(broken, Severity.MAJOR_OUTAGE, is_overall=True)) names = [ - r["name"] for r in APIClient().get(reverse("service-list")).json()["results"] + r["service"]["name"] + for r in APIClient().get(reverse("component-list")).json()["results"] ] assert names.index("Broken") < names.index("Popular") @pytest.mark.django_db -def test_a_featured_service_still_leads(): - # Featured is the cold-start seed. On day one nothing has watchers - # and nothing is polled, so it is the whole list. +def test_a_featured_component_still_leads(): + # Featured is the first key, ahead of severity. An editor's pick + # leads even when something worse is on the same page. plain = ServiceFactory(name="Plain") StatusPageFactory(service=plain) _with_status(plain, Severity.MAJOR_OUTAGE, is_overall=True) - featured = ServiceFactory(name="Featured", is_featured=True) + featured = ServiceFactory(name="Featured") StatusPageFactory(service=featured) - _with_status(featured, Severity.OPERATIONAL, is_overall=True) + row = _with_status(featured, Severity.OPERATIONAL, is_overall=True) + row.is_featured = True + row.save(update_fields=["is_featured"]) names = [ - r["name"] for r in APIClient().get(reverse("service-list")).json()["results"] + r["service"]["name"] + for r in APIClient().get(reverse("component-list")).json()["results"] ] assert names.index("Featured") < names.index("Plain") @pytest.mark.django_db -def test_a_service_with_no_reading_sorts_last_not_first(): +def test_a_featured_leaf_leads_as_well_as_a_featured_rollup(): + # Discover lists every component, and the sort reads the flag on + # every row. Featuring one part of a service surfaces that part. + service = ServiceFactory(name="Twilio") + StatusPageFactory(service=service) + _with_status(service, Severity.MAJOR_OUTAGE, is_overall=True, external_id="roll") + leaf = _with_status(service, Severity.OPERATIONAL, external_id="leaf") + leaf.name = "SMS" + leaf.is_featured = True + leaf.save(update_fields=["name", "is_featured"]) + + names = [ + r["name"] for r in APIClient().get(reverse("component-list")).json()["results"] + ] + assert names[0] == "SMS" + + +@pytest.mark.django_db +def test_a_component_with_no_reading_sorts_last_not_first(): # Never seen is not the same as healthy, and it must not pose as # broken either. unread = ServiceFactory(name="Unread") StatusPageFactory(service=unread) + ComponentFactory(service=unread, is_overall=True) healthy = ServiceFactory(name="Healthy") StatusPageFactory(service=healthy) _with_status(healthy, Severity.OPERATIONAL, is_overall=True) names = [ - r["name"] for r in APIClient().get(reverse("service-list")).json()["results"] + r["service"]["name"] + for r in APIClient().get(reverse("component-list")).json()["results"] ] assert names.index("Healthy") < names.index("Unread") @@ -286,23 +309,57 @@ def test_a_service_with_no_reading_sorts_last_not_first(): def test_a_path_carries_enough_to_link_to_each_step(): # A joined string named the ancestors and gave a client nothing to # click. Each step is a row with its own id. + from catalog.models import ServiceComponent from catalog.serializers import ComponentSerializer service = ServiceFactory(name="Twilio") overall = ComponentFactory(service=service, name="Twilio", is_overall=True) - parent = ComponentFactory( - service=service, name="Programmable Messaging", parent=overall - ) - child = ComponentFactory(service=service, name="SMS", parent=parent) + group = ComponentFactory(service=service, name="Programmable Messaging") + sms = ComponentFactory(service=service, name="SMS", parent=group) + ComponentFactory(service=service, name="Delivery", parent=sms) - path = ComponentSerializer(child).data["path"] + prepared = ServiceComponent.objects.for_display() + path = ComponentSerializer(prepared.get(name="Delivery")).data["path"] - assert [step["name"] for step in path] == ["Twilio", "Programmable Messaging"] - assert path[0]["id"] == str(overall.id) - assert path[0]["is_overall"] is True + assert [step["name"] for step in path] == ["Programmable Messaging", "SMS"] + assert path[0]["id"] == str(group.id) + # The rollup is never a parent, so a real path step never carries it. + assert path[0]["is_overall"] is False # Empty, not null: the overall component is under nothing, and a # client maps over the list without checking first. - assert ComponentSerializer(overall).data["path"] == [] + assert ComponentSerializer(prepared.get(pk=overall.pk)).data["path"] == [] + + +@pytest.mark.django_db +def test_fields_prunes_inside_a_path_step(): + # `path` is a method field, so `_prune` cannot reach the step + # serializer. A dotted path into it was ignored, and the contract + # promises a dotted path prunes wherever it is accepted. + service = ServiceFactory(name="Twilio") + ComponentFactory(service=service, name="Twilio", is_overall=True) + parent = ComponentFactory(service=service, name="Programmable Messaging") + child = ComponentFactory(service=service, name="SMS", parent=parent) + + url = reverse("component-detail", args=[child.id]) + body = APIClient().get(url, {"fields": "path.name"}).json() + + assert body["path"] == [{"name": "Programmable Messaging"}] + + +@pytest.mark.django_db +def test_fields_refuses_an_unknown_name_under_a_path_step(): + # Nothing pruned and nothing refused. A typo read as a step that + # holds every field, which is the opposite of what was asked for. + service = ServiceFactory(name="Twilio") + ComponentFactory(service=service, name="Twilio", is_overall=True) + parent = ComponentFactory(service=service, name="Programmable Messaging") + child = ComponentFactory(service=service, name="SMS", parent=parent) + + url = reverse("component-detail", args=[child.id]) + response = APIClient().get(url, {"fields": "path.nonsense"}) + + assert response.status_code == 400 + assert response.json() == {"fields": ["Unknown field: nonsense."]} @pytest.mark.django_db @@ -353,31 +410,9 @@ def test_a_component_list_costs_the_same_whatever_its_length(): for size in (2, 30): reset_queries() APIClient().get( - reverse("service-components", kwargs={"slug": service.slug}), - {"page_size": size}, + reverse("component-list"), + {"service": service.slug, "page_size": size}, ) counts[size] = len(connection.queries) assert counts[2] == counts[30], counts - - -@pytest.mark.django_db -def test_a_service_list_costs_the_same_whatever_its_length(): - # Three fields asked per service, and one of them serialized a - # component, which asked seven more. - from django.db import connection, reset_queries - from django.test import override_settings - - for n in range(12): - service = ServiceFactory(name=f"Service {n}") - StatusPageFactory(service=service) - _with_status(service, Severity.OPERATIONAL, is_overall=True) - - counts = {} - with override_settings(DEBUG=True): - for size in (2, 12): - reset_queries() - APIClient().get(reverse("service-list"), {"page_size": size}) - counts[size] = len(connection.queries) - - assert counts[2] == counts[12], counts diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index de4fb41..f8c3d83 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -2,7 +2,31 @@ from django.db.utils import IntegrityError from catalog.models import Service, ServiceComponent, StatusPage -from tests.factories import ComponentFactory, PollerFactory, ServiceFactory +from tests.factories import ( + ComponentFactory, + PollerFactory, + ServiceFactory, + UserFactory, + ancestry, + descendants, + track, + watchers, +) + + +def test_a_components_watcher_count_is_distinct_users(db): + # Two boards holding one component is two watchers. One person + # holding it twice is impossible, and one holding a sibling too is + # still one watcher of this component. + component = ComponentFactory() + sibling = ComponentFactory(service=component.service) + first = UserFactory() + track(component, user=first) + track(sibling, user=first) + track(component, user=UserFactory()) + + assert watchers(component) == 2 + assert watchers(sibling) == 1 @pytest.mark.django_db @@ -54,7 +78,6 @@ def test_a_component_can_be_archived_rather_than_deleted(): # Someone may track this component. # Deletion removes it from their board with no warning. component = ComponentFactory() - assert Service.objects.count() == 1 component.is_archived = True component.save() component.refresh_from_db() @@ -97,7 +120,7 @@ def test_a_name_with_no_slug_characters_still_gets_one(): @pytest.mark.django_db def test_archiving_sets_the_date(): - # `updated_at` cannot answer when a provider dropped a component: it + # `updated_at` cannot answer when a provider dropped a component. It # moves on every save, so a later rename would erase it. component = ComponentFactory() assert component.archived_at is None @@ -134,3 +157,247 @@ def test_the_flag_and_the_date_cannot_disagree(): component = ComponentFactory() with pytest.raises(IntegrityError), transaction.atomic(): ServiceComponent.objects.filter(pk=component.pk).update(is_archived=True) + + +@pytest.mark.django_db +def test_the_database_refuses_a_component_parented_under_the_rollup(): + """A bulk write skips `clean`. Only the trigger is left to refuse it. + + `ComponentFactory` calls `objects.create`, never `full_clean`. A + poll or a shell session meets this, never a form's `clean` error. + """ + service = ServiceFactory() + rollup = ComponentFactory(service=service, is_overall=True) + with pytest.raises(IntegrityError): + ComponentFactory(service=service, parent=rollup) + + +@pytest.mark.django_db +def test_the_database_refuses_a_rollup_that_already_has_children(): + """The same rule, met from the other side. + + A component can gain the rollup flag after it already has + children, since `is_overall` is editable. The trigger checks both + directions, or promoting an existing parent would corrupt every + descendant count under it. + """ + from django.db import IntegrityError, transaction + + service = ServiceFactory() + parent = ComponentFactory(service=service) + ComponentFactory(service=service, parent=parent) + + parent.is_overall = True + with pytest.raises(IntegrityError), transaction.atomic(): + parent.save() + + +def test_a_parent_in_another_service_is_not_a_step_in_the_path(db): + # The walk stops at the service boundary. A breadcrumb that went + # past it names a component of another service. The same response + # then says the row has no ancestors. + old = ServiceFactory(name="Twilio") + top = ComponentFactory(service=old, name="Programmable Messaging") + leaf = ComponentFactory(service=old, name="SMS", parent=top) + + top.service = ServiceFactory(name="Vonage") + top.save() + + leaf.refresh_from_db() + assert leaf.ancestors == [] + assert leaf.visible_ancestors == [] + assert leaf.path == "Twilio / SMS" + + +def test_the_tree_reads_every_step_not_one_level(db): + # A component's Components tab lists everything under it, and its + # breadcrumb names every step above it. One level either way would + # hide the grandchild and shorten the path. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + leaf = ComponentFactory(service=service, parent=middle) + + # Root first, which is the order a breadcrumb reads. + assert ancestry(leaf) == [top.id, middle.id] + assert descendants(top) == {middle.id, leaf.id} + # Nothing counts a component among the rows below it. + assert descendants(leaf) == set() + + +def test_the_tree_stops_at_the_service_boundary(db): + # Moving a component to another service leaves its old children + # pointing across. Counting them under the mover names rows that + # service's lists never hold. + old = ServiceFactory() + top = ComponentFactory(service=old) + leaf = ComponentFactory(service=old, parent=top) + + top.service = ServiceFactory() + top.save() + + assert descendants(top) == set() + leaf.refresh_from_db() + assert ancestry(leaf) == [] + + +def test_a_loop_in_the_parent_column_does_not_hang_the_tree(db): + # The column points at its own table, so bad data can make a cycle. + # Unguarded, the walk down never ends and the request never answers. + service = ServiceFactory() + first = ComponentFactory(service=service) + second = ComponentFactory(service=service, parent=first) + # `update` skips `clean`, which is the only thing refusing this. + ServiceComponent.objects.filter(pk=first.pk).update(parent=second) + + assert descendants(first) == {second.id} + first.refresh_from_db() + second.refresh_from_db() + assert ancestry(first) == [second.id] + assert ancestry(second) == [first.id] + + +def test_deleting_a_component_takes_its_ancestry_with_it(db): + # An array of ids held no reference, and a closure table needed a + # rebuild hook on four write paths. A computed answer passes this + # trivially, which is why the table went. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + leaf = ComponentFactory(service=service, parent=middle) + + middle.delete() + + # `parent` is SET_NULL, so the leaf is now a root. Nothing may + # still claim it sits under the grandparent. + leaf.refresh_from_db() + assert ancestry(leaf) == [] + assert descendants(top) == set() + + +def test_for_display_counts_descendants_without_a_query_a_row( + db, django_assert_num_queries +): + # The count is why `for_display` exists. Asked per row, a page of + # fifty components costs fifty extra queries. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + ComponentFactory(service=service, parent=middle) + + rows = list(ServiceComponent.objects.for_display()) + with django_assert_num_queries(0): + counts = sorted(row.descendant_count for row in rows) + + # Two below the top, one below the middle, none below the leaf. + assert counts == [0, 1, 2] + + +def test_the_descendant_count_leaves_out_an_archived_row(db): + # The badge and the list below it read one number. If this counted + # an archived row, the tab would say 2 and then show one component. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + leaf = ComponentFactory(service=service, parent=middle) + leaf.is_archived = True + leaf.save(update_fields=["is_archived"]) + + row = ServiceComponent.objects.for_display().get(pk=top.pk) + assert row.descendant_count == 1 + + +def test_the_descendant_count_walks_through_an_archived_row(db): + # An archived group is not counted, and it does not hide what sits + # under it either. A walk that stopped there would drop live rows + # the Components tab still lists. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + ComponentFactory(service=service, parent=middle) + middle.is_archived = True + middle.save(update_fields=["is_archived"]) + + row = ServiceComponent.objects.for_display().get(pk=top.pk) + assert row.descendant_count == 1 + + +def test_a_component_read_without_for_display_has_no_count(db): + # The count is an annotation and nothing falls back to a query. A + # serving path that forgets `for_display` has to fail loudly, not + # ask once a row. + top = ComponentFactory() + ComponentFactory(service=top.service, parent=top) + + row = ServiceComponent.objects.get(pk=top.pk) + with pytest.raises(AttributeError): + assert row.descendant_count + + +def test_a_services_component_count_leaves_out_an_archived_row(db): + # This is the badge on the service page. The Components tab under it + # lists the same rows, and the two cannot count differently. + service = ServiceFactory() + ComponentFactory(service=service) + gone = ComponentFactory(service=service) + gone.is_archived = True + gone.save() + + assert Service.objects.for_display().get(pk=service.pk).component_count == 1 + + +def test_a_services_tracked_count_leaves_out_an_archived_row(db): + # The client reads a count above zero as "this service is on your + # board". The board list leaves an archived row out. A count that + # held one promised a row the board never returns. + user = UserFactory() + service = ServiceFactory() + gone = ComponentFactory(service=service) + track(gone, user=user) + gone.is_archived = True + gone.save() + + prepared = Service.objects.for_display(user).get(pk=service.pk) + assert prepared.tracked_component_count == 0 + + +def test_a_services_tracked_count_counts_a_component_once_per_owner(db): + # A person may own several boards. The count says how many of this + # service's components are on them. Counting the join rows makes one + # component held twice read as two, above a tab listing one. + from dashboards.models import Dashboard, DashboardItem + + user = UserFactory() + service = ServiceFactory() + component = ComponentFactory(service=service) + track(component, user=user) + second = Dashboard.objects.create(owner=user) + DashboardItem.objects.create(dashboard=second, component=component) + + prepared = Service.objects.for_display(user).get(pk=service.pk) + assert prepared.tracked_component_count == 1 + + +def test_search_reaches_a_component_through_its_services_name(db): + # "twilio" has to find SMS. Its own name says nothing about which + # service publishes it. A name-only match returns an empty page for + # the word people actually type. + service = ServiceFactory(name="Twilio") + rollup = ComponentFactory(service=service, name="Twilio", is_overall=True) + parent = ComponentFactory(service=service, name="Programmable Messaging") + leaf = ComponentFactory(service=service, name="SMS", parent=parent) + + assert set(ServiceComponent.objects.search("twilio")) == {rollup, parent, leaf} + + # OR semantics would let the rollup match too: it says "twilio" but + # never "sms". + assert list(ServiceComponent.objects.search("twilio sms")) == [leaf] + + +def test_search_ignores_the_case_and_matches_part_of_a_word(db): + # People type a fragment in lower case. An exact match would answer + # nothing until the whole name was typed the provider's way. + service = ServiceFactory(name="Twilio") + leaf = ComponentFactory(service=service, name="Programmable Messaging") + + assert list(ServiceComponent.objects.search("messag")) == [leaf] diff --git a/api/tests/test_common_api.py b/api/tests/test_common_api.py index 9859eff..80717f4 100644 --- a/api/tests/test_common_api.py +++ b/api/tests/test_common_api.py @@ -39,9 +39,12 @@ def test_a_dotted_path_prunes_inside_a_nested_object_rather_than_dropping_it(): assert set(result["status"]) == {"severity"} -def test_an_unknown_field_is_ignored_rather_than_erroring(): - # A client asking for a field that does not exist yet gets what does exist. - assert set(_serialize("id,nonsense")) == {"id"} +def test_an_unknown_field_is_refused(): + # Ignoring it popped every field and returned an empty object. A + # client then read a typo as a server holding nothing. + with pytest.raises(serializers.ValidationError) as raised: + _serialize("id,nonsense") + assert raised.value.detail == {"fields": ["Unknown field: nonsense."]} def test_a_serializer_can_opt_out(): @@ -82,5 +85,5 @@ def test_a_view_that_declares_no_permissions_requires_authentication(): def test_the_catalog_stays_public_on_purpose(client): # Browsing without an account is in the spec, so these say AllowAny # rather than inheriting it. - assert client.get(reverse("service-list")).status_code == 200 + assert client.get(reverse("component-list")).status_code == 200 assert client.get(reverse("meta")).status_code == 200 diff --git a/api/tests/test_common_models.py b/api/tests/test_common_models.py index e1dfbfe..405bfbf 100644 --- a/api/tests/test_common_models.py +++ b/api/tests/test_common_models.py @@ -1,5 +1,3 @@ -import uuid - import pytest from django.db import models @@ -11,27 +9,19 @@ def test_base_model_is_abstract(): def test_base_model_orders_newest_first(): - assert BaseModel._meta.ordering == ["-created_at"] + assert list(BaseModel._meta.ordering) == ["-created_at"] def test_primary_key_is_a_uuid_not_an_integer(): + # Version 7 leads with a millisecond timestamp, so a key lands at + # the end of the index. Nothing needs a second column to sort by. pk = BaseModel._meta.get_field("id") assert isinstance(pk, models.UUIDField) assert pk.primary_key is True - assert pk.default is uuid.uuid7 - - -def test_keys_sort_by_the_order_they_were_made(): - # Version 7 leads with a millisecond timestamp. A key lands at the - # end of the index, not in the middle. - keys = [uuid.uuid7() for _ in range(50)] - assert all(key.version == 7 for key in keys) - assert keys == sorted(keys) + assert pk.default().version == 7 def test_audit_columns_exist_and_are_optional(): - for name in ("created_at", "updated_at"): - assert BaseModel._meta.get_field(name) is not None for name in ("created_by", "updated_by"): field = BaseModel._meta.get_field(name) assert field.null is True, ( @@ -53,3 +43,30 @@ def test_the_system_account_exists_before_anything_writes(): assert account.is_active is False # It signed itself, so no row the system wrote is left unsigned. assert account.created_by_id == account.pk + + +def test_a_column_with_two_defaults_declares_one_value(): + """`default` and `db_default` do different jobs, on the same column. + + Without `default` an unsaved instance holds a sentinel rather than + the value. Without `db_default` the column carries no default, and + a writer that predates it fails the insert. + + A field needs both. If the two ever disagree, a row Python makes + and a row SQL makes differ on one column. Nothing would say so. + """ + from django.apps import apps + from django.db.models import NOT_PROVIDED + + both = { + f"{model._meta.label}.{field.name}": (field.default, field.db_default) + for model in apps.get_models() + for field in model._meta.concrete_fields + if field.default is not NOT_PROVIDED and field.db_default is not NOT_PROVIDED + } + # Nothing declaring both would make the check below vacuous. + assert both + disagreeing = { + name: values for name, values in both.items() if values[0] != values[1] + } + assert not disagreeing, f"two defaults, two values: {disagreeing}" diff --git a/api/tests/test_component_api.py b/api/tests/test_component_api.py new file mode 100644 index 0000000..683ced8 --- /dev/null +++ b/api/tests/test_component_api.py @@ -0,0 +1,249 @@ +import pytest +from django.urls import reverse +from django.utils import timezone +from rest_framework.test import APIClient + +from status.choices import EventKind, IncidentPhase, Severity, StatusSource +from status.models import ComponentStatus, ServiceEvent +from tests.factories import ComponentFactory, ServiceFactory, track + +pytestmark = pytest.mark.django_db + + +def _reading(component, severity): + """The open status row, which is what `severity_now` reads.""" + return ComponentStatus.objects.create( + component=component, + severity=severity, + source=StatusSource.PROVIDER, + started_at=timezone.now(), + ) + + +@pytest.fixture +def tree(): + service = ServiceFactory(name="Twilio") + rollup = ComponentFactory( + service=service, name="Twilio", is_overall=True, status_page_order=0 + ) + parent = ComponentFactory( + service=service, name="Programmable Messaging", status_page_order=2 + ) + leaf = ComponentFactory( + service=service, name="SMS", parent=parent, status_page_order=1 + ) + return service, rollup, parent, leaf + + +def test_the_collection_lists_every_component(client, tree): + # Discover searches all of them, rollups included. Narrowing here + # would make the signed-out board and Discover one list. + response = client.get(reverse("component-list")) + assert response.status_code == 200 + assert response.json()["aggregates"]["total"] == 3 + + +def test_is_overall_narrows_to_one_row_per_service(client, tree): + # This is the signed-out board: one row per service, not per part. + response = client.get(reverse("component-list"), {"is_overall": "true"}) + assert [r["name"] for r in response.json()["results"]] == ["Twilio"] + + +def test_ancestor_returns_every_descendant_not_one_level(client, tree): + # A component's Components tab is the same screen at a different + # root, so it cannot count differently from a service's. + _, _, parent, leaf = tree + response = client.get(reverse("component-list"), {"ancestor": str(parent.id)}) + assert [r["id"] for r in response.json()["results"]] == [str(leaf.id)] + + +def test_q_reaches_a_component_through_its_services_name(client, tree): + # Searching "twilio" must find SMS. Its own name says nothing + # about which service it belongs to. + response = client.get(reverse("component-list"), {"q": "twilio sms"}) + assert [r["name"] for r in response.json()["results"]] == ["SMS"] + + +def test_an_explicit_ordering_beats_the_default_a_search_picks(client, tree): + # The view sets a default sort when `q` is present. It is a + # default: a caller who names one reads their own order. Losing + # that leaves the sort control on the screen doing nothing. + response = client.get( + reverse("component-list"), {"q": "twilio", "ordering": "name"} + ) + assert [r["name"] for r in response.json()["results"]] == [ + "Programmable Messaging", + "SMS", + "Twilio", + ] + + +def test_q_reaches_a_component_through_its_parents_name(client, tree): + # Searching "messaging" must find SMS as well as the group above + # it. A part is named for what it does, not for what it sits under. + response = client.get(reverse("component-list"), {"q": "messaging"}) + assert {r["name"] for r in response.json()["results"]} == { + "Programmable Messaging", + "SMS", + } + + +def test_q_combines_a_parents_name_with_the_components_own(client, tree): + # Every word must match, and any of the three names may satisfy it. + # "messaging sms" is how a person narrows to one part of a group. + response = client.get(reverse("component-list"), {"q": "messaging sms"}) + assert [r["name"] for r in response.json()["results"]] == ["SMS"] + + +def test_q_does_not_reach_past_one_level(client, tree): + # The honest limit. A grandparent's name is not stored on the row, + # and matching it would need the ancestry back. + service, _, _, leaf = tree + ComponentFactory(service=service, name="Verify", parent=leaf) + response = client.get(reverse("component-list"), {"q": "programmable verify"}) + assert response.json()["results"] == [] + + +def test_service_narrows_to_one_services_parts(client, tree): + # A service's Components tab. The rollup is excluded there, + # because the header already carries the service's status. + service, _, _, _ = tree + response = client.get( + reverse("component-list"), {"service": service.slug, "is_overall": "false"} + ) + assert {r["name"] for r in response.json()["results"]} == { + "Programmable Messaging", + "SMS", + } + + +def test_severity_in_takes_several_values(client, tree): + # The Severity filter offers all six values at once, so a single + # exact match cannot serve it. Undeclared, django-filter drops the + # parameter without a word and the operational row comes back too. + _, rollup, parent, leaf = tree + _reading(rollup, Severity.MAJOR_OUTAGE) + _reading(parent, Severity.DEGRADED) + _reading(leaf, Severity.OPERATIONAL) + response = client.get(reverse("component-list"), {"status__severity__in": "0,1,2"}) + assert {r["name"] for r in response.json()["results"]} == { + "Twilio", + "Programmable Messaging", + } + + +def test_the_detail_answers_by_uuid(client, tree): + _, _, _, leaf = tree + response = client.get(reverse("component-detail", args=[leaf.id])) + assert response.status_code == 200 + assert response.json()["name"] == "SMS" + + +def test_the_collection_leaves_out_an_archived_component(client, tree): + # A provider dropped it, so nothing lists it. The service badge + # counts the same rows, and the two cannot count differently. + _, _, _, leaf = tree + # `update_fields`, because the fixture rebuilt this row's ancestry + # and its search document after the factory made it. + leaf.is_archived = True + leaf.save(update_fields=["is_archived"]) + + body = client.get(reverse("component-list")).json() + assert body["aggregates"]["total"] == 2 + assert "SMS" not in [r["name"] for r in body["results"]] + + +def test_the_detail_refuses_an_archived_component(client, tree): + # Archived components are not served anywhere. A row the list hides + # and the detail answers would be two rules, not one. + _, _, _, leaf = tree + leaf.is_archived = True + leaf.save(update_fields=["is_archived"]) + + assert client.get(reverse("component-detail", args=[leaf.id])).status_code == 404 + + +def test_a_query_leads_with_the_service_rollup(client, tree): + # Typing a service's name matches every part of it. Without the + # rollup first, eighty rows bury the one row that answers the + # question the reader asked. + response = client.get(reverse("component-list"), {"q": "twilio"}) + assert response.json()["results"][0]["name"] == "Twilio" + + +def test_a_services_parts_keep_the_status_pages_own_order(client, tree): + # The service tab shows the components in the order the provider + # lists them. The suggested sort would put them in name order. + service, _, _, _ = tree + response = client.get( + reverse("component-list"), {"service": service.slug, "is_overall": "false"} + ) + assert [r["name"] for r in response.json()["results"]] == [ + "SMS", + "Programmable Messaging", + ] + + +def test_is_tracked_narrows_to_what_the_viewer_watches(tree): + # Showing All, Tracked or Untracked. The flag is per viewer, so the + # filter has to read the annotation and not a column. + _, _, _, leaf = tree + item = track(leaf) + viewer = APIClient() + viewer.force_authenticate(item.dashboard.owner) + response = viewer.get(reverse("component-list"), {"is_tracked": "true"}) + assert [r["name"] for r in response.json()["results"]] == ["SMS"] + + +def test_signed_out_nothing_is_tracked(client, tree): + # Nobody signed out tracks anything, so Tracked is empty for them. + response = client.get(reverse("component-list"), {"is_tracked": "true"}) + assert response.json()["results"] == [] + + +def test_signed_out_everything_is_untracked(client, tree): + # The same fact makes Untracked the whole list. Comparing against + # the annotation instead reads `NOT NULL`, which SQL answers NULL, + # and Untracked renders empty for a signed-out reader. + response = client.get(reverse("component-list"), {"is_tracked": "false"}) + assert {r["name"] for r in response.json()["results"]} == { + "Twilio", + "Programmable Messaging", + "SMS", + } + + +def test_event_narrows_to_the_components_one_event_affects(client, tree): + # An event's affected list. Only the components the provider names + # on that event belong on it. + service, _, parent, _ = tree + event = ServiceEvent.objects.create( + service=service, + external_id="inc-1", + kind=EventKind.INCIDENT, + title="Messages are delayed", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + ) + event.affected_components.add(parent) + response = client.get(reverse("component-list"), {"event": str(event.id)}) + assert [r["name"] for r in response.json()["results"]] == ["Programmable Messaging"] + + +def test_the_path_leaves_out_an_archived_ancestor(client): + # Each node carries an id a client links to, and the detail answers + # 404 for an archived component. A breadcrumb step pointing at one + # is a link to a page nothing serves. + service = ServiceFactory() + top = ComponentFactory(service=service, name="Messaging") + middle = ComponentFactory(service=service, name="Programmable", parent=top) + leaf = ComponentFactory(service=service, name="SMS", parent=middle) + middle.is_archived = True + middle.save(update_fields=["is_archived"]) + + body = client.get(reverse("component-detail", args=[leaf.id])).json() + + assert [node["name"] for node in body["path"]] == ["Messaging"] + # Shorter, not broken. `parent` is the raw column and still names + # the archived row, so the two disagree by design. + assert body["parent"] == str(middle.id) diff --git a/api/tests/test_contract.py b/api/tests/test_contract.py index 52e9a91..6564850 100644 --- a/api/tests/test_contract.py +++ b/api/tests/test_contract.py @@ -3,8 +3,17 @@ import pytest import yaml from django.urls import reverse +from django.utils import timezone from drf_spectacular.drainage import GENERATOR_STATS, reset_generator_stats from drf_spectacular.generators import SchemaGenerator +from rest_framework.test import APIClient +from rest_framework_simplejwt.tokens import RefreshToken + +from authentication.models import User +from dashboards.models import DashboardItem +from status.choices import EventKind, IncidentPhase +from status.models import EventUpdate, ServiceEvent +from tests.factories import ComponentFactory, ServiceFactory, StatusPageFactory CONTRACT = Path(__file__).resolve().parents[2] / "docs" / "api" / "openapi.yaml" @@ -19,12 +28,25 @@ def generated(): return SchemaGenerator().get_schema(request=None, public=True) +VERBS = ("get", "post", "delete", "patch", "put") + + def _paths(schema): return { f"{verb.upper()} {path}" for path, ops in schema["paths"].items() for verb in ops - if verb in ("get", "post", "delete", "patch", "put") + if verb in VERBS + } + + +def _responses(schema): + return { + f"{verb.upper()} {path} -> {code}" + for path, ops in schema["paths"].items() + for verb, operation in ops.items() + if verb in VERBS + for code in operation.get("responses", {}) } @@ -38,6 +60,23 @@ def test_no_operation_exists_that_the_contract_does_not_document(committed, gene assert not extra, f"implemented but undocumented: {sorted(extra)}" +def test_every_documented_response_code_is_declared_in_the_code(committed, generated): + # The board endpoint documented a 409 no code path returned. The + # path check above passes on that, because the path exists. A + # client is generated from this file and branches on the code. + missing = _responses(committed) - _responses(generated) + assert not missing, f"documented but not returned: {sorted(missing)}" + + +def test_no_response_code_is_declared_that_the_contract_does_not_document( + committed, generated +): + # The other direction. A failure a client cannot see is one it + # cannot handle, so it falls through as an unexpected error. + extra = _responses(generated) - _responses(committed) + assert not extra, f"returned but undocumented: {sorted(extra)}" + + def test_every_endpoint_accepts_the_fields_parameter(generated): # ?fields= is baked in at the base layer, so it must appear on every # operation without a per-view annotation. @@ -49,6 +88,150 @@ def test_every_endpoint_accepts_the_fields_parameter(generated): assert "fields" in names, f"{verb.upper()} {path} does not accept ?fields=" +def _authed_client(): + """A client bearing a fresh user's access token.""" + user = User.objects.create( + email=f"fields-probe-{User.objects.count()}@example.test" + ) + api = APIClient() + api.credentials( + HTTP_AUTHORIZATION=f"Bearer {RefreshToken.for_user(user).access_token}" + ) + return api, user + + +def _meta_probe(): + return APIClient(), reverse("meta"), "max_page_size" + + +def _me_probe(): + api, _user = _authed_client() + return api, reverse("me"), "email" + + +def _service_detail_probe(): + service = ServiceFactory() + StatusPageFactory(service=service) + ComponentFactory(service=service, is_overall=True) + return APIClient(), reverse("service-detail", kwargs={"slug": service.slug}), "id" + + +def _event_list_probe(): + service = ServiceFactory() + ServiceEvent.objects.create( + service=service, + external_id="probe", + kind=EventKind.INCIDENT, + title="x", + phase=IncidentPhase.DETECTED, + starts_at=timezone.now(), + ) + return APIClient(), reverse("event-list"), "id" + + +def _event_detail_probe(): + service = ServiceFactory() + event = ServiceEvent.objects.create( + service=service, + external_id="probe", + kind=EventKind.INCIDENT, + title="x", + phase=IncidentPhase.DETECTED, + starts_at=timezone.now(), + ) + url = reverse("event-detail", kwargs={"uuid": event.id}) + return APIClient(), url, "id" + + +def _event_updates_probe(): + service = ServiceFactory() + event = ServiceEvent.objects.create( + service=service, + external_id="probe", + kind=EventKind.INCIDENT, + title="x", + phase=IncidentPhase.DETECTED, + starts_at=timezone.now(), + ) + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.DETECTED, + body="x", + posted_at=timezone.now(), + ) + url = reverse("event-updates", kwargs={"uuid": event.id}) + return APIClient(), url, "phase" + + +def _component_list_probe(): + service = ServiceFactory() + ComponentFactory(service=service, is_overall=True) + return APIClient(), reverse("component-list"), "id" + + +def _component_detail_probe(): + service = ServiceFactory() + component = ComponentFactory(service=service, is_overall=True) + url = reverse("component-detail", kwargs={"uuid": component.id}) + return APIClient(), url, "id" + + +def _board_components_probe(): + api, user = _authed_client() + board = user.default_dashboard + service = ServiceFactory() + component = ComponentFactory(service=service, is_overall=True) + DashboardItem.objects.create(dashboard=board, component=component) + url = reverse("board-components", kwargs={"uuid": board.id}) + return api, url, "id" + + +# One builder per documented GET path: a client, a live URL, and a field +# to keep. +FIELDS_PROBES = { + "/meta/": _meta_probe, + "/me/": _me_probe, + "/catalog/services/{slug}/": _service_detail_probe, + "/catalog/components/": _component_list_probe, + "/catalog/components/{uuid}/": _component_detail_probe, + "/dashboards/{uuid}/components/": _board_components_probe, + "/events/": _event_list_probe, + "/events/{uuid}/": _event_detail_probe, + "/events/{uuid}/updates/": _event_updates_probe, +} + + +def test_the_fields_probes_cover_every_documented_get(committed): + # A GET added later and left out here would ship unpruned and + # nothing would say so, the way /meta/ did. + documented = {path for path, ops in committed["paths"].items() if "get" in ops} + assert documented == set(FIELDS_PROBES) + + +@pytest.mark.django_db +@pytest.mark.parametrize("path", sorted(FIELDS_PROBES)) +def test_fields_prunes_the_response(path): + # Declaring ?fields= on a view is not the same as honouring it. + # /meta/ built its body by hand and never ran it through the + # serializer that prunes, so the parameter did nothing. + client, url, field = FIELDS_PROBES[path]() + body = client.get(url, {"fields": field}).json() + kept = body["results"][0] if isinstance(body, dict) and "results" in body else body + assert set(kept) == {field}, f"GET {path} ignored ?fields=: {sorted(kept)}" + + +@pytest.mark.django_db +@pytest.mark.parametrize("path", sorted(FIELDS_PROBES)) +def test_fields_refuses_a_name_that_is_no_field(path): + # Nothing checked that a name in the tree named a field. An unknown + # one popped them all and answered `200 {}`. The contract promises + # 400. A typo must not read as a server with nothing to say. + client, url, _field = FIELDS_PROBES[path]() + response = client.get(url, {"fields": "total_nonsense"}) + assert response.status_code == 400, f"GET {path} answered {response.status_code}" + assert response.json() == {"fields": ["Unknown field: total_nonsense."]} + + @pytest.mark.django_db def test_the_component_schema_has_exactly_the_documented_fields(committed): from catalog.serializers import ComponentSerializer diff --git a/api/tests/test_cross_service.py b/api/tests/test_cross_service.py index 634f0d0..9d652e9 100644 --- a/api/tests/test_cross_service.py +++ b/api/tests/test_cross_service.py @@ -69,6 +69,21 @@ def test_a_component_may_sit_under_its_own_services_component(two_services): child.full_clean() +@pytest.mark.django_db +def test_a_component_cannot_sit_under_the_rollup(two_services): + # The rollup is a peer of the top-level components, not their + # parent. A database trigger refuses the row; this is what a form + # sees instead of that trigger's error. + mine, _ = two_services + rollup = ComponentFactory(service=mine, is_overall=True) + child = ComponentFactory(service=mine, external_id="child") + child.parent = rollup + + with pytest.raises(ValidationError) as raised: + child.full_clean() + assert "parent" in raised.value.message_dict + + @pytest.mark.django_db def test_a_reading_cannot_cite_a_poll_of_another_service(two_services): mine, theirs = two_services @@ -164,6 +179,7 @@ def test_an_event_keeps_a_component_the_provider_stopped_listing(two_services): ) ], StatusSource.PROVIDER, + _run(mine), ) event = ServiceEvent.objects.get(external_id="inc-1") @@ -190,6 +206,7 @@ def test_a_poll_invents_no_component(two_services): ) ], StatusSource.PROVIDER, + _run(mine), ) assert not ServiceComponent.objects.filter(external_id="never-seen").exists() diff --git a/api/tests/test_dashboards_models.py b/api/tests/test_dashboards_models.py index ef2ddf1..d815815 100644 --- a/api/tests/test_dashboards_models.py +++ b/api/tests/test_dashboards_models.py @@ -60,12 +60,6 @@ def test_a_component_is_tracked_once_per_dashboard(): DashboardItem.objects.create(dashboard=board, component=component) -@pytest.mark.django_db -def test_an_item_carries_no_position(): - # Order is a query, not a column. A stored position needs a rewrite on every insert. - assert not hasattr(DashboardItem, "position") - - @pytest.mark.django_db def test_the_system_account_cannot_sign_in(): system = User.objects.system() @@ -152,3 +146,17 @@ def test_closing_an_account_still_takes_its_boards(): user.delete() assert not Dashboard.objects.filter(owner_id=user.pk).exists() + + +@pytest.mark.django_db +def test_a_factory_user_and_a_tracked_one_never_collide(): + # `track` minted its own address from a counter that resets per + # test. UserFactory's persists across the session, and email is + # unique. The first test to call both would raise far from the cause. + from tests.factories import UserFactory, track + + UserFactory.reset_sequence(0) + UserFactory() + track(ComponentFactory()) + + assert User.objects.filter(email__startswith="watcher").count() == 2 diff --git a/api/tests/test_event_api.py b/api/tests/test_event_api.py new file mode 100644 index 0000000..df937f3 --- /dev/null +++ b/api/tests/test_event_api.py @@ -0,0 +1,313 @@ +import pytest +from django.urls import reverse +from django.utils import timezone + +from authentication.models import User +from status.choices import EventKind, EventSource, IncidentPhase, MaintenancePhase +from status.models import EventUpdate, ServiceEvent +from tests.factories import ComponentFactory, ServiceFactory, track + +pytestmark = pytest.mark.django_db + + +def _event(service, component, **kwargs): + fields = { + "kind": EventKind.INCIDENT, + "title": "Something broke", + "phase": IncidentPhase.INVESTIGATING, + "starts_at": timezone.now(), + } + fields.update(kwargs) + event = ServiceEvent.objects.create(service=service, **fields) + event.affected_components.set([component]) + return event + + +def test_the_feed_lists_events_newest_first(client): + # A feed is read from the top. Oldest first would open on an + # incident from last month. + service = ServiceFactory() + component = ComponentFactory(service=service) + _event(service, component, external_id="1", title="Older") + new = _event(service, component, external_id="2", title="Newer") + ServiceEvent.objects.filter(pk=new.pk).update( + starts_at=timezone.now() + timezone.timedelta(minutes=5) + ) + + results = client.get(reverse("event-list")).json()["results"] + assert [r["title"] for r in results] == ["Newer", "Older"] + + +def test_a_feed_row_names_its_service(client): + # /events/?dashboard= spans every service on a board. A row cannot + # label itself without the service it belongs to. + service = ServiceFactory(slug="pagerduty") + _event(service, ComponentFactory(service=service), external_id="1") + + results = client.get(reverse("event-list")).json()["results"] + assert results[0]["service"]["slug"] == "pagerduty" + + +def test_service_narrows_the_feed(client): + # A service's Updates tab. Anything from another service on it + # would be answering a question nobody asked on that page. + first = ServiceFactory() + second = ServiceFactory() + _event(first, ComponentFactory(service=first), external_id="1", title="Mine") + _event(second, ComponentFactory(service=second), external_id="2", title="Theirs") + + results = client.get(reverse("event-list"), {"service": first.slug}).json()[ + "results" + ] + assert [r["title"] for r in results] == ["Mine"] + + +def test_component_narrows_the_feed(client): + # A component's Updates tab reads the same collection one level + # down, so it cannot be a different endpoint. + service = ServiceFactory() + wanted = ComponentFactory(service=service) + other = ComponentFactory(service=service) + _event(service, wanted, external_id="1", title="Wanted") + _event(service, other, external_id="2", title="Other") + + results = client.get(reverse("event-list"), {"component": str(wanted.id)}).json()[ + "results" + ] + assert [r["title"] for r in results] == ["Wanted"] + + +def test_kind_narrows_the_feed(client): + # Incidents and maintenance are one collection. A screen showing + # only one of them says which. + service = ServiceFactory() + component = ComponentFactory(service=service) + _event(service, component, external_id="1", title="Broke") + _event( + service, + component, + external_id="2", + title="Planned", + kind=EventKind.MAINTENANCE, + phase=MaintenancePhase.SCHEDULED, + ) + + both = client.get(reverse("event-list")).json()["results"] + incidents = client.get(reverse("event-list"), {"kind": "incident"}).json()[ + "results" + ] + assert len(both) == 2 + assert [r["title"] for r in incidents] == ["Broke"] + + +def test_the_feed_aggregate_is_by_phase_only(client): + # A severity belongs to a component. An event carries a phase, so + # a severity count here would answer about the wrong row. + aggregates = client.get(reverse("event-list")).json()["aggregates"] + assert set(aggregates) == {"total", "by_phase"} + + +def test_an_event_carries_no_component_ids(client): + # The projection runs one way. A component carries its event, and + # the reverse would put an unbounded list on every row. + service = ServiceFactory() + _event(service, ComponentFactory(service=service), external_id="1") + + row = client.get(reverse("event-list")).json()["results"][0] + assert set(row) == { + "id", + "kind", + "title", + "phase", + "starts_at", + "ends_at", + "detected_by", + "service", + } + + +def test_phase_open_and_closed_draw_one_line(client): + # `CLOSED_PHASES` lives in status/choices.py. A client restating + # which phases are terminal is a second copy that can drift. + service = ServiceFactory() + component = ComponentFactory(service=service) + _event(service, component, external_id="1", title="Open") + _event( + service, + component, + external_id="2", + title="Done", + phase=IncidentPhase.RESOLVED, + ) + _event( + service, + component, + external_id="3", + title="Finished", + kind=EventKind.MAINTENANCE, + phase=MaintenancePhase.COMPLETED, + ) + + open_titles = client.get(reverse("event-list"), {"phase": "open"}).json()["results"] + closed = client.get(reverse("event-list"), {"phase": "closed"}).json()["results"] + assert [r["title"] for r in open_titles] == ["Open"] + assert {r["title"] for r in closed} == {"Done", "Finished"} + + +def test_no_phase_parameter_returns_every_event(client): + # Omitting `phase` is not a filter. The code reads as though open + # were the default, and a change that made it one would be silent. + service = ServiceFactory() + component = ComponentFactory(service=service) + _event(service, component, external_id="1", title="Open") + _event( + service, + component, + external_id="2", + title="Done", + phase=IncidentPhase.RESOLVED, + ) + + results = client.get(reverse("event-list")).json()["results"] + assert {r["title"] for r in results} == {"Open", "Done"} + + +def test_dashboard_narrows_to_what_you_track(authenticated_client, board): + # Home's Updates tab. Everything posted across the services on + # your board, and nothing else. + # + # `DEFAULT_AUTHENTICATION_CLASSES` is JWT only, so a plain + # `force_login` leaves this view answering 401. + # `authenticated_client` bears board.owner's own access token. + tracked = ComponentFactory() + untracked = ComponentFactory() + track(tracked, user=board.owner) + _event(tracked.service, tracked, external_id="1", title="Yours") + _event(untracked.service, untracked, external_id="2", title="Not yours") + + results = authenticated_client.get( + reverse("event-list"), {"dashboard": str(board.id)} + ).json()["results"] + assert [r["title"] for r in results] == ["Yours"] + + +def test_an_anonymous_caller_cannot_narrow_to_a_dashboard(client, board): + # `get_object_or_404(..., owner=request.user)` would compare against + # `AnonymousUser` and raise `TypeError` instead of answering cleanly. + response = client.get(reverse("event-list"), {"dashboard": str(board.id)}) + assert response.status_code == 401 + + +def test_a_dashboard_that_is_not_yours_is_not_readable(authenticated_client): + # 404, not 403: a 403 would let anyone holding a valid token + # confirm whether a board id exists. + other = User.objects.create(email="someone-else@example.test") + response = authenticated_client.get( + reverse("event-list"), {"dashboard": str(other.default_dashboard.id)} + ) + assert response.status_code == 404 + + +def test_a_malformed_dashboard_id_is_a_400_not_a_500(authenticated_client): + # `EventFilter`'s `dashboard` field is a `UUIDFilter`. A bad value + # reaches django-filter and comes back like any other malformed + # filter value. It must not surface as an unhandled ValueError. + response = authenticated_client.get( + reverse("event-list"), {"dashboard": "not-a-uuid"} + ) + assert response.status_code == 400 + + +def test_the_dashboard_filter_does_not_repeat_an_event(authenticated_client, board): + # One event naming two components tracked on the same board is one + # outage. Without .distinct() the M2M join returns one row per + # matching component, and the feed shows it twice. + service = ServiceFactory() + first = ComponentFactory(service=service) + second = ComponentFactory(service=service) + track(first, user=board.owner) + track(second, user=board.owner) + event = _event(service, first, external_id="1", title="Shared") + event.affected_components.add(second) + + results = authenticated_client.get( + reverse("event-list"), {"dashboard": str(board.id)} + ).json()["results"] + assert [r["title"] for r in results] == ["Shared"] + + +def test_the_detail_carries_both_tab_counts(client): + # The header draws `Timeline 3` and `Affects 2` before either tab + # is opened, so neither can wait for that tab's request. + service = ServiceFactory() + first = ComponentFactory(service=service) + second = ComponentFactory(service=service) + event = _event(service, first, external_id="1") + event.affected_components.add(second) + for minute in range(3): + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.INVESTIGATING, + body=f"update {minute}", + posted_at=timezone.now(), + ) + + body = client.get(reverse("event-detail", args=[event.id])).json() + assert body["update_count"] == 3 + assert body["affected_count"] == 2 + + +def test_the_affects_badge_leaves_out_an_archived_component(client): + # The badge and the Affects tab count the same rows. The tab reads + # `?event=`, which does not serve an archived component. + service = ServiceFactory() + kept = ComponentFactory(service=service) + gone = ComponentFactory(service=service) + event = _event(service, kept, external_id="1") + event.affected_components.add(gone) + gone.is_archived = True + gone.save() + + badge = client.get(reverse("event-detail", args=[event.id])).json() + listed = client.get(reverse("component-list"), {"event": str(event.id)}).json() + assert badge["affected_count"] == 1 + assert listed["aggregates"]["total"] == 1 + + +def test_the_timeline_is_its_own_paged_list(client): + # A provider's log has no ceiling, so it cannot ride on the detail. + service = ServiceFactory() + event = _event(service, ComponentFactory(service=service), external_id="1") + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.IDENTIFIED, + body="Cause found", + posted_at=timezone.now(), + source=EventSource.PROVIDER, + ) + + body = client.get(reverse("event-updates", args=[event.id])).json() + assert body["aggregates"]["total"] == 1 + assert body["results"][0]["source"] == "provider" + + +def test_the_timeline_is_oldest_first(client): + # A story is read forwards. The feed is newest first because it is + # a feed; one event's log is a narrative. + service = ServiceFactory() + event = _event(service, ComponentFactory(service=service), external_id="1") + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.IDENTIFIED, + body="Second", + posted_at=timezone.now() + timezone.timedelta(minutes=1), + ) + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.INVESTIGATING, + body="First", + posted_at=timezone.now(), + ) + + results = client.get(reverse("event-updates", args=[event.id])).json()["results"] + assert [r["body"] for r in results] == ["First", "Second"] diff --git a/api/tests/test_import.py b/api/tests/test_import.py index d282770..a4d5d5b 100644 --- a/api/tests/test_import.py +++ b/api/tests/test_import.py @@ -88,8 +88,9 @@ def test_urls_are_deduplicated_after_normalising(): @pytest.mark.django_db -def test_the_imported_service_is_identical_in_shape_to_a_listed_one(): - # If a field is missing here it belongs in the list too, not in a second shape. +def test_the_imported_service_is_identical_in_shape_to_a_read_one(): + # If a field is missing here it belongs on the service page too, + # not in a second shape. created = ( APIClient() .post( @@ -99,8 +100,8 @@ def test_the_imported_service_is_identical_in_shape_to_a_listed_one(): ) .json() ) - listed = APIClient().get(reverse("service-list")).json()["results"][0] - assert set(created) == set(listed) + url = reverse("service-detail", args=[created["slug"]]) + assert set(created) == set(APIClient().get(url).json()) @pytest.mark.django_db @@ -119,10 +120,12 @@ def test_the_response_carries_the_detected_provider(): @pytest.mark.django_db def test_a_missing_status_page_url_is_a_400(): - assert ( - APIClient().post(reverse("catalog-import"), {}, format="json").status_code - == 400 - ) + response = APIClient().post(reverse("catalog-import"), {}, format="json") + assert response.status_code == 400 + # The serializer's own field map, not the `Error` shape. The + # contract documents both under this code, and a client tells them + # apart by whether `code` is there. + assert response.json() == {"status_page_url": ["This field is required."]} def test_normalising_keeps_the_www_prefix(): @@ -231,3 +234,47 @@ def test_a_second_import_of_the_same_page_returns_the_first_service(): assert again is False assert second.pk == first.pk assert Service.objects.count() == 1 + + +@pytest.mark.django_db +def test_an_import_records_the_caller_who_asked_for_it(user): + # The bot signs automated rows only. Dropping the caller here + # records a person's import as the system account. + from tests.conftest import jwt_client + + jwt_client(user).post( + reverse("catalog-import"), + {"status_page_url": "https://status.twilio.com/"}, + format="json", + ) + + assert Service.objects.get().created_by == user + + +@pytest.mark.django_db +def test_an_anonymous_import_names_nobody_rather_than_the_bot(): + # The endpoint is AllowAny, so a signed-out person can import. A + # service request answers this the same way, with null. A person + # asked, and the system account would claim a job did. + APIClient().post( + reverse("catalog-import"), + {"status_page_url": "https://status.twilio.com/"}, + format="json", + ) + + assert Service.objects.get().created_by is None + + +@pytest.mark.django_db +def test_an_imported_service_says_it_was_imported(): + # The human tells a service added by hand from one read off a page. + # A row left at the default claims somebody typed it in. + from catalog.choices import ServiceSource + + APIClient().post( + reverse("catalog-import"), + {"status_page_url": "https://status.twilio.com/"}, + format="json", + ) + + assert Service.objects.get().source == ServiceSource.IMPORT diff --git a/api/tests/test_meta.py b/api/tests/test_meta.py index bc2be8e..849a44f 100644 --- a/api/tests/test_meta.py +++ b/api/tests/test_meta.py @@ -36,7 +36,9 @@ def test_meta_publishes_every_enum(): "status_source", "status_page_provider", "event_kind", + "event_source", "event_phase", + "event_phase_state", } assert body["enums"]["severity"]["0"] == "Major outage" assert body["enums"]["severity"]["5"] == "Operational" @@ -44,6 +46,69 @@ def test_meta_publishes_every_enum(): assert set(body["enums"]["event_phase"]) == {"incident", "maintenance"} assert "investigating" in body["enums"]["event_phase"]["incident"] assert "scheduled" in body["enums"]["event_phase"]["maintenance"] + # The `phase=` filter's own two values. Without them a client reads + # the values off the OpenAPI enum and writes the labels itself. + assert body["enums"]["event_phase_state"] == {"open": "Open", "closed": "Closed"} + + +@pytest.mark.django_db +def test_fields_prunes_the_top_level(): + # /meta/ built its dict and skipped the serializer that prunes it. + # ?fields= was declared but did nothing. + body = APIClient().get(reverse("meta") + "?fields=max_page_size").json() + assert body == {"max_page_size": 200} + + +@pytest.mark.django_db +def test_fields_prunes_inside_enums(): + # `enums` is a plain dict. FieldsMixin only prunes a nested + # serializer, so a dotted path into `enums` reached nothing. + url = reverse("meta") + "?fields=enums.status_source" + body = APIClient().get(url).json() + assert body == { + "enums": { + "status_source": { + "provider": "Provider", + "components": "Components", + "incidents": "Incidents", + } + } + } + + +@pytest.mark.django_db +def test_fields_enums_with_no_dotted_path_keeps_every_enum(): + # `?fields=enums` names the whole map, not one entry in it. + body = APIClient().get(reverse("meta") + "?fields=enums").json() + assert set(body) == {"enums"} + assert set(body["enums"]) == { + "severity", + "status_source", + "status_page_provider", + "event_kind", + "event_source", + "event_phase", + "event_phase_state", + } + + +@pytest.mark.django_db +def test_fields_keeps_two_dotted_paths_under_enums(): + # Two dotted paths under enums must both survive. Parsing only the + # last one would silently drop status_source here. + url = reverse("meta") + "?fields=enums.status_source,enums.event_kind" + body = APIClient().get(url).json() + assert set(body["enums"]) == {"status_source", "event_kind"} + + +@pytest.mark.django_db +def test_fields_refuses_an_unknown_name_under_enums(): + # `enums` is a plain dict, filtered by hand, so `_prune` never saw + # this name. It answered `200 {"enums": {}}`, which is the silent + # empty payload the 400 exists to prevent. + response = APIClient().get(reverse("meta") + "?fields=enums.nonsense") + assert response.status_code == 400 + assert response.json() == {"fields": ["Unknown field: nonsense."]} @pytest.mark.django_db @@ -156,7 +221,7 @@ def test_every_throttled_view_names_a_rate_that_exists(): from api.defaults import Throttle from authentication.views import MagicLinkView - from catalog.views import CatalogImportView + from catalog.views.imports import CatalogImportView named = {MagicLinkView.throttle_scope, CatalogImportView.throttle_scope} rates = settings.REST_FRAMEWORK["DEFAULT_THROTTLE_RATES"] diff --git a/api/tests/test_ordering.py b/api/tests/test_ordering.py index d168e49..1aff1e0 100644 --- a/api/tests/test_ordering.py +++ b/api/tests/test_ordering.py @@ -1,4 +1,9 @@ +import pytest +from django.urls import reverse +from rest_framework.exceptions import ValidationError + from common.ordering import MappedOrderingFilter +from tests.factories import ComponentFactory, ServiceFactory, track def _view(ordering_fields, ordering_map): @@ -17,12 +22,23 @@ def test_a_related_path_ordering_does_not_reach_the_paginator(): assert all("__" not in term for term in out) -def test_an_unmapped_unknown_value_is_dropped(): +def test_an_unmapped_unknown_value_is_refused(): + # Dropping it sorted the list by the default instead. The caller + # asked for one order and read another, with nothing to say so. view = _view(["name"], {}) - assert ( + with pytest.raises(ValidationError) as raised: MappedOrderingFilter().remove_invalid_fields(None, ["nonsense"], view, None) - == [] - ) + assert raised.value.detail == {"ordering": ["Unknown ordering: nonsense."]} + + +def test_reversing_a_key_that_names_its_own_directions_is_refused(): + # `-suggested` prefixed every mapped term and built `--is_featured`. + # The ORM raised FieldError from there, which reached the caller as + # a 500. + view = _view(["name"], {"suggested": ["-is_featured", "name"]}) + with pytest.raises(ValidationError) as raised: + MappedOrderingFilter().remove_invalid_fields(None, ["-suggested"], view, None) + assert raised.value.detail == {"ordering": ["Cannot reverse: -suggested."]} def test_a_value_that_is_not_a_field_expands_to_several(): @@ -37,3 +53,47 @@ def test_a_plain_declared_field_passes_through(): assert MappedOrderingFilter().remove_invalid_fields( None, ["-name"], view, None ) == ["-name"] + + +def _component_list(client, authed, board): + ComponentFactory(service=ServiceFactory(), is_overall=True) + return client, reverse("component-list") + + +def _event_list(client, authed, board): + return client, reverse("event-list") + + +def _board_components(client, authed, board): + track(ComponentFactory(), user=board.owner) + return authed, reverse("board-components", kwargs={"uuid": board.id}) + + +# One builder per operation the contract gives an `ordering` parameter. +# A list added later and left out here would shrug again. +ORDERING_PROBES = { + "/catalog/components/": _component_list, + "/events/": _event_list, + "/dashboards/{uuid}/components/": _board_components, +} + + +@pytest.mark.django_db +@pytest.mark.parametrize("path", sorted(ORDERING_PROBES)) +def test_an_unknown_ordering_is_refused(path, client, authenticated_client, board): + # `?fields=` refuses an unknown name and `?ordering=` shrugged. One + # parameter that answers 400 and one that does not is two contracts. + caller, url = ORDERING_PROBES[path](client, authenticated_client, board) + response = caller.get(url, {"ordering": "nonsense"}) + assert response.status_code == 400, f"GET {path} answered {response.status_code}" + assert response.json() == {"ordering": ["Unknown ordering: nonsense."]} + + +@pytest.mark.django_db +def test_reversing_suggested_is_a_400_not_a_500(client, authenticated_client, board): + # `suggested` is four terms, two of them already descending. The + # prefix built `--is_featured` and the ORM raised on it. + caller, url = _component_list(client, authenticated_client, board) + response = caller.get(url, {"ordering": "-suggested"}) + assert response.status_code == 400 + assert response.json() == {"ordering": ["Cannot reverse: -suggested."]} diff --git a/api/tests/test_polling.py b/api/tests/test_polling.py index 7fa9b7f..aa2c286 100644 --- a/api/tests/test_polling.py +++ b/api/tests/test_polling.py @@ -43,11 +43,11 @@ def test_a_poller_inside_its_cooldown_is_not_due(): @pytest.mark.django_db def test_two_boards_tracking_one_service_produce_one_poller(): - # The cooldown is on the service, not on a user or a board. - service = ServiceFactory(tracked=1) + # The cooldown is on the service, not on a user or a board. `due()` + # asks whether anybody tracks the service. Asked as a join instead, + # it would return this poller once per board and poll twice as often. + service = ServiceFactory(tracked=2) StatusPageFactory(service=service) - PollerFactory(service=service) - assert Poller.objects.filter(service=service).count() == 1 assert len(list(Poller.objects.due())) == 1 @@ -244,8 +244,25 @@ def fetch_service_metadata(self): @pytest.mark.django_db -def test_a_poller_nobody_added_is_signed_by_the_system(): - """The signal adds it, so the author says the system, not nobody. +def test_a_poller_carries_the_author_of_the_service_that_spawned_it(): + """A person caused the service, so the poller records that person. + + Stamping the system account here would name the bot for work a + person did. + """ + from catalog.models import Service + from tests.factories import UserFactory + + person = UserFactory() + + service = Service.objects.create(name="Owned", created_by=person) + + assert service.poller.created_by == person + + +@pytest.mark.django_db +def test_a_poller_for_a_service_with_no_author_is_signed_by_the_system(): + """The fallback, for a service nobody signed. A blank author reads the same as one that was lost. """ @@ -320,8 +337,9 @@ def test_an_import_schedules_the_same_way_a_poll_does(): # Two callers wrote this. The import set a bare interval, with no # backoff and no jitter. A hundred imported at once came due on the # same second. - from polling.models import JITTER - + # + # The window is a tenth either side, written out. Read from the + # constant, this would pass with the constant itself set wrong. poller = PollerFactory(service=ServiceFactory(tracked=1)) at = timezone.now() @@ -331,7 +349,7 @@ def test_an_import_schedules_the_same_way_a_poll_does(): assert poller.consecutive_failure_count == 0 interval = poller.effective_interval_seconds gap = (poller.next_at - at).total_seconds() - assert interval * (1 - JITTER) <= gap <= interval * (1 + JITTER) + assert interval * 0.9 <= gap <= interval * 1.1 @pytest.mark.django_db diff --git a/api/tests/test_reconcile.py b/api/tests/test_reconcile.py index 7c7f78f..789e12e 100644 --- a/api/tests/test_reconcile.py +++ b/api/tests/test_reconcile.py @@ -8,9 +8,15 @@ NormalisedUpdate, ) from polling.reconcile import apply_fetch -from status.choices import EventKind, IncidentPhase, Severity, StatusSource +from status.choices import ( + EventKind, + EventSource, + IncidentPhase, + Severity, + StatusSource, +) from status.models import ComponentStatus, ServiceEvent -from tests.factories import ComponentFactory, ServiceFactory +from tests.factories import ComponentFactory, ServiceFactory, ancestry, poll_run def _component(external_id="a", name="SMS", severity=Severity.OPERATIONAL, **kw): @@ -27,7 +33,7 @@ def _component(external_id="a", name="SMS", severity=Severity.OPERATIONAL, **kw) @pytest.mark.django_db def test_a_new_component_is_created(): service = ServiceFactory() - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, poll_run(service)) assert ( ServiceComponent.objects.filter(service=service, external_id="a").count() == 1 ) @@ -38,7 +44,11 @@ def test_a_rename_updates_the_existing_row_rather_than_creating_one(): service = ServiceFactory() ComponentFactory(service=service, external_id="a", name="SMS") apply_fetch( - service, [_component(name="Programmable Messaging")], [], StatusSource.PROVIDER + service, + [_component(name="Programmable Messaging")], + [], + StatusSource.PROVIDER, + poll_run(service), ) rows = ServiceComponent.objects.filter(service=service) assert rows.count() == 1 @@ -50,7 +60,7 @@ def test_a_vanished_component_is_archived_not_deleted(): # Someone may be tracking it. Deleting would silently remove it from their board. service = ServiceFactory() ComponentFactory(service=service, external_id="gone", name="Old") - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, poll_run(service)) old = ServiceComponent.objects.get(service=service, external_id="gone") assert old.archived_at is not None assert ServiceComponent.objects.filter(service=service).count() == 2 @@ -60,7 +70,7 @@ def test_a_vanished_component_is_archived_not_deleted(): def test_a_component_that_reappears_is_unarchived(): service = ServiceFactory() ComponentFactory(service=service, external_id="a", archived_at=timezone.now()) - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, poll_run(service)) assert ( ServiceComponent.objects.get(service=service, external_id="a").archived_at is None @@ -75,6 +85,7 @@ def test_the_parent_child_structure_is_refreshed(): [_component("parent", "Group"), _component("child", "SMS", parent="parent")], [], StatusSource.PROVIDER, + poll_run(service), ) child = ServiceComponent.objects.get(service=service, external_id="child") assert child.parent.external_id == "parent" @@ -83,8 +94,8 @@ def test_the_parent_child_structure_is_refreshed(): @pytest.mark.django_db def test_an_unchanged_severity_does_not_open_a_second_status_row(): service = ServiceFactory() - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, poll_run(service)) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, poll_run(service)) component = ServiceComponent.objects.get(service=service, external_id="a") assert ComponentStatus.objects.filter(component=component).count() == 1 @@ -92,9 +103,13 @@ def test_an_unchanged_severity_does_not_open_a_second_status_row(): @pytest.mark.django_db def test_a_changed_severity_closes_the_old_row_and_opens_a_new_one(): service = ServiceFactory() - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, poll_run(service)) apply_fetch( - service, [_component(severity=Severity.MAJOR_OUTAGE)], [], StatusSource.PROVIDER + service, + [_component(severity=Severity.MAJOR_OUTAGE)], + [], + StatusSource.PROVIDER, + poll_run(service), ) component = ServiceComponent.objects.get(service=service, external_id="a") rows = ComponentStatus.objects.filter(component=component).order_by("started_at") @@ -114,7 +129,7 @@ def test_an_event_is_upserted_on_its_provider_id(): phase=IncidentPhase.INVESTIGATING, starts_at=timezone.now(), ) - apply_fetch(service, [], [event], StatusSource.PROVIDER) + apply_fetch(service, [], [event], StatusSource.PROVIDER, poll_run(service)) apply_fetch( service, [], @@ -128,6 +143,7 @@ def test_an_event_is_upserted_on_its_provider_id(): ) ], StatusSource.PROVIDER, + poll_run(service), ) rows = ServiceEvent.objects.filter(service=service) assert rows.count() == 1 @@ -152,26 +168,65 @@ def test_an_event_is_linked_to_the_components_it_names(): ) ], StatusSource.PROVIDER, + poll_run(service), ) event = ServiceEvent.objects.get(service=service) assert [c.external_id for c in event.affected_components.all()] == ["a"] +@pytest.mark.django_db +def test_a_later_provider_post_joins_the_event_the_poll_opened(): + # A poll saw the drop before the provider wrote it up. Without the + # claim the feed shows two cards for one outage. + service = ServiceFactory() + apply_fetch( + service, + [_component("a", severity=Severity.DEGRADED)], + [], + StatusSource.PROVIDER, + poll_run(service), + ) + ours = ServiceEvent.objects.get(service=service) + assert ours.detected_by == EventSource.SYSTEM + + apply_fetch( + service, + [_component("a", severity=Severity.DEGRADED)], + [ + NormalisedEvent( + external_id="inc-1", + kind=EventKind.INCIDENT, + title="Elevated SMS delivery failures", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + affected_external_ids=("a",), + updates=( + NormalisedUpdate( + phase=IncidentPhase.INVESTIGATING, + body="We are looking into it", + posted_at=timezone.now(), + ), + ), + ) + ], + StatusSource.PROVIDER, + poll_run(service), + ) + + event = ServiceEvent.objects.get(service=service) + assert event.pk == ours.pk + assert event.external_id == "inc-1" + # We found it first, and a claim never rewrites that. + assert event.detected_by == EventSource.SYSTEM + assert event.updates.count() == 2 + + @pytest.mark.django_db def test_a_run_stamps_what_it_wrote(): # A wrong or stale reading is otherwise untraceable: you can see what # it says and not where it came from. - from polling.models import PollRun - from tests.factories import PollerFactory, StatusPageFactory - service = ServiceFactory() - page = StatusPageFactory(service=service) - run = PollRun.objects.create( - poller=PollerFactory(service=service), - url=page.url, - provider=page.provider, - started_at=timezone.now(), - ) + run = poll_run(service) event = NormalisedEvent( external_id="inc-1", kind=EventKind.INCIDENT, @@ -185,14 +240,6 @@ def test_a_run_stamps_what_it_wrote(): assert ServiceEvent.objects.get(service=service).poll_run == run -@pytest.mark.django_db -def test_a_hand_seeded_row_has_no_run(): - # apply_fetch is called without one when nothing polled. - service = ServiceFactory() - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) - assert ComponentStatus.objects.get(component__service=service).poll_run is None - - @pytest.mark.django_db def test_a_poll_signs_everything_it_writes(): """Nobody types any of this, so nothing is left without an author. @@ -220,7 +267,9 @@ def test_a_poll_signs_everything_it_writes(): ) ], ) - apply_fetch(service, [_component()], [event], StatusSource.PROVIDER) + apply_fetch( + service, [_component()], [event], StatusSource.PROVIDER, poll_run(service) + ) system = get_user_model().objects.system() for model in (ServiceComponent, ComponentStatus, ServiceEvent, EventUpdate): @@ -235,11 +284,42 @@ def test_a_second_poll_does_not_rewrite_who_made_the_row(): # A row records who made it once. Re-signing it on every poll would # make `created_at` and `created_by` disagree. service = ServiceFactory() - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, poll_run(service)) first = ServiceComponent.objects.get() - apply_fetch(service, [_component(name="Renamed")], [], StatusSource.PROVIDER) + apply_fetch( + service, + [_component(name="Renamed")], + [], + StatusSource.PROVIDER, + poll_run(service), + ) again = ServiceComponent.objects.get() assert again.created_by_id == first.created_by_id assert again.created_at == first.created_at + + +@pytest.mark.django_db +def test_a_reparent_moves_the_whole_subtree(): + # A provider can lift a group out from under another. The poll + # writes one parent, and the grandchild below it has to read its + # new place without a second write. + service = ServiceFactory() + nested = [ + _component(external_id="a"), + _component(external_id="b", parent="a"), + _component(external_id="c", parent="b"), + ] + apply_fetch(service, nested, [], StatusSource.PROVIDER, poll_run(service)) + + moved = [ + _component(external_id="a"), + _component(external_id="b"), + _component(external_id="c", parent="b"), + ] + apply_fetch(service, moved, [], StatusSource.PROVIDER, poll_run(service)) + + assert ancestry(ServiceComponent.objects.get(external_id="c")) == [ + ServiceComponent.objects.get(external_id="b").id + ] diff --git a/api/tests/test_security.py b/api/tests/test_security.py index 1157e4f..465ebb2 100644 --- a/api/tests/test_security.py +++ b/api/tests/test_security.py @@ -120,7 +120,7 @@ def test_reading_the_catalog_is_not_throttled_away(db): # The wide rates are the point: the catalog is public and scanned. ServiceFactory() client = APIClient() - codes = [client.get(reverse("service-list")).status_code for _ in range(30)] + codes = [client.get(reverse("component-list")).status_code for _ in range(30)] assert set(codes) == {200} diff --git a/api/tests/test_service_requests.py b/api/tests/test_service_requests.py new file mode 100644 index 0000000..49af60d --- /dev/null +++ b/api/tests/test_service_requests.py @@ -0,0 +1,118 @@ +import pytest +from django.urls import reverse + +from catalog.models import ServiceRequest +from tests.conftest import jwt_client + +pytestmark = pytest.mark.django_db + + +def test_asking_records_the_url(client): + # The write itself. Regressing here loses the row, and the admin + # list this endpoint feeds has nothing to show. + response = client.post( + reverse("catalog-request"), + {"url": "https://status.fastmail.com/"}, + content_type="application/json", + ) + assert response.status_code == 202 + row = ServiceRequest.objects.get() + assert row.url == "https://status.fastmail.com" + assert row.request_count == 1 + # Anonymous stays anonymous. A future edit that stopped guarding + # `is_authenticated` would only fail here, not on save. + assert row.created_by is None + + +def test_asking_twice_counts_rather_than_duplicates(client): + # State belongs to the URL, not to a request. A row per request + # would answer one question twice once a URL was triaged. + for _ in range(2): + client.post( + reverse("catalog-request"), + {"url": "https://status.fastmail.com/"}, + content_type="application/json", + ) + assert ServiceRequest.objects.count() == 1 + assert ServiceRequest.objects.get().request_count == 2 + + +def test_two_spellings_of_one_url_are_one_row(client): + # `url` is unique on the normalised form. Skipping normalisation + # before the lookup would let a trailing slash duplicate the row + # `normalise_url` exists to prevent. + client.post( + reverse("catalog-request"), + {"url": "https://status.example.com/"}, + content_type="application/json", + ) + client.post( + reverse("catalog-request"), + {"url": "https://status.example.com"}, + content_type="application/json", + ) + assert ServiceRequest.objects.count() == 1 + assert ServiceRequest.objects.get().request_count == 2 + + +def test_the_answer_is_the_same_whether_we_hold_it_or_not(client): + # Always 202. A different code would reveal which URLs are + # already in the catalog to anyone who asked. + first = client.post( + reverse("catalog-request"), + {"url": "https://status.one.example/"}, + content_type="application/json", + ) + second = client.post( + reverse("catalog-request"), + {"url": "https://status.one.example/"}, + content_type="application/json", + ) + assert first.status_code == second.status_code == 202 + assert first.content == second.content + + +def test_a_signed_in_asker_is_recorded(user): + # created_by carries it. There is no requested_by: that is what + # BaseModel already holds. `force_login` leaves this view + # answering 401, because there is no session backend. + jwt_client(user).post( + reverse("catalog-request"), + {"url": "https://status.two.example/"}, + format="json", + ) + assert ServiceRequest.objects.get().created_by == user + + +def test_a_malformed_url_is_refused(client): + # Refused before it is stored. Skipping validation would let junk + # into the row the admin list is ordered by. + response = client.post( + reverse("catalog-request"), + {"url": "not a url"}, + content_type="application/json", + ) + assert response.status_code == 400 + + +def test_a_failed_import_does_not_spend_this_endpoints_budget(client, monkeypatch): + # Add-by-URL tries the import, then reports the same URL here. One + # shared scope would let the import spend this endpoint's budget. + # An asker who had already tried six times could never send us + # the URL that failed. + def refuses(url, author=None): + raise ValueError(f"no adapter for {url}") + + monkeypatch.setattr("catalog.views.imports.import_from_url", refuses) + for _ in range(6): + client.post( + reverse("catalog-import"), + {"status_page_url": "https://status.example.com/"}, + content_type="application/json", + ) + response = client.post( + reverse("catalog-request"), + {"url": "https://status.example.com/"}, + content_type="application/json", + ) + assert response.status_code == 202 diff --git a/api/tests/test_status_models.py b/api/tests/test_status_models.py index 04c624b..37ceb1b 100644 --- a/api/tests/test_status_models.py +++ b/api/tests/test_status_models.py @@ -4,7 +4,9 @@ from django.utils import timezone from status.choices import ( + CLOSED_PHASES, EventKind, + EventSource, IncidentPhase, MaintenancePhase, Severity, @@ -87,16 +89,39 @@ def test_a_matching_phase_validates(): event.full_clean() -@pytest.mark.django_db -def test_an_event_may_name_no_component_at_all(): - # A provider publishes some events against the service. - # A component FK cannot hold them. - event = ServiceEvent.objects.create( - service=ServiceFactory(), - external_id="1", - kind=EventKind.MAINTENANCE, - title="x", - phase=MaintenancePhase.SCHEDULED, - starts_at=timezone.now(), - ) - assert event.affected_components.count() == 0 +def test_two_system_events_can_sit_on_one_service(db): + # Neither has a provider id. The unique key is partial, so a null + # does not collide with another null. + service = ServiceFactory() + for _ in range(2): + ServiceEvent.objects.create( + service=service, + external_id=None, + kind=EventKind.INCIDENT, + title="Degraded", + phase=IncidentPhase.DETECTED, + detected_by=EventSource.SYSTEM, + starts_at=timezone.now(), + ) + assert ServiceEvent.objects.filter(service=service).count() == 2 + + +def test_a_provider_id_is_still_unique_per_service(db): + # A second poll of the same page must update the row, never add one. + service = ServiceFactory() + fields = { + "service": service, + "kind": EventKind.INCIDENT, + "title": "Outage", + "phase": IncidentPhase.INVESTIGATING, + "starts_at": timezone.now(), + } + ServiceEvent.objects.create(external_id="abc", **fields) + with pytest.raises(IntegrityError): + ServiceEvent.objects.create(external_id="abc", **fields) + + +def test_detected_is_an_open_phase(db): + # A detected event is running. Listing it as closed would hide + # every outage no provider ever wrote up. + assert IncidentPhase.DETECTED not in CLOSED_PHASES diff --git a/api/tests/test_system_events.py b/api/tests/test_system_events.py new file mode 100644 index 0000000..598ff66 --- /dev/null +++ b/api/tests/test_system_events.py @@ -0,0 +1,423 @@ +from datetime import timedelta + +import pytest +from django.contrib.auth import get_user_model +from django.utils import timezone + +from polling.system_events import claim, reconcile_system_events +from status.choices import ( + EventKind, + EventSource, + IncidentPhase, + MaintenancePhase, + Severity, + StatusSource, +) +from status.models import ComponentStatus, EventUpdate, ServiceEvent +from tests.factories import ComponentFactory, ServiceFactory + +pytestmark = pytest.mark.django_db + + +def _status(component, severity, started_at=None): + ComponentStatus.objects.create( + component=component, + severity=severity, + source=StatusSource.PROVIDER, + started_at=started_at or timezone.now(), + ) + + +def _author(): + return get_user_model().objects.system() + + +def test_an_unexplained_outage_opens_an_event(): + # Without this the feed hides every outage a provider never wrote + # up, and the closed span is exposed nowhere else. + service = ServiceFactory() + component = ComponentFactory(service=service, name="SMS") + _status(component, Severity.DEGRADED) + + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.detected_by == EventSource.SYSTEM + assert event.external_id is None + assert event.phase == IncidentPhase.DETECTED + assert event.kind == EventKind.INCIDENT + assert list(event.affected_components.all()) == [component] + assert event.updates.count() == 1 + assert event.updates.first().source == EventSource.SYSTEM + + +def test_an_explained_outage_opens_nothing(): + # The provider already told the story. A second card for one + # outage is the thing this design exists to avoid. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.MAJOR_OUTAGE) + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Outage", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + ) + provider.affected_components.set([component]) + + reconcile_system_events(service, _author()) + + assert ServiceEvent.objects.filter(detected_by=EventSource.SYSTEM).count() == 0 + + +def test_a_service_wide_incident_explains_its_service(): + # One outage is one card. A provider incident naming no component + # covers every component of that service. + service = ServiceFactory() + _status(ComponentFactory(service=service), Severity.DEGRADED) + ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Everything is down", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + ) + + reconcile_system_events(service, _author()) + + assert ServiceEvent.objects.filter(detected_by=EventSource.SYSTEM).count() == 0 + + +def test_a_service_wide_maintenance_window_explains_nothing(): + # A planned window is not an account of an outage. Read as one, a + # provider doing scheduled work hides every outage beside it. + service = ServiceFactory() + _status(ComponentFactory(service=service), Severity.DEGRADED) + ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.MAINTENANCE, + title="Planned work", + phase=MaintenancePhase.IN_PROGRESS, + starts_at=timezone.now(), + ) + + reconcile_system_events(service, _author()) + + assert ServiceEvent.objects.filter(detected_by=EventSource.SYSTEM).count() == 1 + + +def test_unknown_opens_nothing(): + # Severity 3 is our own poll failing to read their page. Calling + # that their incident would report our fault as theirs. + service = ServiceFactory() + _status(ComponentFactory(service=service), Severity.UNKNOWN) + + reconcile_system_events(service, _author()) + + assert ServiceEvent.objects.count() == 0 + + +def test_every_transition_while_open_writes_an_update(): + # A card shows how the outage moved. One update at the start would + # say a major outage had been degraded the whole time. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + + ComponentStatus.objects.filter(component=component, ended_at__isnull=True).update( + ended_at=timezone.now() + ) + _status(component, Severity.MAJOR_OUTAGE) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.updates.count() == 2 + assert event.ends_at is None + + +def test_a_poll_that_changes_nothing_posts_nothing(): + # Without this every poll beat posts a duplicate update to an open + # card. The timeline then reads one line per beat, not per change. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + + reconcile_system_events(service, _author()) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.updates.count() == 1 + + +def test_unknown_holds_the_event_open(): + # Unknown is our poll failing to read their page. Closing on it + # tells somebody an outage ended when we stopped being able to see. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.MAJOR_OUTAGE) + reconcile_system_events(service, _author()) + + ComponentStatus.objects.filter(component=component, ended_at__isnull=True).update( + ended_at=timezone.now() + ) + _status(component, Severity.UNKNOWN) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.phase == IncidentPhase.DETECTED + assert event.ends_at is None + + +def test_recovery_closes_the_event(): + # An event that cannot close leaves a permanently red row on + # somebody's board. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.MAJOR_OUTAGE) + reconcile_system_events(service, _author()) + + ComponentStatus.objects.filter(component=component, ended_at__isnull=True).update( + ended_at=timezone.now() + ) + _status(component, Severity.OPERATIONAL) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.phase == IncidentPhase.RESOLVED + assert event.ends_at is not None + assert event.updates.count() == 2 + + +def test_a_provider_event_claims_the_one_we_opened(): + # One outage is one card. Two cards for one outage is what the + # claim exists to prevent. + service = ServiceFactory() + component = ComponentFactory(service=service, name="SMS") + voice = ComponentFactory(service=service, name="Voice") + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Elevated SMS delivery failures", + phase=IncidentPhase.INVESTIGATING, + starts_at=ours.starts_at - timedelta(minutes=10), + ) + provider.affected_components.set([component, voice]) + EventUpdate.objects.create( + event=provider, + phase=IncidentPhase.INVESTIGATING, + body="We are looking into it", + posted_at=provider.starts_at, + ) + + assert claim(provider, _author()) is True + + ours.refresh_from_db() + assert ours.external_id == "abc" + assert ours.title == "Elevated SMS delivery failures" + assert ours.phase == IncidentPhase.INVESTIGATING + # Ours records who found it, and a claim never rewrites that. + assert ours.detected_by == EventSource.SYSTEM + assert not ServiceEvent.objects.filter(pk=provider.pk).exists() + # Their post moves before their row goes. Deleting first would + # cascade the timeline away. + assert ours.updates.count() == 2 + assert {u.source for u in ours.updates.all()} == { + EventSource.SYSTEM, + EventSource.PROVIDER, + } + # Their event named a component ours did not. Losing it would drop + # that component from the card. + assert set(ours.affected_components.all()) == {component, voice} + + +def test_a_stale_provider_event_claims_nothing(): + # An incident that began a day before ours is a different outage. + # Merging them would put one card's updates on another's timeline. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Yesterday", + phase=IncidentPhase.INVESTIGATING, + starts_at=ours.starts_at - timedelta(days=1), + ) + provider.affected_components.set([component]) + + assert claim(provider, _author()) is False + assert ServiceEvent.objects.count() == 2 + + +def test_a_maintenance_window_claims_nothing(): + # A claim copies their phase onto ours. A maintenance phase on an + # incident is a combination no provider publishes. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.MAINTENANCE, + title="Planned work", + phase=MaintenancePhase.IN_PROGRESS, + starts_at=ours.starts_at, + ) + provider.affected_components.set([component]) + + assert claim(provider, _author()) is False + ours.refresh_from_db() + assert ours.phase == IncidentPhase.DETECTED + + +def test_one_of_ours_claims_nothing(): + # A claim runs on a provider's row. Ours would match itself, take + # its own id and delete the row it was folding into. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + assert claim(ours, _author()) is False + assert ServiceEvent.objects.count() == 1 + + +def test_a_claimed_event_is_not_claimed_twice(): + # The next poll finds ours by the id it took, so `update_or_create` + # updates it in place. A second claim would delete the row it just + # matched. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Outage", + phase=IncidentPhase.INVESTIGATING, + starts_at=ours.starts_at, + ) + provider.affected_components.set([component]) + claim(provider, _author()) + + ours.refresh_from_db() + assert claim(ours, _author()) is False + assert ServiceEvent.objects.count() == 1 + + +def test_archiving_a_component_closes_our_open_event(): + # We can no longer watch it recover, so the event could never + # close. A red row would stay on a board for good. + service = ServiceFactory() + component = ComponentFactory(service=service, name="SMS") + _status(component, Severity.MAJOR_OUTAGE) + reconcile_system_events(service, _author()) + + component.is_archived = True + component.save(update_fields=["is_archived"]) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + component.refresh_from_db() + assert event.phase == IncidentPhase.RESOLVED + # The end is when the provider dropped it, not when we noticed. + # Stamping now would report an outage that ran until this poll. + assert event.ends_at == component.archived_at + assert event.updates.latest("posted_at").body == "SMS is no longer published" + + +def test_an_archived_component_opens_nothing(): + # Opening one we can never watch recover writes a card that closes + # in the same pass, once per poll. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.MAJOR_OUTAGE) + component.is_archived = True + component.save(update_fields=["is_archived"]) + + reconcile_system_events(service, _author()) + + assert ServiceEvent.objects.count() == 0 + + +def test_the_nearest_start_wins_when_several_could_be_claimed(): + # Their post explains the outage that began beside it. The further + # event is another outage, and their updates would land on it. + service = ServiceFactory() + early = ComponentFactory(service=service, name="SMS") + late = ComponentFactory(service=service, name="Voice") + now = timezone.now() + _status(early, Severity.DEGRADED, started_at=now - timedelta(minutes=30)) + _status(late, Severity.DEGRADED, started_at=now - timedelta(minutes=5)) + reconcile_system_events(service, _author()) + further = ServiceEvent.objects.get(affected_components=early) + nearer = ServiceEvent.objects.get(affected_components=late) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Elevated failures", + phase=IncidentPhase.INVESTIGATING, + starts_at=now - timedelta(minutes=6), + ) + provider.affected_components.set([early, late]) + + assert claim(provider, _author()) is True + + nearer.refresh_from_db() + further.refresh_from_db() + assert nearer.external_id == "abc" + assert further.external_id is None + + +def test_a_claimed_event_stays_open_while_one_component_is_bad(): + # A claim merges their components onto ours. One recovery is no + # account of the others, and a closed card is minted again next poll. + service = ServiceFactory() + sms = ComponentFactory(service=service, name="SMS") + voice = ComponentFactory(service=service, name="Voice") + _status(sms, Severity.MAJOR_OUTAGE) + _status(voice, Severity.OPERATIONAL) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Elevated failures", + phase=IncidentPhase.INVESTIGATING, + starts_at=ours.starts_at, + ) + provider.affected_components.set([sms, voice]) + assert claim(provider, _author()) is True + + reconcile_system_events(service, _author()) + reconcile_system_events(service, _author()) + + ours.refresh_from_db() + assert ours.phase == IncidentPhase.INVESTIGATING + assert ours.ends_at is None + assert ServiceEvent.objects.count() == 1 diff --git a/bin/check_prose.py b/bin/check_prose.py index 46f913c..2497bd3 100755 --- a/bin/check_prose.py +++ b/bin/check_prose.py @@ -28,6 +28,12 @@ # A directive is not prose. DIRECTIVE = re.compile(r"^#\s*(noqa|type:|ruff:|pragma:|fmt:|mypy:|!)") CODE_ISH = re.compile(r"^\s*[\w.]+\([^)]*\)\s*$|^\s*[-*]\s|^\s{4,}\S") +# A code span, however many tokens inside, is one thing to read. +CODE_SPAN = re.compile(r"`[^`]*`") + +# The repository root, from this file's own location. A hand run from +# `api/` or `bin/` then resolves the same default as one from the root. +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent def sentences(blocks): @@ -52,7 +58,15 @@ def sentences(blocks): def words(sentence): - """Count words, ignoring anything that is code rather than prose.""" + """Count words. A code span counts as one, however many it wraps. + + A span like `external_id IS NULL` is one thing to read, not three + words to weigh. A lone backtick has no partner to pair into a + span. It falls back to the old per-token filter, instead of + raising. + """ + if sentence.count("`") % 2 == 0: + sentence = CODE_SPAN.sub("codespan", sentence) countable = [ w for w in sentence.split() @@ -159,25 +173,67 @@ def prose_problems(blocks): return problems +def default_targets(): + """The directories scanned when no path is given: api/ and bin/. + + A stdlib script checks its own prose too, not only Django's. + """ + return [REPO_ROOT / "api", REPO_ROOT / "bin"] + + +def default_paths(): + """Every `.py` file under a default target that exists.""" + return sorted( + path + for target in default_targets() + if target.is_dir() + for path in target.rglob("*.py") + ) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("paths", nargs="*", type=pathlib.Path) parser.add_argument("--limit", type=int, default=8) args = parser.parse_args() - paths = args.paths or sorted(pathlib.Path("api").rglob("*.py")) + if args.paths: + paths = args.paths + label = ", ".join(str(p) for p in paths) + else: + targets = default_targets() + if not any(target.is_dir() for target in targets): + existing = ", ".join(str(t) for t in targets) + print(f"error: no scan target exists: {existing}", file=sys.stderr) + return 1 + paths = default_paths() + label = " and ".join( + str(t.relative_to(REPO_ROOT)) for t in targets if t.is_dir() + ) + total = 0 + examined = 0 for path in paths: if path.suffix != ".py" or not path.is_file(): continue if any(part in {".venv", "__pycache__", "migrations"} for part in path.parts): continue + examined += 1 for line, rule, detail in check(path, args.limit): print(f"{path}:{line}: {rule}: {detail}") total += 1 + + # A checker that examines nothing has not passed. It has not run. + if examined == 0: + print(f"error: no files examined in {label}.", file=sys.stderr) + return 1 if total: print(f"\n{total} problems. See AGENTS.md.", file=sys.stderr) - return 1 if total else 0 + return 1 + + noun = "file" if examined == 1 else "files" + print(f"Checked {examined} {noun} in {label}. Clean.") + return 0 if __name__ == "__main__": diff --git a/bin/tests/test_check_prose.py b/bin/tests/test_check_prose.py new file mode 100644 index 0000000..c0d3708 --- /dev/null +++ b/bin/tests/test_check_prose.py @@ -0,0 +1,147 @@ +"""Behaviour tests for `bin/check_prose.py`. + +`bin/` has no dependency manager and no Django. These tests use only the +standard library, so they run on a machine that has done nothing but +`just init`. Run them with `python3 -m unittest discover -s bin/tests`. +""" + +import io +import os +import subprocess +import sys +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import check_prose + +SCRIPT = Path(check_prose.__file__).resolve() + + +class WordsTests(unittest.TestCase): + def test_a_multi_word_code_span_counts_as_one_word(self): + # The old filter dropped only the opening token. A five-word span + # leaked four words into the count instead of naming one. + with_span = check_prose.words( + "Reject the row unless `external_id IS NULL` matches" + ) + without_span = check_prose.words("Reject the row unless matches") + self.assertEqual(with_span, without_span + 1) + + def test_a_single_token_code_span_still_counts_as_one_word(self): + # The old filter dropped a lone backtick-prefixed token outright, + # so a sentence naming one field was undercounted by one. + self.assertEqual( + check_prose.words("Reject the row unless `external_id` matches"), + check_prose.words("Reject the row unless `external_id IS NULL` matches"), + ) + + def test_a_code_span_followed_by_a_comma_counts_only_the_span(self): + # A padded substitution once tore the comma into its own token, + # counting a span and its punctuation as two words. + self.assertEqual(check_prose.words("The `foo`, which is set, wins"), 6) + + def test_a_code_span_followed_by_a_full_stop_counts_only_the_span(self): + # Same bug at a sentence end. The period must not become a free + # word once the span collapses. + self.assertEqual(check_prose.words("Set by `foo`. Then read"), 5) + + def test_an_unmatched_backtick_does_not_raise(self): + # sentences() can in principle hand words() a fragment holding half + # a span. That must fall back to counting, not crash the checker. + self.assertEqual(check_prose.words("A stray ` mark stays here"), 5) + + +class ProseProblemsTests(unittest.TestCase): + def test_a_twenty_word_sentence_is_not_flagged(self): + sentence = " ".join(["word"] * 20) + self.assertFalse(check_prose.prose_problems([(sentence, 1)])) + + def test_a_twenty_one_word_sentence_is_flagged(self): + sentence = " ".join(["word"] * 21) + problems = check_prose.prose_problems([(sentence, 1)]) + self.assertEqual([rule for _line, rule, _detail in problems], ["long"]) + + +class DefaultTargetTests(unittest.TestCase): + def test_default_paths_are_the_same_from_the_repository_root_and_a_subdirectory( + self, + ): + # The default used to resolve "api" against the caller's cwd. + # A run from api/ then scanned api/api/, a handful of settings + # files, instead of api/. The default must not depend on cwd. + cwd = os.getcwd() + try: + os.chdir(check_prose.REPO_ROOT) + from_root = check_prose.default_paths() + os.chdir(check_prose.REPO_ROOT / "api") + from_subdir = check_prose.default_paths() + finally: + os.chdir(cwd) + self.assertEqual(from_root, from_subdir) + self.assertTrue(from_root, "the default target found no files at all") + + def test_a_missing_default_target_is_not_silently_skipped(self): + # A target that used to hold Python and no longer does must + # fail loudly. It must not report a clean run over nothing. + real_targets = check_prose.default_targets + real_argv = sys.argv + check_prose.default_targets = lambda: [check_prose.REPO_ROOT / "does-not-exist"] + sys.argv = ["check_prose.py"] + out, err = io.StringIO(), io.StringIO() + try: + with redirect_stdout(out), redirect_stderr(err): + status = check_prose.main() + finally: + check_prose.default_targets = real_targets + sys.argv = real_argv + self.assertNotEqual(status, 0) + self.assertIn("no scan target exists", err.getvalue()) + + +class MainExitStatusTests(unittest.TestCase): + """Run the real script as a subprocess: main()'s exit code is the contract.""" + + def run_script(self, *args, cwd=None): + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + capture_output=True, + text=True, + cwd=cwd, + check=False, + ) + + def test_a_clean_default_run_exits_zero_and_names_what_it_scanned(self): + result = self.run_script(cwd=str(check_prose.REPO_ROOT)) + self.assertEqual(result.returncode, 0, result.stderr) + # Silence on success is what let two runs from api/ report clean + # while checking a handful of settings files. The count must show. + self.assertIn("Checked", result.stdout) + self.assertIn("api", result.stdout) + self.assertIn("bin", result.stdout) + + def test_the_default_run_is_identical_from_the_repository_root_and_bin(self): + from_root = self.run_script(cwd=str(check_prose.REPO_ROOT)) + from_bin = self.run_script(cwd=str(check_prose.REPO_ROOT / "bin")) + self.assertEqual(from_root.returncode, 0, from_root.stderr) + self.assertEqual(from_root.stdout, from_bin.stdout) + + def test_an_explicit_path_matching_no_file_exits_non_zero(self): + # A caller who passes a path that matches nothing has the same + # silent-pass problem as a broken default. Treated the same way. + result = self.run_script("does/not/exist.py", cwd=str(check_prose.REPO_ROOT)) + self.assertNotEqual(result.returncode, 0) + self.assertIn("no files examined", result.stderr) + + def test_an_explicit_directory_with_no_python_files_exits_non_zero(self): + # check() only ever receives files listed directly in args.paths, + # so a directory argument matches zero files. That must error, + # not report a clean run. + result = self.run_script("docs", cwd=str(check_prose.REPO_ROOT)) + self.assertNotEqual(result.returncode, 0) + self.assertIn("no files examined", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/bin/tests/test_merge_gate.py b/bin/tests/test_merge_gate.py new file mode 100644 index 0000000..ad7e2a0 --- /dev/null +++ b/bin/tests/test_merge_gate.py @@ -0,0 +1,98 @@ +"""Every check `just check` runs must also run in CI. + +Not the reverse. CI also scans the image for CVEs, which needs the +network and a vulnerability database. So a green `just check` is not a +green merge gate, and the justfile says so. + +`bin/` has no dependency manager and no Django. These tests use only the +standard library, so they run on a machine that has done nothing but +`just init`. Run them with `python3 -m unittest discover -s bin/tests`. +""" + +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +JUSTFILE = ROOT / "justfile" +WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" + + +def recipe(name): + """One recipe's commands, with a line continuation joined up.""" + lines = [] + inside = False + for line in JUSTFILE.read_text().splitlines(): + if re.match(rf"^{re.escape(name)}(\s|:)", line): + inside = True + continue + if not inside: + continue + if line and not line[0].isspace(): + break + body = line.strip() + if not body or body.startswith("#"): + continue + if lines and lines[-1].endswith("\\"): + lines[-1] = lines[-1][:-1] + body + else: + lines.append(body) + return lines + + +def commands(): + """Every command `just check` runs, with a called recipe inlined.""" + for line in recipe("check"): + called = re.match(r"@?just\s+([\w-]+)", line) + yield from recipe(called.group(1)) if called else [line] + + +def compose_services(command): + """The service a `docker compose run` names, past its flags.""" + _, _, tail = command.partition("run --rm ") + tokens = tail.split() + while tokens: + token = tokens.pop(0) + if token == "--entrypoint": + tokens.pop(0) + elif not token.startswith("-"): + return [token] + return [] + + +class MergeGateTests(unittest.TestCase): + """CI is the gate a merge waits on. `just check` is not. + + The bin/ tests ran in `just check` and in nothing CI did. So they + gated nothing, which is how they came to run only on a laptop. + """ + + def setUp(self): + self.workflow = WORKFLOW.read_text() + # Teardown, not a check. It proves nothing about the code. + self.commands = [c for c in commands() if "down -v" not in c] + + def test_every_container_check_also_runs_in_ci(self): + for command in self.commands: + for service in compose_services(command): + self.assertIn( + f"run --rm {service}", self.workflow, f"CI skips {service}" + ) + + def test_every_host_check_also_runs_in_ci(self): + for command in self.commands: + if "docker compose" in command: + continue + for script in re.findall(r"bin/[\w./-]+", command): + self.assertIn(script, self.workflow, f"CI skips {script}") + + def test_the_migration_check_also_runs_in_ci(self): + # It runs through an entrypoint override, so the service name + # alone does not say the check itself is there. + wanted = [c for c in self.commands if "makemigrations" in c] + self.assertTrue(wanted, "just check no longer checks for a missing migration") + self.assertIn("makemigrations --check --dry-run", self.workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index da3ec24..710f98f 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -13,6 +13,23 @@ info: To track a service, track its overall component. To track one part of a service, track that component. Both are the same call with a different id. + ## An archived component is not served + + A provider stops publishing a component. The row is archived rather than deleted, because + somebody may be tracking it. + + Archived components are left out of every list, every count and the detail. The detail + answers `404`, and so does tracking one. + + The one exception is `Service.overall_component`. A service whose rollup is archived still + needs a status in its header. + + Three consequences follow. A tracked component that is archived leaves the board with no + notice. An event names the components it affected, and a link to an archived one answers + `404`. `Component.path` lists the ancestors that are served, so a breadcrumb can skip a + level, and its last node is not always `Component.parent`. `parent` is the raw column and + still names an archived row, so fetch a breadcrumb from `path`. + ## Severity is the state `severity` is an integer from 0 to 5. Lower is worse. This matches the SEV0/SEV1 convention. @@ -41,7 +58,7 @@ info: Do not derive one from the other. - ## One schema, one model + ## One schema, one model, usually Every schema maps to one Django model, and every nested key is a real relation a `ModelSerializer` can traverse. Nest a related model under a key. Do not flatten it into the @@ -54,8 +71,8 @@ info: | `Service.overall_component` | the `ServiceComponent` where `is_overall` | | `Component.status` | the `ComponentStatus` row where `ended_at` is null | | `Component.active_incident` | newest unresolved `ServiceEvent` with `kind=incident`, across the reverse M2M | - | `ServiceEvent.updates` | reverse FK from `EventUpdate` | | `Component.service` | `ForeignKey` | + | `ServiceEvent.service` | `ForeignKey` | `status` is its own object because `ComponentStatus` is its own table. A `ModelSerializer` with a nested serializer produces these shapes directly. @@ -63,24 +80,33 @@ info: The rule applies transitively. A `Service` nests its overall component. It does not copy that component's status. - **One model, one schema.** There is no list variant and detail variant of the same thing. - A second schema exists only to withhold fields, and every field here is either a column on the - row or a `select_related` join — nothing a variant would save is expensive. `?fields=` lets a - client ask for a smaller shape, and unlike a schema variant the client chooses. + **A second schema never exists to withhold fields.** `?fields=` does that, and the client + chooses. One exists only to layer a value too expensive to compute for every row in a list. - `allOf` therefore appears only where two schemas cover two models. + `allOf` appears where two schemas cover two models. It also appears where a detail schema + layers that kind of value onto its own list row. - Query parameters use the ORM path. django-filter generates them from `filterset_fields` and - `ordering_fields`. No `FilterSet` class is necessary. + Query parameters are declared in a `FilterSet`. Almost none is generated from a column of + the same name. `is_overall` and `kind` are the two that are. `q` is no filter at all. + `ComponentListView.filter_queryset` narrows on it before the backends run. - The nesting is visible in the URL. Read a field's path in the response body to build its - filter. This holds for fields added later. + So a parameter is not an ORM path, and a field's path in the response body does not build + its filter. `service` takes a slug where the body carries a nested `ServiceRef`. `component` + and `ancestor` name nothing in any body. `phase` takes `open` or `closed` where the body + carries `investigating`. - Three parameters are declared, not generated. They have no ORM path: - `overall_component__is_tracked`, `ordering=suggested`, and `ordering=next_transition`. + Read the operation's own parameter list instead. It says what each one takes and what it + narrows. - Model field names are chosen to read well in a URL, because a field name appears in `?fields=` - and in a filter path. + Eleven narrow the three collections: `q`, `service`, `ancestor`, `event`, `component`, + `dashboard`, `is_tracked`, `phase`, and the three `status__severity` forms. The board list + takes those three severity forms and nothing else. + + Two ordering values are declared as well, `suggested` and `next_transition`. Neither names a + column. A view annotates the fields each one sorts on. + + Model field names are chosen to read well in a URL, because a field name appears in + `?fields=`. ## Conventions @@ -93,6 +119,9 @@ info: Lists return `{ aggregates, next, results }`. `aggregates` covers the whole collection, not the page. `ordering` follows DRF. Prefix a field with `-` to reverse it. GETs are ETagged. + An `ordering` name the list does not declare answers `400`, as an unknown `fields` name + does. `suggested` names its own directions, so `-suggested` answers `400` as well. + ## Identity Every resource has a UUID. `Service` also has a stable `slug`. Public catalog routes use the @@ -104,7 +133,9 @@ tags: - name: auth - name: me - name: catalog - description: Public. Services, their components, and incidents. + description: Public. Services and their components. + - name: status + description: Public. The event feed, one collection for Home, a service and a component. - name: board paths: @@ -134,6 +165,13 @@ paths: schema: { type: object, required: [email], properties: { email: { type: string, format: email } } } responses: '202': { description: Accepted } + '400': + description: | + No email address in the body. A bare `detail` string, hand rolled here — not the + `Error` shape, so there is no `code` to branch on. + content: + application/json: + schema: { type: object, properties: { detail: { type: string } } } '429': { $ref: '#/components/responses/Throttled' } /auth/verify/: post: @@ -163,7 +201,10 @@ paths: post: tags: [auth] summary: Blacklist the refresh token - responses: { '204': { description: No content } } + responses: + '200': + description: Blacklisted. The body is an empty object. + content: { application/json: { schema: { type: object } } } /me/: get: @@ -182,39 +223,6 @@ paths: description: Removes the dashboard, its tracked components and the user. Catalog and status data are untouched — they are not personal. responses: { '204': { description: No content } } - /catalog/services/: - get: - tags: [catalog] - summary: List services - description: | - This endpoint serves Discover, the suggested lists, and the public "down right now" view. - Use parameters to select the view. - - With `q`, results are ordered by relevance. Without `q`, the default ordering is - `suggested`: `(-is_featured, severity, -watcher_count, name)`. A broken service outranks a - more popular service that is working. - security: [{}, { bearer: [] }] - parameters: - - { name: q, in: query, schema: { type: string } } - - { name: overall_component__status__severity__lte, in: query, schema: { type: integer, minimum: 0, maximum: 5 }, description: 'A service''s severity is its overall component''s, so the path runs through it.' } - - { name: overall_component__status__severity, in: query, schema: { type: integer, minimum: 0, maximum: 5 }, description: 'Maintenance chip is =4' } - - { name: overall_component__is_tracked, in: query, schema: { type: boolean }, description: 'Authenticated only. Narrower: whether the overall component specifically is tracked, which is what the ＋ on a service row toggles.' } - - { name: ordering, in: query, schema: { type: string, enum: [suggested, overall_component__status__severity, -overall_component__status__severity, name, -name, updated_at, -updated_at], default: suggested } } - - { $ref: '#/components/parameters/PageSize' } - - { $ref: '#/components/parameters/Cursor' } - - { $ref: '#/components/parameters/Fields' } - responses: - '200': - description: OK - content: - application/json: - schema: - allOf: - - $ref: '#/components/schemas/Envelope' - - type: object - properties: - aggregates: { $ref: '#/components/schemas/StatusAggregates' } - results: { type: array, items: { $ref: '#/components/schemas/Service' } } /catalog/import/: post: tags: [catalog] @@ -242,37 +250,84 @@ paths: '201': description: Service created content: { application/json: { schema: { $ref: '#/components/schemas/Service' } } } - '422': - description: No supported status page found at that URL + '400': + description: | + Two failures share this code. A missing or malformed `status_page_url` answers a + field-errors object, one array of messages per field name. A URL no adapter can + read answers the `Error` shape with `code: no_status_page_found`. Branch on + whether `code` is present. + content: + application/json: + schema: + oneOf: + - type: object + additionalProperties: { type: array, items: { type: string } } + - $ref: '#/components/schemas/Error' + '429': { $ref: '#/components/responses/Throttled' } + '502': + description: | + The provider did not answer. `code: provider_unreachable`. The URL may still be a + status page, so this is worth retrying and `400` is not. content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } - - /catalog/services/{slug}/: - get: + /catalog/requests/: + post: tags: [catalog] - summary: Service detail + summary: Ask us for a status page we could not read + description: | + The Add-by-URL not-found screen. Always `202`, whether or not the catalog already + holds the URL, so asking cannot reveal what we have. + + One row per URL, counted. Throttled per caller. + + Signing in is optional. A signed-in asker is recorded on the row. security: [{}, { bearer: [] }] - parameters: - - { $ref: '#/components/parameters/Slug' } - - { $ref: '#/components/parameters/Fields' } + requestBody: + required: true + content: + application/json: + schema: { type: object, required: [url], properties: { url: { type: string, format: uri } } } responses: - '200': { description: OK, content: { application/json: { schema: { $ref: '#/components/schemas/Service' } } } } - '404': { $ref: '#/components/responses/Error' } + '202': { description: Recorded } + '400': + description: | + Missing or malformed url. A field-errors object, one array of messages per + field name — not the `Error` shape, which this response never raises. + content: + application/json: + schema: + type: object + additionalProperties: { type: array, items: { type: string } } + '429': { $ref: '#/components/responses/Throttled' } - /catalog/services/{slug}/components/: + /catalog/components/: get: tags: [catalog] - summary: List a service's components + summary: Search and list components description: | - The component tree, flattened. `parent` links a child to its group. The overall component - is first. `aggregates` gives the tab labels. + One collection, four screens. Discover with no parameters, a service's parts with + `service`, a component's descendants with `ancestor`, an event's affected list with + `event`. + + `q` matches part of a component's name, its service's name, or its parent's. So `twilio` + finds `SMS`, and so does `messaging`. The parent is one level: a grandparent's name does + not reach it. Several words are an AND. The service rollup leads the results. Without `q` + the default is `suggested`, except under `service`, where it is `status_page_order`. + + Archived components are never listed. An event whose only affected component was archived + returns an empty page. security: [{}, { bearer: [] }] parameters: - - { $ref: '#/components/parameters/Slug' } - - { name: is_tracked, in: query, schema: { type: boolean }, description: Drives Showing All / Tracked / Untracked } - - { name: status__severity__lte, in: query, schema: { type: integer } } - - { name: status__severity, in: query, schema: { type: integer } } - - { name: ordering, in: query, schema: { type: string, enum: [status_page_order, status__severity, -status__severity, name, -name], default: status_page_order }, description: '`status_page_order` keeps the order the status page lists them in.' } + - { name: q, in: query, schema: { type: string } } + - { name: service, in: query, schema: { type: string }, description: Service slug } + - { name: ancestor, in: query, schema: { type: string, format: uuid }, description: Every descendant, at any depth } + - { name: event, in: query, schema: { type: string, format: uuid }, description: The components this event affects } + - { name: is_overall, in: query, schema: { type: boolean }, description: '`true` is one row per service, which is signed-out Home.' } + - { name: is_tracked, in: query, schema: { type: boolean }, description: 'Authenticated only. Drives Showing All / Tracked / Untracked.' } + - { name: status__severity, in: query, schema: { type: integer, minimum: 0, maximum: 5 } } + - { name: status__severity__lte, in: query, schema: { type: integer, minimum: 0, maximum: 5 } } + - { name: status__severity__in, in: query, schema: { type: string }, description: 'Comma separated, for a Severity filter offering every value at once.' } + - { name: ordering, in: query, schema: { type: string, enum: [suggested, status_page_order, -status_page_order, status__severity, -status__severity, name, -name, updated_at, -updated_at], default: suggested } } - { $ref: '#/components/parameters/PageSize' } - { $ref: '#/components/parameters/Cursor' } - { $ref: '#/components/parameters/Fields' } @@ -288,34 +343,56 @@ paths: properties: aggregates: { $ref: '#/components/schemas/StatusAggregates' } results: { type: array, items: { $ref: '#/components/schemas/Component' } } + '400': { $ref: '#/components/responses/RejectedParameter' } - /catalog/services/{slug}/events/: + /catalog/components/{uuid}/: get: tags: [catalog] - summary: List a service's events + summary: Component detail description: | - Incidents and maintenance are one collection. Filter with `kind`. + `404` for an archived component, as well as for one that never existed. A provider + stopped publishing it, so no screen holds it. + security: [{}, { bearer: [] }] + parameters: + - { $ref: '#/components/parameters/Uuid' } + - { $ref: '#/components/parameters/Fields' } + responses: + '200': { description: OK, content: { application/json: { schema: { $ref: '#/components/schemas/Component' } } } } + '404': { $ref: '#/components/responses/Error' } - The Incidents tab passes `kind=incident&ends_at__isnull=true`. The Maintenance tab passes - `kind=maintenance` with a window filter. + /catalog/services/{slug}/: + get: + tags: [catalog] + summary: Service detail security: [{}, { bearer: [] }] parameters: - { $ref: '#/components/parameters/Slug' } - - { name: kind, in: query, schema: { type: string, enum: [incident, maintenance] }, description: Omit for both. } - - { name: phase, in: query, schema: { type: string }, description: 'Valid values depend on `kind`. See `/meta/` `enums.event_phase`.' } - - { name: phase__in, in: query, schema: { type: string }, description: Comma-separated. } - - { name: ends_at__isnull, in: query, schema: { type: boolean }, description: '`true` is the open set — unresolved, or a window still to close.' } - - { name: starts_at__gte, in: query, schema: { type: string, format: date-time } } - - { name: starts_at__lte, in: query, schema: { type: string, format: date-time }, description: 'With `ends_at__gte`, answers "what happens between now and Thursday".' } - - { name: ends_at__gte, in: query, schema: { type: string, format: date-time } } - # The M2M filter stays although no field exposes it. django-filter generates it free, - # and narrowing events to one component is a query a screen may want. The field was - # different: a prefetch and bytes on every row that nothing rendered. - - { name: affected_components, in: query, schema: { type: string, format: uuid }, description: Events naming this component. } + - { $ref: '#/components/parameters/Fields' } + responses: + '200': { description: OK, content: { application/json: { schema: { $ref: '#/components/schemas/Service' } } } } + '404': { $ref: '#/components/responses/Error' } + + /events/: + get: + tags: [status] + summary: The event feed + description: | + One feed, three scopes. Home with `dashboard`, a service with `service`, a component + with `component`. + + `phase` takes `open` or `closed`. The line between them is `CLOSED_PHASES` in the + code, so a client never restates which phases are terminal. + security: [{}, { bearer: [] }] + parameters: + - { name: dashboard, in: query, schema: { type: string, format: uuid }, description: Requires auth, and must be your own } + - { name: service, in: query, schema: { type: string }, description: Service slug } + - { name: component, in: query, schema: { type: string, format: uuid } } + - { name: kind, in: query, schema: { type: string, enum: [incident, maintenance] } } + - { name: phase, in: query, schema: { type: string, enum: [open, closed] }, description: 'Omit it and both come back. Render the label from `/meta/` `enums.event_phase_state`.' } - { name: ordering, in: query, schema: { type: string, enum: [-starts_at, starts_at, ends_at, -ends_at], default: -starts_at } } - - { $ref: '#/components/parameters/PageSize' } - - { $ref: '#/components/parameters/Cursor' } - { $ref: '#/components/parameters/Fields' } + - { $ref: '#/components/parameters/Cursor' } + - { $ref: '#/components/parameters/PageSize' } responses: '200': description: OK @@ -328,35 +405,64 @@ paths: properties: aggregates: { $ref: '#/components/schemas/EventAggregates' } results: { type: array, items: { $ref: '#/components/schemas/ServiceEvent' } } + '400': { $ref: '#/components/responses/RejectedParameter' } + + /events/{uuid}/: + get: + tags: [status] + summary: One event + security: [] + parameters: + - { $ref: '#/components/parameters/Uuid' } + - { $ref: '#/components/parameters/Fields' } + responses: + '200': { description: OK, content: { application/json: { schema: { $ref: '#/components/schemas/ServiceEventDetail' } } } } + '404': { $ref: '#/components/responses/Error' } + /events/{uuid}/updates/: + get: + tags: [status] + summary: One event's log, oldest first + description: Its own operation because a provider's log has no ceiling. + security: [] + parameters: + - { $ref: '#/components/parameters/Uuid' } + - { $ref: '#/components/parameters/Fields' } + - { $ref: '#/components/parameters/Cursor' } + - { $ref: '#/components/parameters/PageSize' } + responses: + '200': + description: OK + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/Envelope' + - type: object + properties: + aggregates: { $ref: '#/components/schemas/Aggregates' } + results: { type: array, items: { $ref: '#/components/schemas/EventUpdate' } } /dashboards/{uuid}/components/: get: tags: [board] summary: The components tracked on this dashboard - description: Filtering, sorting and pagination are server-side without exception — nothing assumes a board is small. + description: | + Filtering, sorting and pagination are server-side without exception. Nothing assumes a + board is small. + + An archived component drops off the board. The item stays tracked, so the row returns if + the provider publishes the component again. parameters: - { $ref: '#/components/parameters/Uuid' } - - name: event - in: query - schema: { type: string, enum: [incident, maintenance] } - description: | - The Incidents and Maintenance tabs. Keep a row if the component has an open - `ServiceEvent` of this kind. Omit the parameter for the All tab. - - Declared, not generated. `ServiceEvent` and `ServiceComponent` are many-to-many, and - two generated parameters would apply as two `.filter()` calls. On a many-to-many each - call joins the relation again, so `events__kind` and `events__phase` can be satisfied - by two different events: a component with one closed incident and one running - maintenance would match `event=incident`. One `.filter()` holding both conditions - binds them to the same row. - - { name: status__severity__lte, in: query, schema: { type: integer }, description: 'Severity, not events. It works inside any tab — `event=incident&status__severity__lte=3` is written-up incidents that are also down now.' } + - { name: status__severity__lte, in: query, schema: { type: integer } } - { name: status__severity, in: query, schema: { type: integer } } + - { name: status__severity__in, in: query, schema: { type: string }, description: 'Comma separated, for a Severity filter offering every value at once.' } - name: ordering in: query schema: { type: string, enum: [status__severity, -status__severity, name, -name, updated_at, -updated_at, next_transition], default: status__severity } description: | - `next_transition` sorts the Maintenance tab. It uses `ends_at` for a running window - and `starts_at` for one that has not started. Running windows sort first. This + `next_transition` sorts by the next maintenance step. It uses `ends_at` for a running + window and `starts_at` for one that has not started. Running windows sort first. This ordering is declared, not generated. - { $ref: '#/components/parameters/PageSize' } - { $ref: '#/components/parameters/Cursor' } @@ -373,6 +479,7 @@ paths: properties: aggregates: { $ref: '#/components/schemas/StatusAggregates' } results: { type: array, items: { $ref: '#/components/schemas/Component' } } + '400': { $ref: '#/components/responses/RejectedParameter' } post: tags: [board] summary: Track a component @@ -387,7 +494,8 @@ paths: schema: { type: object, required: [component_id], properties: { component_id: { type: string, format: uuid } } } responses: '201': { description: Created, content: { application/json: { schema: { $ref: '#/components/schemas/Component' } } } } - '409': { description: Already tracked } + '200': { description: 'Already tracked. Tracking twice is not an error.', content: { application/json: { schema: { $ref: '#/components/schemas/Component' } } } } + '404': { description: 'No such board, or no such component. An archived component is not tracked.' } /dashboards/{uuid}/components/{component_id}/: delete: @@ -418,7 +526,11 @@ components: Use a dotted path for a nested field. `overall_component.status.severity` returns `overall_component` containing `status` containing `severity` only. - A path that names an unknown field returns `400`. + A dotted path reaches into `enums` on `/meta/` too. `enums.severity` returns that one + label map. `enums` alone returns all of them. + + A path that names an unknown field returns `400`. The body is a field-errors object, + one array of messages under `fields` — not the `Error` shape. PageSize: { name: page_size, in: query, schema: { type: integer, default: 50, minimum: 1, maximum: 200 } } Slug: { name: slug, in: path, required: true, schema: { type: string }, example: twilio } Uuid: { name: uuid, in: path, required: true, schema: { type: string, format: uuid } } @@ -430,6 +542,19 @@ components: Error: description: Error content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } + RejectedParameter: + description: | + A query parameter names something the list does not serve, or carries a value it + cannot read. `fields`, `ordering` and every filter refuse the same way, so a client + branches once. + + The body is a field-errors object, one array of messages keyed by the parameter + name. It is not the `Error` shape. + content: + application/json: + schema: + type: object + additionalProperties: { type: array, items: { type: string } } schemas: Error: type: object @@ -485,7 +610,18 @@ components: additionalProperties: { type: string } description: | Both sets, keyed by kind: - `{"incident": {"investigating": "Investigating", …}, "maintenance": {"scheduled": "Scheduled", …}}` + `{"incident": {"detected": "Detected", …}, "maintenance": {"scheduled": "Scheduled", …}}` + event_phase_state: + type: object + additionalProperties: { type: string } + description: | + What the `phase` filter on `/events/` takes: + `{"open": "Open", "closed": "Closed"}`. Which phases are terminal stays on + the server, so a client labels the control from here. + event_source: + type: object + additionalProperties: { type: string } + description: '{"provider": "Provider", "system": "Statusboard"}' TokenPair: type: object properties: @@ -535,23 +671,13 @@ components: allOf: - $ref: '#/components/schemas/Aggregates' - type: object - description: For any list of things that carry a status — services, components, a board. + description: For any list of components. Discover, a service's parts, a board. properties: by_severity: type: object additionalProperties: { type: integer } description: | Keys are severities 0–5. Sorting and the dot read from this. No tab is a severity. - by_event_kind: - type: object - additionalProperties: { type: integer } - description: | - `{"incident": 3, "maintenance": 1}` — how many rows each value of `event` returns. - These are the Incidents and Maintenance tab counts. - - Neither can be read off `by_severity`. A maintenance window three days out leaves - the component at severity 5, and a provider can open an incident on components it - still reports as operational. next_refresh_at: type: string format: date-time @@ -721,10 +847,13 @@ components: rather than a copy of its fields. One FK, one nested serializer. It is a plain `Component`, the same shape a component has anywhere else, so one - renderer draws every row. Six of its fields are fixed by what an overall component is: - `parent` and `path` are null, `child_count` is 0, `is_overall` is true, `archived_at` - is null, and `name` is the service name. They are returned rather than omitted — a - client that special-cased them would need a second renderer to gain nothing. + renderer draws every row. Five of its fields follow from what an overall component is: + `parent` is null, `path` is empty, `descendant_count` is 0, `is_overall` is true, and + `name` is the service name. They are returned rather than omitted. A client that + special-cased them would need a second renderer to gain nothing. + + This is the one place an archived component is served. A provider can stop publishing + its top-level indicator, and the header still needs a status. Read `archived_at` here. Read severity at `overall_component.status.severity`. The filter uses the same path. tracked_component_count: @@ -735,10 +864,6 @@ components: A value above 0 means the service is on your board. This differs from `overall_component.is_tracked`, which covers the overall component only. - description: - type: string - nullable: true - description: The provider's own summary of the service. The About tab shows it. in_catalog_since: type: string format: date @@ -769,20 +894,25 @@ components: path: type: array description: | - Every component above this one, top down. Each step carries its own id, so a client - builds a breadcrumb it can click. Empty on the overall component, which is under - nothing. + Every component above this one that is served, top down. Each step carries its own + id, so a client builds a breadcrumb it can click and every node here is fetchable. + Empty on the overall component, which is under nothing. + + An archived ancestor is left out. The chain is then shorter than the real tree, and + its last node is not `parent`. It is never broken, so nothing needs a placeholder. items: { $ref: '#/components/schemas/PathNode' } parent: { type: string, format: uuid, nullable: true } - child_count: + descendant_count: type: integer description: | - A component is a group when this is above zero. There is no is_group flag. + How many components sit anywhere under this one. A component is a group when this + is above zero. There is no is_group flag. - Always 0 on the overall component. It is a peer of the top-level components, not their - parent. + Always 0 on the overall component. It is a peer of the top-level components, not + their parent. A database trigger refuses a component parented under it. The count + is a guarantee, not just what the data happens to hold. is_overall: { type: boolean, description: The component Statusboard creates per service, carrying the provider's top-level indicator. Always first, parent is null. } - archived_at: { type: string, format: date-time, nullable: true, description: Set when the provider stops publishing it. The row stays, reads unknown, and says so — deleting it would silently drop it from someone's board. } + archived_at: { type: string, format: date-time, nullable: true, description: 'When the provider stopped publishing the component. An archived component is not served, so this is null everywhere but `Service.overall_component`.' } status: { $ref: '#/components/schemas/Status' } upcoming_maintenance: allOf: [{ $ref: '#/components/schemas/EventRef' }] @@ -814,7 +944,8 @@ components: The full list is not on the component. History is unbounded, and a row renders from the soonest plus this count. Read the list from - `GET /catalog/services/{slug}/events/?kind=maintenance` when a screen needs it. + `GET /events/?component={uuid}&kind=maintenance&phase=open` when a screen needs it. + Without `phase=open` that call returns every window ever, which this never counts. active_incident: allOf: [{ $ref: '#/components/schemas/EventRef' }] description: | @@ -874,20 +1005,44 @@ components: phase: type: string description: | - `investigating, identified, monitoring, resolved` for an incident. + `detected, investigating, identified, monitoring, resolved` for an incident. `scheduled, in_progress, verifying, completed` for maintenance. Render the label from `/meta/` `enums.event_phase`, keyed by `kind`. starts_at: { type: string, format: date-time, description: When it began, or when the window opens. } ends_at: { type: string, format: date-time, nullable: true, description: When it resolved, or when the window closes. } - updates: - type: array - items: - type: object - properties: - phase: { type: string } - body: { type: string } - posted_at: { type: string, format: date-time } + detected_by: + type: string + enum: [provider, system] + description: | + Who opened it: the provider's own page, or this deployment noticing a severity drop + first. Never rewritten by a later claim. Render the label from `/meta/` + `enums.event_source`. + service: { $ref: '#/components/schemas/ServiceRef' } + ServiceEventDetail: + allOf: + - $ref: '#/components/schemas/ServiceEvent' + - type: object + description: | + The event screen's header. Both counts are here because the header draws both tab + badges before either tab has made a request. + properties: + update_count: { type: integer } + affected_count: { type: integer, description: 'The Affects badge. Archived components are left out, the same as the tab it labels. It can be 0.' } + last_update_at: { type: string, format: date-time, nullable: true } + EventUpdate: + type: object + description: One posted update within an event's timeline. + properties: + phase: { type: string } + body: { type: string } + posted_at: { type: string, format: date-time } + source: + type: string + enum: [provider, system] + description: | + Who wrote this update. `system` is a severity change we recorded ourselves. + Render the label from `/meta/` `enums.event_source`. ServiceRef: type: object properties: diff --git a/docs/check_docs.py b/docs/check_docs.py index d77920e..5dd4342 100755 --- a/docs/check_docs.py +++ b/docs/check_docs.py @@ -34,7 +34,9 @@ "PathNode": "ServiceComponent", "Status": "ComponentStatus", "ServiceEvent": "ServiceEvent", + "ServiceEventDetail": "ServiceEvent", "EventRef": "ServiceEvent", + "EventUpdate": "EventUpdate", "Me": "User", } PLAIN = { @@ -60,16 +62,18 @@ "Poller.cooldown_seconds": "own override or /meta/ default", "Service.poller": "OneToOneField", "Component.path": "walk of parent", - "Component.child_count": "Count of reverse parent", + "Component.descendant_count": "recursive walk down parent, archived rows left out", "Component.is_tracked": "per-user annotation over DashboardItem", "Component.upcoming_maintenance": "soonest unfinished maintenance event across that M2M", "Component.upcoming_maintenance_count": "Count of unfinished maintenance across that M2M", "Component.active_incident": "newest unresolved ServiceEvent kind=incident across that M2M", "Component.active_incident_count": "Count of unresolved across that M2M", - "ServiceEvent.updates": "reverse FK from EventUpdate", "Me.default_dashboard_id": "Dashboard where is_default", "Service.in_catalog_since": "BaseModel.created_at", "Component.service": "ForeignKey", + "ServiceEventDetail.update_count": "Count of EventUpdate for this event", + "ServiceEventDetail.affected_count": "Count of affected_components for this event", + "ServiceEventDetail.last_update_at": "newest EventUpdate.posted_at for this event", } INHERITED = {"id", "created_at", "updated_at", "created_by", "updated_by"} diff --git a/docs/superpowers/plans/2026-08-26-statusboard-backend.md b/docs/plans/2026-08-26-statusboard-backend.md similarity index 99% rename from docs/superpowers/plans/2026-08-26-statusboard-backend.md rename to docs/plans/2026-08-26-statusboard-backend.md index 3ca2ffc..3f9271f 100644 --- a/docs/superpowers/plans/2026-08-26-statusboard-backend.md +++ b/docs/plans/2026-08-26-statusboard-backend.md @@ -8,8 +8,8 @@ **Tech Stack:** Django 5, DRF, SimpleJWT, django-filter, drf-spectacular, django-unfold, django-celery-beat, Celery + Redis, Postgres 17, uv, ruff, pytest + pytest-django + factory_boy. -**Spec:** [`docs/specs/2026-08-23-statusboard-design.md`](../../specs/2026-08-23-statusboard-design.md) -**Contract:** [`docs/api/openapi.yaml`](../../api/openapi.yaml) +**Spec:** [`docs/specs/2026-08-23-statusboard-design.md`](../specs/2026-08-23-statusboard-design.md) +**Contract:** [`docs/api/openapi.yaml`](../api/openapi.yaml) ## Scope @@ -5168,14 +5168,3 @@ positional arguments in Tasks 14 and 17 and defined that way in Task 13. `detect class in Tasks 9, 14 and 17. `NormalisedComponent` field names match between Tasks 9, 10, 12, 13 and 17. `ComponentSerializer` field list matches the contract's `Component` properties, asserted by Task 18. - -## Execution Handoff - -Plan complete and saved to `docs/superpowers/plans/2026-08-26-statusboard-backend.md`. Two -execution options: - -**1. Subagent-Driven (recommended)** — a fresh subagent per task, review between tasks, fast -iteration. - -**2. Inline Execution** — execute tasks in this session using executing-plans, batch execution -with checkpoints for review. diff --git a/docs/plans/2026-09-03-statusboard-api.md b/docs/plans/2026-09-03-statusboard-api.md new file mode 100644 index 0000000..401452f --- /dev/null +++ b/docs/plans/2026-09-03-statusboard-api.md @@ -0,0 +1,3200 @@ +# Statusboard API Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Serve the six endpoints the approved designs need, and delete the three they replaced. + +**Architecture:** Every trackable thing is a `ServiceComponent`, so one component collection +replaces the service list and both nested service routes. A severity change nobody explains +becomes a `ServiceEvent` we write ourselves, so the Updates feed reads one table. Ancestry +moves into a stored array, which serves descendant queries, breadcrumbs and path search at once. + +**Tech Stack:** Django 5, DRF, django-filter, drf-spectacular, Postgres 17 full-text search, +pytest, factory_boy. + +**Spec:** `docs/specs/2026-09-03-statusboard-client-design.md` + +## Global Constraints + +- Python 3.14. Primary keys are `uuid.uuid7`, which does not exist earlier. +- Every model extends `common.models.BaseModel`: UUID pk, `created_at`, `updated_at`, + `created_by`, `updated_by`, `ordering = ["-created_at"]`. +- Every behaviour change carries a test. The suite runs with no network: `tests/conftest.py` + blocks `socket.socket.connect`. +- Settings split by who named them. Django or a package named it, `api/settings.py`. We named + it, `api/defaults.py`. +- Comments and docstrings follow ASD-STE100: one idea per sentence, under 20 words, active + voice, present tense, no em dashes. A comment says why, never what the line already says. +- A comment on a test says what would break if the assertion failed. +- One definition, many readers. A filter, a threshold or a set of rules lives in one place. +- An invariant the code depends on belongs in a database constraint. +- Commits use a lowercase conventional prefix, then a declarative clause. The body says why. +- `docs/api/openapi.yaml` is the contract. `api/tests/test_contract.py` fails when the code + and the contract disagree, in either direction. Both move in the same commit. +- `docs/check_docs.py` reads the field-level mermaid ER diagram in section 3 of + `docs/specs/2026-08-23-statusboard-design.md`. Every new column lands there too. +- Run `just test` for the suite and `just lint` before every commit. + +--- + +## File Structure + +**Created** + +| File | Responsibility | +| --- | --- | +| `api/catalog/views_components.py` | The component collection and its detail | +| `api/status/views.py` | The event feed, an event, and its update log | +| `api/status/filters.py` | What a caller narrows `/events/` by | +| `api/polling/system_events.py` | Opening, updating, closing and claiming a system event | +| `api/tests/test_component_api.py` | The component collection | +| `api/tests/test_event_api.py` | The event feed and an event | +| `api/tests/test_system_events.py` | The reconcile writer | +| `api/tests/test_service_requests.py` | `POST /catalog/requests/` | + +**Modified** + +| File | Change | +| --- | --- | +| `api/catalog/models.py` | `ServiceComponent.is_featured`, `ancestor_ids`, `search_document`, `descendant_count`. `Service.is_featured` and `description` go. | +| `api/catalog/queries.py` | `COMPONENT_WATCHER_COUNT` in, `WATCHER_COUNT` out | +| `api/catalog/filters.py` | `ComponentFilter` grows. `ServiceFilter` goes. | +| `api/catalog/serializers.py` | `ComponentSerializer.descendant_count`. `Service.description` out. | +| `api/catalog/views.py` | `ServiceViewSet` becomes retrieve only | +| `api/catalog/urls.py` | The component routes and the requests route | +| `api/catalog/admin.py` | Featuring, watchers, service requests | +| `api/status/models.py` | `ServiceEvent.detected_by`, nullable `external_id`, `EventUpdate.source` | +| `api/status/choices.py` | `IncidentPhase.DETECTED`, `EventUpdateSource` | +| `api/status/serializers.py` | `ServiceEventSerializer` grows a detail shape | +| `api/polling/reconcile.py` | Calls the system-event writer | +| `api/dashboards/filters.py` | `event` out, `status__severity__in` in | +| `api/common/aggregates.py` | `by_event_kind` out | +| `api/tests/factories.py` | `watchers` reads the component annotation | +| `docs/api/openapi.yaml` | Three operations out, six in | +| `docs/specs/2026-08-23-statusboard-design.md` | Sections 5 to 7 out, ER diagram updated | + +--- + +## Task 1: Retire the v1 spec's client sections + +**Files:** +- Modify: `docs/specs/2026-08-23-statusboard-design.md` + +**Interfaces:** +- Consumes: nothing +- Produces: nothing. This is a documentation change with no code behind it. + +Sections 5, 6 and 7 describe a cream palette, tabs that no longer exist and a status dot the +row no longer carries. Rewriting them would leave two documents describing one API. + +- [ ] **Step 1: Confirm the checker does not read those sections** + +Run: `grep -n 'findall\|spec\[' docs/check_docs.py` + +Expected: it reads `re.findall(r"```mermaid\n(.*?)```", spec, re.DOTALL)[1]`, the second +mermaid block, which is in section 3. Sections 5 to 7 hold no mermaid block. + +- [ ] **Step 2: Run the checker before touching anything** + +Run: `docker compose -f docker-compose.test.yml run --rm docs` +Expected: PASS. This is the baseline the next step must not move. + +- [ ] **Step 3: Delete sections 5, 6 and 7 and leave a pointer** + +Delete everything from the line `## 5. API surface` up to but not including +`## 8. Testing, infra, deploy`. Put this in their place: + +```markdown +## 5. The API, the client and the design system + +Those three sections are now +[`2026-09-03-statusboard-client-design.md`](2026-09-03-statusboard-client-design.md). + +They described a palette, tabs and a row that the approved decks replaced on 2026-08-28. +Keeping a second account of one API is how two answers to the same question drift apart. +``` + +Renumber `## 8. Testing, infra, deploy` to `## 6. Testing, infra, deploy`. + +- [ ] **Step 4: Run the checker again** + +Run: `docker compose -f docker-compose.test.yml run --rm docs` +Expected: PASS, unchanged. + +- [ ] **Step 5: Commit** + +```bash +git add docs/specs/2026-08-23-statusboard-design.md +git commit -m "docs: one account of the API, not two + +The v1 spec's API, frontend and design sections predate the approved +decks. They name tabs that no longer exist and a palette that was +replaced. A pointer to the client design replaces them." +``` + +--- + +## Task 2: A component can be featured, and counts its own watchers + +**Files:** +- Modify: `api/catalog/models.py` +- Modify: `api/catalog/queries.py` +- Modify: `api/tests/factories.py` +- Create: `api/catalog/migrations/0003_component_is_featured.py` (generated) +- Test: `api/tests/test_catalog_models.py` + +**Interfaces:** +- Consumes: `catalog.queries.related_count`, `tests.factories.ComponentFactory`, `track` +- Produces: + - `ServiceComponent.is_featured: bool` + - `catalog.queries.COMPONENT_WATCHER_COUNT`, a `Count` expression. Annotate it as + `watcher_count` on a `ServiceComponent` queryset. + - `tests.factories.watchers(component)`, replacing the `Service` version, returns `int` + +`Service.watcher_count` was a column a signal kept true, and four write paths never reached +the signal. Migration `catalog/0002` dropped it. A component repeats neither. + +- [ ] **Step 1: Write the failing test** + +In `api/tests/test_catalog_models.py`: + +```python +def test_a_components_watcher_count_is_distinct_users(db): + # Two boards holding one component is two watchers. One person + # holding it twice is impossible, and one holding a sibling too is + # still one watcher of this component. + component = ComponentFactory() + sibling = ComponentFactory(service=component.service) + first = UserFactory() + track(component, user=first) + track(sibling, user=first) + track(component, user=UserFactory()) + + assert watchers(component) == 2 + assert watchers(sibling) == 1 +``` + +Add to `api/tests/factories.py`, replacing the `Service` version of `watchers`: + +```python +class UserFactory(factory.django.DjangoModelFactory): + class Meta: + model = settings.AUTH_USER_MODEL + + email = factory.Sequence(lambda n: f"watcher{n}@example.test") + + +def watchers(component): + """How many people track it, counted the way the app counts.""" + from catalog.models import ServiceComponent + from catalog.queries import COMPONENT_WATCHER_COUNT + + return ( + ServiceComponent.objects.annotate(n=COMPONENT_WATCHER_COUNT) + .get(pk=component.pk) + .n + ) +``` + +`UserFactory` needs `from django.conf import settings` and +`django.contrib.auth.get_user_model()`. Use `model = get_user_model()` rather than the +settings string, because factory_boy wants the class. + +- [ ] **Step 2: Run it to make sure it fails** + +Run: `cd api && uv run pytest tests/test_catalog_models.py::test_a_components_watcher_count_is_distinct_users -v` +Expected: FAIL with `ImportError: cannot import name 'COMPONENT_WATCHER_COUNT'`. + +- [ ] **Step 3: Add the annotation and the column** + +In `api/catalog/queries.py`, beside `WATCHER_COUNT`: + +```python +# Who tracks one component. `DashboardItem` points straight at it, so +# this is one join. The service version needed three, and only the +# service sort read it. +COMPONENT_WATCHER_COUNT = Count("boards__owner", distinct=True) +``` + +In `api/catalog/models.py`, on `ServiceComponent`, beside `is_overall`: + +```python + # Ticked on the overall component to feature a whole service. It is + # the first key of the suggested sort, so on day one it is the + # whole of it: every watcher count starts at zero. + is_featured = models.BooleanField(verbose_name="Featured", default=False) +``` + +- [ ] **Step 4: Make the migration and run the test** + +Run: +```bash +cd api && uv run python manage.py makemigrations catalog -n component_is_featured +uv run pytest tests/test_catalog_models.py::test_a_components_watcher_count_is_distinct_users -v +``` +Expected: PASS. + +- [ ] **Step 5: Fix the callers the rename broke** + +Run: `cd api && uv run pytest tests/ -x -q` + +`tests/test_board_api.py` calls `watchers(service)`. Change each call to name the component +the test tracked. Expected after: the whole suite passes. + +- [ ] **Step 6: Add the column to the ER diagram** + +In `docs/specs/2026-08-23-statusboard-design.md`, in the second mermaid block, inside +`ServiceComponent { ... }`: + +``` + bool is_featured "first key of the suggested sort" +``` + +- [ ] **Step 7: Commit** + +```bash +git add api/catalog/models.py api/catalog/queries.py api/catalog/migrations api/tests docs/specs +git commit -m "feat: a component is featured and counts its own watchers + +Discover searches components, so the suggested sort needs both keys on +a component. The watcher count is an annotation, not a column: the +service had one, and four write paths never reached the signal that +kept it true." +``` + +--- + +## Task 3: Ancestry is a stored array + +**Files:** +- Modify: `api/catalog/models.py` +- Modify: `api/catalog/serializers.py` +- Modify: `api/polling/reconcile.py` +- Create: `api/catalog/migrations/0004_component_ancestor_ids.py` (generated) +- Test: `api/tests/test_catalog_models.py`, `api/tests/test_reconcile.py` + +**Interfaces:** +- Consumes: `ServiceComponent.parent`, `polling.reconcile._upsert_components` +- Produces: + - `ServiceComponent.ancestor_ids: list[UUID]`, top down, the root first, excluding self + - `ServiceComponent.descendant_count() -> int`, replacing `child_count()` + - `polling.reconcile.rebuild_ancestry(service) -> None` + +A component's Components tab lists every descendant, not one level. A self-FK lookup returns +one level, so the query needs a stored path. + +- [ ] **Step 1: Write the failing test** + +In `api/tests/test_catalog_models.py`: + +```python +def test_ancestor_ids_are_stored_top_down(db): + # A descendant query reads this array. Walking `parent` per row + # cannot use an index, and `for_display` only selects two levels up. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + leaf = ComponentFactory(service=service, parent=middle) + rebuild_ancestry(service) + + leaf.refresh_from_db() + assert leaf.ancestor_ids == [top.id, middle.id] + + # Every descendant, not the direct children. `top` has two below it. + assert ServiceComponent.objects.filter(ancestor_ids__contains=[top.id]).count() == 2 +``` + +- [ ] **Step 2: Run it to make sure it fails** + +Run: `cd api && uv run pytest tests/test_catalog_models.py::test_ancestor_ids_are_stored_top_down -v` +Expected: FAIL with `ImportError: cannot import name 'rebuild_ancestry'`. + +- [ ] **Step 3: Add the column** + +In `api/catalog/models.py`, add the import and the field: + +```python +from django.contrib.postgres.fields import ArrayField +from django.contrib.postgres.indexes import GinIndex +``` + +On `ServiceComponent`, beside `parent`: + +```python + # The chain above this one, root first, written by reconcile. A + # component's Components tab lists every descendant, and a self-FK + # lookup returns one level. Walking the chain per row cannot use an + # index either. + ancestor_ids = ArrayField( + models.UUIDField(), default=list, blank=True, editable=False + ) +``` + +In `class Meta`, add to `indexes`: + +```python + GinIndex(fields=["ancestor_ids"], name="components_by_ancestor"), +``` + +`ServiceComponent.Meta` has no `indexes` list yet. Add one beside `constraints`. + +- [ ] **Step 4: Replace child_count with descendant_count** + +In `api/catalog/models.py`, replace the `child_count` method: + +```python + def descendant_count(self): + """How many components sit anywhere under this one.""" + if self.is_overall: + return 0 + prepared = getattr(self, "_descendant_count", None) + if prepared is None: + return ServiceComponent.objects.filter(ancestor_ids__contains=[self.pk]).count() + return prepared +``` + +In `ServiceComponentQuerySet.for_display`, replace the `_child_count` annotation: + +```python + _descendant_count=related_count( + self.model.objects, "ancestor_ids__contains", ref="pk" + ), +``` + +`related_count` builds `filter(**{group_by: OuterRef(ref)})`, so `ancestor_ids__contains` +needs the ref wrapped in a list. Write the subquery directly instead: + +```python + _descendant_count=Coalesce( + Subquery( + self.model.objects.filter( + ancestor_ids__contains=Func( + OuterRef("pk"), function="ARRAY", template="ARRAY[%(expressions)s]" + ) + ) + .order_by() + .values("service") + .annotate(total=models.Count("pk")) + .values("total"), + output_field=models.IntegerField(), + ), + 0, + ), +``` + +Import `Coalesce`, `Func`, `Subquery` from `django.db.models` and +`django.db.models.functions`. + +In `api/catalog/serializers.py`, rename the field on `ComponentSerializer`: + +```python + descendant_count = serializers.SerializerMethodField() +``` + +and the method, and the entry in `Meta.fields`: + +```python + @extend_schema_field(serializers.IntegerField()) + def get_descendant_count(self, row): + return row.descendant_count() +``` + +- [ ] **Step 5: Write the rebuild in reconcile** + +In `api/polling/reconcile.py`: + +```python +def rebuild_ancestry(service): + """Write every component's chain of ancestors, root first. + + A rename does not touch this. A reparent moves a whole subtree, so + the pass rewrites the service rather than one row. A service holds + hundreds of components, not millions. + """ + parents = dict( + ServiceComponent.objects.filter(service=service).values_list("id", "parent_id") + ) + chains = {} + + def chain(node): + if node in chains: + return chains[node] + parent = parents.get(node) + # A self-referencing column allows a loop. Stop rather than hang. + chains[node] = [] if parent is None or parent == node else [ + *chain(parent), + parent, + ] + return chains[node] + + rows = list(ServiceComponent.objects.filter(service=service)) + for row in rows: + row.ancestor_ids = chain(row.id) + ServiceComponent.objects.bulk_update(rows, ["ancestor_ids"]) +``` + +`chain` recurses on the parent map, not the database, so a deep tree costs one query. +The guard stops a row that points at itself. + +Call it at the end of `_upsert_components`, after the parent pass: + +```python + rebuild_ancestry(service) + return rows +``` + +- [ ] **Step 6: Run the tests** + +Run: `cd api && uv run python manage.py makemigrations catalog -n component_ancestor_ids && uv run pytest tests/ -q` +Expected: PASS. `tests/test_catalog_api.py` asserts on `child_count`; rename those +assertions to `descendant_count`. + +- [ ] **Step 7: Update the contract and the diagram** + +In `docs/api/openapi.yaml`, rename `child_count` to `descendant_count` in the `Component` +schema, and change its description: + +```yaml + descendant_count: + type: integer + description: | + How many components sit anywhere under this one. A component is a group when this + is above zero. There is no separate flag. +``` + +In the ER diagram, inside `ServiceComponent { ... }`: + +``` + uuid[] ancestor_ids "root first, written by reconcile" +``` + +`api/tests/test_contract.py::test_the_component_schema_has_exactly_the_documented_fields` +holds these together. + +- [ ] **Step 8: Commit** + +```bash +git add api docs +git commit -m "feat: a component stores the chain above it + +A component's Components tab lists every descendant. A self-FK lookup +returns one level, and walking the chain per row cannot use an index. +Reconcile writes the array, and a reparent rewrites the subtree." +``` + +--- + +## Task 4: `q` matches a component's whole path + +**Files:** +- Modify: `api/catalog/models.py` +- Modify: `api/polling/reconcile.py` +- Create: `api/catalog/migrations/0005_component_search_document.py` (generated) +- Test: `api/tests/test_catalog_models.py` + +**Interfaces:** +- Consumes: `polling.reconcile.rebuild_ancestry` +- Produces: + - `ServiceComponent.search_document: SearchVectorField` + - `polling.reconcile.rebuild_search(service) -> None` + - `ServiceComponentQuerySet.search(q)` returns the queryset filtered and annotated with + `rank`, ordered by `-rank` + +Searching `twilio` finds `SMS`, because Twilio is in its path. `twilio sms` finds it too. + +- [ ] **Step 1: Write the failing test** + +In `api/tests/test_catalog_models.py`: + +```python +def test_search_matches_a_components_path_and_ranks_the_rollup_first(db): + # The rollup's own name is an exact hit at weight A. A leaf matches + # only through its service at weight C, so it must sort below. + service = ServiceFactory(name="Twilio") + rollup = ComponentFactory(service=service, name="Twilio", is_overall=True) + parent = ComponentFactory(service=service, name="Programmable Messaging") + leaf = ComponentFactory(service=service, name="SMS", parent=parent) + rebuild_ancestry(service) + rebuild_search(service) + + found = list(ServiceComponent.objects.search("twilio")) + assert found[0] == rollup + assert leaf in found + + # Two words are an AND across the whole path, not two searches. + assert list(ServiceComponent.objects.search("twilio sms")) == [leaf] +``` + +- [ ] **Step 2: Run it to make sure it fails** + +Run: `cd api && uv run pytest tests/test_catalog_models.py::test_search_matches_a_components_path_and_ranks_the_rollup_first -v` +Expected: FAIL with `ImportError: cannot import name 'rebuild_search'`. + +- [ ] **Step 3: Add the column and the queryset method** + +In `api/catalog/models.py`: + +```python +from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVectorField +``` + +On `ServiceComponent`, beside `ancestor_ids`: + +```python + # The whole path, weighted, written by reconcile. Searching a + # service's name has to reach its components, and a path built by + # walking `parent` at query time cannot use an index. + search_document = SearchVectorField(null=True, editable=False) +``` + +In `Meta.indexes`: + +```python + GinIndex(fields=["search_document"], name="components_by_search"), +``` + +On `ServiceComponentQuerySet`: + +```python + def search(self, q): + """Rank a query against the stored path. + + Websearch mode, so several words are an AND. A caller ordering + by anything else drops the rank and keeps the filter. + """ + query = SearchQuery(q, search_type="websearch") + return ( + self.filter(search_document=query) + .annotate(rank=SearchRank(models.F("search_document"), query)) + .order_by("-rank") + ) +``` + +- [ ] **Step 4: Write the rebuild** + +In `api/polling/reconcile.py`: + +```python +from django.contrib.postgres.search import SearchVector +from django.db.models import Value + + +def rebuild_search(service): + """Write each component's weighted path. + + `A` is its own name, `B` its ancestors', `C` its service's. So an + exact hit on a rollup outranks a leaf that matched through its + service alone. + + Ancestor names are joined here rather than in SQL. Postgres cannot + build a vector from an array of foreign keys. + """ + rows = list( + ServiceComponent.objects.filter(service=service).select_related("service") + ) + names = {row.id: row.name for row in rows} + for row in rows: + ancestors = " ".join(names.get(a, "") for a in row.ancestor_ids) + row.search_document = ( + SearchVector(Value(row.name), weight="A") + + SearchVector(Value(ancestors), weight="B") + + SearchVector(Value(row.service.name), weight="C") + ) + ServiceComponent.objects.bulk_update(rows, ["search_document"]) +``` + +Call it in `_upsert_components`, after `rebuild_ancestry`: + +```python + rebuild_ancestry(service) + rebuild_search(service) + return rows +``` + +`rebuild_search` must run second. It reads `ancestor_ids`. + +- [ ] **Step 5: Run the test** + +Run: +```bash +cd api && uv run python manage.py makemigrations catalog -n component_search_document +uv run pytest tests/test_catalog_models.py -v +``` +Expected: PASS. + +- [ ] **Step 6: Prove a reparent rewrites the subtree** + +Add to `api/tests/test_reconcile.py`: + +```python +def test_a_reparent_rewrites_every_descendants_path(db): + # Ancestry and the search vector both carry the chain. A provider + # moving one node must not leave its children searchable under + # where they used to sit. + service = ServiceFactory(name="Acme") + old = ComponentFactory(service=service, name="Legacy", external_id="old") + new = ComponentFactory(service=service, name="Platform", external_id="new") + leaf = ComponentFactory(service=service, name="Queue", external_id="leaf", parent=old) + rebuild_ancestry(service) + rebuild_search(service) + + leaf.parent = new + leaf.save(update_fields=["parent"]) + rebuild_ancestry(service) + rebuild_search(service) + + leaf.refresh_from_db() + assert leaf.ancestor_ids == [new.id] + assert list(ServiceComponent.objects.search("legacy queue")) == [] + assert list(ServiceComponent.objects.search("platform queue")) == [leaf] +``` + +- [ ] **Step 7: Run it and commit** + +Run: `cd api && uv run pytest tests/ -q && uv run ruff check --fix . && uv run ruff format .` +Expected: PASS. + +Add to the ER diagram, inside `ServiceComponent { ... }`: + +``` + tsvector search_document "weighted path, written by reconcile" +``` + +```bash +git add api docs +git commit -m "feat: a search matches a component's whole path + +Searching a service's name has to reach its components. The vector is +weighted so an exact hit on a rollup outranks a leaf that matched only +through its service. A reparent rewrites the subtree." +``` + +--- + +## Task 5: The component collection + +**Files:** +- Create: `api/catalog/views_components.py` +- Modify: `api/catalog/filters.py` +- Modify: `api/catalog/urls.py` +- Modify: `docs/api/openapi.yaml` +- Test: `api/tests/test_component_api.py` + +**Interfaces:** +- Consumes: `ServiceComponent.objects.for_display`, `.search`, + `catalog.queries.COMPONENT_WATCHER_COUNT`, `status.queries.CURRENT_SEVERITY`, + `common.aggregates.StatusAggregateSet`, `catalog.serializers.ComponentSerializer` +- Produces: + - `GET /catalog/components/` named `component-list` + - `GET /catalog/components/{uuid}/` named `component-detail` + - `ComponentFilter` with `service`, `ancestor`, `event`, `is_overall`, `is_tracked`, + `status__severity`, `status__severity__lte`, `status__severity__in` + +One collection serves Discover, a service's Components tab, a component's descendants and an +event's affected list. + +- [ ] **Step 1: Write the failing tests** + +Create `api/tests/test_component_api.py`: + +```python +import pytest +from django.urls import reverse + +from polling.reconcile import rebuild_ancestry, rebuild_search +from tests.factories import ComponentFactory, ServiceFactory, track + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def tree(): + service = ServiceFactory(name="Twilio") + rollup = ComponentFactory(service=service, name="Twilio", is_overall=True) + parent = ComponentFactory(service=service, name="Programmable Messaging") + leaf = ComponentFactory(service=service, name="SMS", parent=parent) + rebuild_ancestry(service) + rebuild_search(service) + return service, rollup, parent, leaf + + +def test_the_collection_lists_every_component(client, tree): + # Discover searches all of them, rollups included. Narrowing here + # would make the signed-out board and Discover one list. + response = client.get(reverse("component-list")) + assert response.status_code == 200 + assert response.json()["aggregates"]["total"] == 3 + + +def test_is_overall_narrows_to_one_row_per_service(client, tree): + # This is the signed-out board: one row per service, not per part. + response = client.get(reverse("component-list"), {"is_overall": "true"}) + assert [r["name"] for r in response.json()["results"]] == ["Twilio"] + + +def test_ancestor_returns_every_descendant_not_one_level(client, tree): + # A component's Components tab is the same screen at a different + # root, so it cannot count differently from a service's. + _, _, parent, leaf = tree + response = client.get(reverse("component-list"), {"ancestor": str(parent.id)}) + assert [r["id"] for r in response.json()["results"]] == [str(leaf.id)] + + +def test_q_reaches_a_component_through_its_services_name(client, tree): + # Searching "twilio" must find SMS. Its own name says nothing + # about which service it belongs to. + response = client.get(reverse("component-list"), {"q": "twilio sms"}) + assert [r["name"] for r in response.json()["results"]] == ["SMS"] + + +def test_service_narrows_to_one_services_parts(client, tree): + # A service's Components tab. The rollup is excluded there, + # because the header already carries the service's status. + service, _, _, _ = tree + response = client.get( + reverse("component-list"), {"service": service.slug, "is_overall": "false"} + ) + assert {r["name"] for r in response.json()["results"]} == { + "Programmable Messaging", + "SMS", + } + + +def test_severity_in_takes_several_values(client, tree): + # The Severity filter offers all six values at once, so a single + # exact match cannot serve it. + response = client.get(reverse("component-list"), {"status__severity__in": "0,1,2"}) + assert response.status_code == 200 + + +def test_the_detail_answers_by_uuid(client, tree): + _, _, _, leaf = tree + response = client.get(reverse("component-detail", args=[leaf.id])) + assert response.status_code == 200 + assert response.json()["name"] == "SMS" +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_component_api.py -v` +Expected: FAIL with `NoReverseMatch: Reverse for 'component-list' not found`. + +- [ ] **Step 3: Grow the filter** + +Replace `ComponentFilter` in `api/catalog/filters.py`: + +```python +class ComponentFilter(filters.FilterSet): + # `status` is not a relation. A component has a history of statuses + # and the open one is current. Same contract name, same annotation. + status__severity = filters.NumberFilter(field_name="severity_now") + status__severity__lte = filters.NumberFilter( + field_name="severity_now", lookup_expr="lte" + ) + status__severity__in = filters.BaseInFilter( + field_name="severity_now", lookup_expr="in" + ) + service = filters.CharFilter(field_name="service__slug") + # Every descendant, not one level. `parent` would name a query this + # does not run. + ancestor = filters.UUIDFilter(method="filter_ancestor") + event = filters.UUIDFilter(field_name="events__id") + is_tracked = filters.BooleanFilter(field_name="_is_tracked") + + class Meta: + model = ServiceComponent + fields = {"is_overall": ["exact"], "is_featured": ["exact"]} + + def filter_ancestor(self, queryset, name, value): + return queryset.filter(ancestor_ids__contains=[value]) +``` + +- [ ] **Step 4: Write the views** + +Create `api/catalog/views_components.py`: + +```python +"""Every read of a component, in one place. + +Discover, a service's Components tab, a component's descendants and an +event's affected list are one collection with different parameters. +Four nested routes would have been four copies of this queryset. +""" + +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework import generics +from rest_framework.permissions import AllowAny + +from catalog.filters import ComponentFilter +from catalog.models import ServiceComponent +from catalog.queries import COMPONENT_WATCHER_COUNT +from catalog.serializers import ComponentSerializer +from common.aggregates import StatusAggregateSet +from common.filters import FieldsBackend +from common.ordering import MappedOrderingFilter +from status.queries import CURRENT_SEVERITY + + +class ComponentQueryMixin: + permission_classes = [AllowAny] + serializer_class = ComponentSerializer + queryset = ServiceComponent.objects.none() + + def get_queryset(self): + return ( + ServiceComponent.objects.for_display(self.request.user) + .annotate( + severity_now=CURRENT_SEVERITY, + watcher_count=COMPONENT_WATCHER_COUNT, + ) + .distinct() + ) + + +class ComponentListView(ComponentQueryMixin, generics.ListAPIView): + aggregate_set = StatusAggregateSet + filterset_class = ComponentFilter + filter_backends = [DjangoFilterBackend, MappedOrderingFilter, FieldsBackend] + ordering_fields = ["name", "status_page_order", "updated_at"] + # Severity ahead of popularity, the same as the service sort this + # replaces. Lower severity is worse, so ascending puts the broken + # first. + SUGGESTED = ["-is_featured", "severity_now", "-watcher_count", "name"] + ordering_map = { + "suggested": SUGGESTED, + "status__severity": ["severity_now"], + } + ordering = SUGGESTED + + def filter_queryset(self, queryset): + """A query ranks by relevance. Without one it is the suggested sort. + + One control and one label, Smart, on every list. A separate + "best match" would name a ranking that does not exist until + somebody types. + """ + q = self.request.query_params.get("q") + if q: + queryset = queryset.search(q) + return super().filter_queryset(queryset) + + +class ComponentDetailView(ComponentQueryMixin, generics.RetrieveAPIView): + filter_backends = [FieldsBackend] + lookup_field = "pk" +``` + +`search` returns an ordered queryset. `MappedOrderingFilter` applies the view's `ordering` +after it, so the rank is lost unless the caller asked for no ordering. Guard it by making +`ordering` conditional: + +```python + def get_ordering(self): + return [] if self.request.query_params.get("q") else self.ordering +``` + +DRF's `OrderingFilter` reads `view.ordering`, not a method, so set the attribute in +`initial` instead: + +```python + def initial(self, request, *args, **kwargs): + super().initial(request, *args, **kwargs) + if request.query_params.get("q"): + # `search` already ordered by rank. A default sort applied + # after it would throw the ranking away. + self.ordering = ["-rank"] +``` + +and add `"rank"` to `ordering_fields` so `MappedOrderingFilter` does not drop it. + +- [ ] **Step 5: Route them** + +In `api/catalog/urls.py`, above the router include: + +```python + path( + "catalog/components/", + ComponentListView.as_view(), + name="component-list", + ), + path( + "catalog/components//", + ComponentDetailView.as_view(), + name="component-detail", + ), +``` + +- [ ] **Step 6: Run the tests** + +Run: `cd api && uv run pytest tests/test_component_api.py -v` +Expected: PASS, all seven. + +- [ ] **Step 7: Document both operations** + +In `docs/api/openapi.yaml`, add under `paths:`: + +```yaml + /catalog/components/: + get: + tags: [catalog] + summary: Search and list components + description: | + One collection, four screens. Discover with no parameters, a service's parts with + `service`, a component's descendants with `ancestor`, an event's affected list with + `event`. + + `q` matches a component's whole path, so `twilio` finds `SMS`. Results are ranked by + relevance. Without `q` the default is `suggested`. + security: [] + parameters: + - { name: q, in: query, schema: { type: string } } + - { name: service, in: query, schema: { type: string }, description: Service slug } + - { name: ancestor, in: query, schema: { type: string, format: uuid }, description: Every descendant, at any depth } + - { name: event, in: query, schema: { type: string, format: uuid } } + - { name: is_overall, in: query, schema: { type: boolean } } + - { name: is_featured, in: query, schema: { type: boolean } } + - { name: is_tracked, in: query, schema: { type: boolean } } + - { name: status__severity, in: query, schema: { type: integer } } + - { name: status__severity__lte, in: query, schema: { type: integer } } + - { name: status__severity__in, in: query, schema: { type: string }, description: 'Comma separated, e.g. `0,1,2`' } + - { name: ordering, in: query, schema: { type: string, enum: [suggested, status_page_order, name, -name, status__severity, -status__severity], default: suggested } } + - { $ref: '#/components/parameters/Fields' } + - { $ref: '#/components/parameters/Cursor' } + - { $ref: '#/components/parameters/PageSize' } + responses: + '200': + description: OK + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/Envelope' + - type: object + properties: + aggregates: { $ref: '#/components/schemas/StatusAggregates' } + results: { type: array, items: { $ref: '#/components/schemas/Component' } } + /catalog/components/{uuid}/: + get: + tags: [catalog] + summary: One component + security: [] + parameters: + - { $ref: '#/components/parameters/Uuid' } + - { $ref: '#/components/parameters/Fields' } + responses: + '200': { description: OK, content: { application/json: { schema: { $ref: '#/components/schemas/Component' } } } } + '404': { $ref: '#/components/responses/Error' } +``` + +- [ ] **Step 8: Run the contract test and commit** + +Run: `cd api && uv run pytest tests/test_contract.py -v && uv run pytest tests/ -q` +Expected: PASS. + +```bash +git add api docs +git commit -m "feat: one component collection, four screens + +Discover, a service's parts, a component's descendants and an event's +affected list are one query with different parameters. Four nested +routes would have been four copies of it." +``` + +--- + +## Task 6: An event records who found it, and an update records who wrote it + +**Files:** +- Modify: `api/status/choices.py` +- Modify: `api/status/models.py` +- Create: `api/status/migrations/0003_event_detected_by.py` (generated) +- Test: `api/tests/test_status_models.py` + +**Interfaces:** +- Consumes: `status.choices.IncidentPhase`, `CLOSED_PHASES` +- Produces: + - `status.choices.EventSource`, with `PROVIDER` and `SYSTEM` + - `status.choices.IncidentPhase.DETECTED` + - `ServiceEvent.detected_by: str`, `ServiceEvent.external_id` nullable + - `EventUpdate.source: str` + +`external_id IS NULL` cannot answer "did we find this first". A claim fills it in, so the +origin fact needs a column of its own. + +- [ ] **Step 1: Write the failing test** + +In `api/tests/test_status_models.py`: + +```python +def test_two_system_events_can_sit_on_one_service(db): + # Neither has a provider id. The unique key is partial, so a null + # does not collide with another null. + service = ServiceFactory() + for _ in range(2): + ServiceEvent.objects.create( + service=service, + external_id=None, + kind=EventKind.INCIDENT, + title="Degraded", + phase=IncidentPhase.DETECTED, + detected_by=EventSource.SYSTEM, + starts_at=timezone.now(), + ) + assert ServiceEvent.objects.filter(service=service).count() == 2 + + +def test_a_provider_id_is_still_unique_per_service(db): + # A second poll of the same page must update the row, never add one. + service = ServiceFactory() + fields = dict( + service=service, + kind=EventKind.INCIDENT, + title="Outage", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + ) + ServiceEvent.objects.create(external_id="abc", **fields) + with pytest.raises(IntegrityError): + ServiceEvent.objects.create(external_id="abc", **fields) + + +def test_detected_is_an_open_phase(db): + # A detected event is running. Listing it as closed would hide + # every outage no provider ever wrote up. + assert IncidentPhase.DETECTED not in CLOSED_PHASES +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_status_models.py -k "system_events or provider_id_is_still or detected_is_an_open" -v` +Expected: FAIL with `ImportError: cannot import name 'EventSource'`. + +- [ ] **Step 3: Add the choices** + +In `api/status/choices.py`, add `DETECTED` as the first value of `IncidentPhase`: + +```python +class IncidentPhase(models.TextChoices): + # First, because it precedes anything a provider posts. We write it + # when a severity drops with nothing explaining it. + DETECTED = "detected", "Detected" + INVESTIGATING = "investigating", "Investigating" + IDENTIFIED = "identified", "Identified" + MONITORING = "monitoring", "Monitoring" + RESOLVED = "resolved", "Resolved" +``` + +`CLOSED_PHASES` is unchanged. `DETECTED` is open, so it is not in it. + +Add below `MaintenancePhase`: + +```python +class EventSource(models.TextChoices): + """Who produced a row: the provider's page, or this deployment. + + An event records who opened it. An update records who wrote it. A + claimed event has both, one per update. + """ + + PROVIDER = "provider", "Provider" + SYSTEM = "system", "Statusboard" +``` + +- [ ] **Step 4: Change the models** + +In `api/status/models.py`, on `ServiceEvent`: + +```python + # Null until a provider claims this event. We open events a + # provider never published, and they have no id of theirs. + external_id = models.CharField( + verbose_name="Provider ID", max_length=200, null=True, blank=True + ) + # Who opened it, never rewritten by a claim. `external_id IS NULL` + # cannot answer this: claiming fills the column in, which destroys + # the fact that we found the outage first. + detected_by = models.CharField( + max_length=32, + choices=EventSource.choices, + default=EventSource.PROVIDER, + db_default=EventSource.PROVIDER, + ) +``` + +Replace the constraint in `Meta`: + +```python + constraints = [ + # Partial. Two events we opened both hold null, and null + # does not collide with null in Postgres anyway. Stating + # the condition says the exemption is deliberate. + models.UniqueConstraint( + fields=["service", "external_id"], + condition=models.Q(external_id__isnull=False), + name="one_event_per_provider_id", + ) + ] +``` + +On `EventUpdate`: + +```python + # Who wrote this post. A claimed event holds both: our detection + # first, then the provider's log. + source = models.CharField( + max_length=32, + choices=EventSource.choices, + default=EventSource.PROVIDER, + db_default=EventSource.PROVIDER, + ) +``` + +- [ ] **Step 5: Run the tests** + +Run: +```bash +cd api && uv run python manage.py makemigrations status -n event_detected_by +uv run pytest tests/test_status_models.py -v +``` +Expected: PASS. + +- [ ] **Step 6: Update the diagram and the contract** + +In the ER diagram, inside `ServiceEvent { ... }` change `external_id` and add a line: + +``` + string external_id "provider's id, null until claimed" + enum detected_by "provider or system, never rewritten" +``` + +Inside `EventUpdate { ... }`: + +``` + enum source "provider or system" +``` + +In `docs/api/openapi.yaml`, add `detected` to the phase enum documentation under +`Meta.enums.event_phase`, and add to `EventUpdate`'s schema: + +```yaml + source: + type: string + enum: [provider, system] + description: | + Who wrote this update. `system` is a severity change we recorded ourselves. + Render the label from `/meta/` `enums.event_source`. +``` + +Add `event_source` to `Meta.enums` in the same file. + +- [ ] **Step 7: Publish the new enum** + +In `api/common/views.py`, find where `MetaView` builds `enums` and add: + +```python + "event_source": dict(EventSource.choices), +``` + +- [ ] **Step 8: Run everything and commit** + +Run: `cd api && uv run pytest tests/ -q && uv run ruff check --fix . && uv run ruff format .` +Expected: PASS. + +```bash +git add api docs +git commit -m "feat: an event records who found it + +A provider claims an event we opened, which fills in their id. That +destroys the fact that we saw the outage first, so the origin needs a +column. An update records its own author, so one card can hold both." +``` + +--- + +## Task 7: A severity change nobody explains becomes an event + +**Files:** +- Create: `api/polling/system_events.py` +- Modify: `api/polling/reconcile.py` +- Modify: `api/api/defaults.py` +- Test: `api/tests/test_system_events.py` + +**Interfaces:** +- Consumes: `status.models.ServiceEvent`, `EventUpdate`, `ComponentStatus`, + `status.choices.EventSource`, `IncidentPhase`, `Severity` +- Produces: + - `polling.system_events.SYSTEM_EVENT_MAX_SEVERITY = 2` + - `polling.system_events.reconcile_system_events(service, author) -> None` + +A provider can move a component to Degraded and never write an incident. An event-only feed +would hide it, and those spans are exposed nowhere else. + +- [ ] **Step 1: Add the setting** + +In `api/api/defaults.py`, beside the other `POLL_*` values: + +```python +# How far back a provider's event may start and still claim one we +# opened. Providers back-date `starts_at` to when an incident really +# began, which is before our poll saw it. +EVENT_CLAIM_WINDOW = timedelta( + seconds=int(os.environ.get("EVENT_CLAIM_WINDOW_SECONDS", 3600)) +) +``` + +- [ ] **Step 2: Write the failing tests** + +Create `api/tests/test_system_events.py`: + +```python +import pytest +from django.utils import timezone + +from polling.system_events import reconcile_system_events +from status.choices import EventKind, EventSource, IncidentPhase, Severity, StatusSource +from status.models import ComponentStatus, ServiceEvent +from tests.factories import ComponentFactory, ServiceFactory + +pytestmark = pytest.mark.django_db + + +def _status(component, severity): + ComponentStatus.objects.create( + component=component, + severity=severity, + source=StatusSource.PROVIDER, + started_at=timezone.now(), + ) + + +def _author(): + from django.contrib.auth import get_user_model + + return get_user_model().objects.system() + + +def test_an_unexplained_outage_opens_an_event(db): + # Without this the feed hides every outage a provider never wrote + # up, and the closed span is exposed nowhere else. + service = ServiceFactory() + component = ComponentFactory(service=service, name="SMS") + _status(component, Severity.DEGRADED) + + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.detected_by == EventSource.SYSTEM + assert event.external_id is None + assert event.phase == IncidentPhase.DETECTED + assert event.kind == EventKind.INCIDENT + assert list(event.affected_components.all()) == [component] + assert event.updates.count() == 1 + assert event.updates.first().source == EventSource.SYSTEM + + +def test_an_explained_outage_opens_nothing(db): + # The provider already told the story. A second card for one + # outage is the thing this design exists to avoid. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.MAJOR_OUTAGE) + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Outage", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + ) + provider.affected_components.set([component]) + + reconcile_system_events(service, _author()) + + assert ServiceEvent.objects.filter(detected_by=EventSource.SYSTEM).count() == 0 + + +def test_unknown_opens_nothing(db): + # Severity 3 is our own poll failing to read their page. Calling + # that their incident would report our fault as theirs. + service = ServiceFactory() + _status(ComponentFactory(service=service), Severity.UNKNOWN) + + reconcile_system_events(service, _author()) + + assert ServiceEvent.objects.count() == 0 + + +def test_every_transition_while_open_writes_an_update(db): + # A card shows how the outage moved. One update at the start would + # say a major outage had been degraded the whole time. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + + ComponentStatus.objects.filter(component=component, ended_at__isnull=True).update( + ended_at=timezone.now() + ) + _status(component, Severity.MAJOR_OUTAGE) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.updates.count() == 2 + assert event.ends_at is None + + +def test_recovery_closes_the_event(db): + # An event that cannot close leaves a permanently red row on + # somebody's board. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.MAJOR_OUTAGE) + reconcile_system_events(service, _author()) + + ComponentStatus.objects.filter(component=component, ended_at__isnull=True).update( + ended_at=timezone.now() + ) + _status(component, Severity.OPERATIONAL) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.phase == IncidentPhase.RESOLVED + assert event.ends_at is not None + assert event.updates.count() == 2 +``` + +- [ ] **Step 3: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_system_events.py -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'polling.system_events'`. + +- [ ] **Step 4: Write the module** + +Create `api/polling/system_events.py`: + +```python +"""Events we open when a provider explains nothing. + +A provider can move a component to Degraded and never write an +incident. An event-only feed would hide that outage, and the closed +`ComponentStatus` span is exposed nowhere else. + +`ComponentStatus` stays the truth. An event here is a projection of it, +written by this module alone and rebuildable from nothing. That is what +keeps it from being a second answer to the same question. +""" + +from django.conf import settings +from django.utils import timezone + +from status.choices import ( + CLOSED_PHASES, + EventKind, + EventSource, + IncidentPhase, + Severity, +) +from status.models import ComponentStatus, EventUpdate, ServiceEvent + +# Worse than this and nothing is wrong. Severity 3 is our own poll +# failing to read their page, and severity 4 is a window a provider +# always announces. Neither is an outage of theirs to report. +SYSTEM_EVENT_MAX_SEVERITY = Severity.DEGRADED + + +def reconcile_system_events(service, author): + """Open, extend and close the events this deployment owns.""" + for status in _bad_statuses(service): + _open_or_extend(status, author) + _close_recovered(service, author) + + +def _bad_statuses(service): + """Open spans bad enough to be somebody's outage.""" + return ComponentStatus.objects.filter( + component__service=service, + ended_at__isnull=True, + severity__lte=SYSTEM_EVENT_MAX_SEVERITY, + ).select_related("component") + + +def _explained(component): + """Whether a live provider event already names this component.""" + return ( + ServiceEvent.objects.live() + .filter(affected_components=component, detected_by=EventSource.PROVIDER) + .exists() + ) + + +def _ours(component): + """The open event we opened for this component, if there is one.""" + return ( + ServiceEvent.objects.live() + .filter(affected_components=component, detected_by=EventSource.SYSTEM) + .first() + ) + + +def _open_or_extend(status, author): + component = status.component + if _explained(component): + return + event = _ours(component) + if event is None: + event = ServiceEvent.objects.create( + service=component.service, + external_id=None, + detected_by=EventSource.SYSTEM, + kind=EventKind.INCIDENT, + title=_title(component, status.severity), + phase=IncidentPhase.DETECTED, + starts_at=status.started_at, + created_by=author, + updated_by=author, + ) + event.affected_components.set([component]) + _record(event, status, author) + + +def _record(event, status, author): + """One update per severity span, keyed on when the span began. + + A poll that changes nothing runs this again. `get_or_create` on the + span's start is what stops a duplicate post per beat. + """ + EventUpdate.objects.get_or_create( + event=event, + posted_at=status.started_at, + defaults={ + "phase": event.phase, + "source": EventSource.SYSTEM, + "body": _title(status.component, status.severity), + "created_by": author, + "updated_by": author, + }, + ) + + +def _close_recovered(service, author): + """Close ours once the component it names is no longer bad.""" + open_events = ServiceEvent.objects.filter( + service=service, detected_by=EventSource.SYSTEM + ).exclude(phase__in=CLOSED_PHASES) + for event in open_events: + component = event.affected_components.first() + if component is None: + continue + current = ComponentStatus.objects.filter( + component=component, ended_at__isnull=True + ).first() + if current is not None and current.severity <= SYSTEM_EVENT_MAX_SEVERITY: + continue + _close(event, component, current, author) + + +def _close(event, component, current, author): + now = timezone.now() + severity = current.severity if current is not None else Severity.OPERATIONAL + EventUpdate.objects.get_or_create( + event=event, + posted_at=now, + defaults={ + "phase": IncidentPhase.RESOLVED, + "source": EventSource.SYSTEM, + "body": _title(component, severity), + "created_by": author, + "updated_by": author, + }, + ) + event.phase = IncidentPhase.RESOLVED + event.ends_at = now + event.updated_by = author + event.save(update_fields=["phase", "ends_at", "updated_by"]) + + +def _title(component, severity): + """What the card says before a provider gives it a better name.""" + return f"{component.name} {Severity(severity).label.lower()}" +``` + +`settings.EVENT_CLAIM_WINDOW` is unused here. Task 8 uses it. + +- [ ] **Step 5: Call it from reconcile** + +In `api/polling/reconcile.py`, at the end of `apply_fetch`: + +```python + _upsert_events(service, events, rows, author, run) + # After the provider's events, so a component they explained does + # not also get one of ours in the same pass. + reconcile_system_events(service, author) +``` + +- [ ] **Step 6: Run the tests** + +Run: `cd api && uv run pytest tests/test_system_events.py -v` +Expected: PASS, all five. + +- [ ] **Step 7: Run the whole suite and commit** + +Run: `cd api && uv run pytest tests/ -q && uv run ruff check --fix . && uv run ruff format .` +Expected: PASS. + +```bash +git add api docs +git commit -m "feat: a status change with no provider post becomes an event + +A provider can move a component to Degraded and never write an +incident. An event-only feed hid those, and the closed status span was +exposed nowhere else. The client now reads one table." +``` + +--- + +## Task 8: A provider claims the event we opened + +**Files:** +- Modify: `api/polling/system_events.py` +- Modify: `api/polling/reconcile.py` +- Test: `api/tests/test_system_events.py` + +**Interfaces:** +- Consumes: `settings.EVENT_CLAIM_WINDOW`, `polling.system_events._ours` +- Produces: `polling.system_events.claim(event, author) -> bool`, true when it took one over + +One outage is one card. A provider posting ten minutes after our poll saw the change is +more updates on the same event, not a second event. + +- [ ] **Step 1: Write the failing tests** + +Append to `api/tests/test_system_events.py`: + +```python +from datetime import timedelta + +from polling.system_events import claim + + +def test_a_provider_event_claims_the_one_we_opened(db): + # One outage is one card. Two cards for one outage is what the + # claim exists to prevent. + service = ServiceFactory() + component = ComponentFactory(service=service, name="SMS") + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Elevated SMS delivery failures", + phase=IncidentPhase.INVESTIGATING, + starts_at=ours.starts_at - timedelta(minutes=10), + ) + provider.affected_components.set([component]) + + assert claim(provider, _author()) is True + + ours.refresh_from_db() + assert ours.external_id == "abc" + assert ours.title == "Elevated SMS delivery failures" + # Ours records who found it, and a claim never rewrites that. + assert ours.detected_by == EventSource.SYSTEM + assert not ServiceEvent.objects.filter(pk=provider.pk).exists() + + +def test_a_stale_provider_event_claims_nothing(db): + # An incident that began a day before ours is a different outage. + # Merging them would put one card's updates on another's timeline. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Yesterday", + phase=IncidentPhase.INVESTIGATING, + starts_at=ours.starts_at - timedelta(days=1), + ) + provider.affected_components.set([component]) + + assert claim(provider, _author()) is False + assert ServiceEvent.objects.count() == 2 + + +def test_archiving_a_component_closes_our_open_event(db): + # We can no longer watch it recover, so the event could never + # close on its own. A red row would stay on a board for good. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.MAJOR_OUTAGE) + reconcile_system_events(service, _author()) + + component.is_archived = True + component.save(update_fields=["is_archived"]) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.phase == IncidentPhase.RESOLVED + assert event.ends_at is not None +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_system_events.py -k "claims or archiving" -v` +Expected: FAIL with `ImportError: cannot import name 'claim'`. + +- [ ] **Step 3: Write the claim** + +In `api/polling/system_events.py`: + +```python +def claim(provider_event, author): + """Fold a provider's event into the one we opened for the same outage. + + A provider often posts minutes after our poll saw the change. Two + rows would be two cards for one outage. + + The provider's row is deleted and ours takes its id, so the fact + that we found it first survives. Deleting theirs rather than ours + is what keeps `detected_by` meaningful. + """ + if provider_event.detected_by != EventSource.PROVIDER: + return False + ours = _claimable(provider_event) + if ours is None: + return False + ours.external_id = provider_event.external_id + ours.title = provider_event.title + ours.phase = provider_event.phase + ours.ends_at = provider_event.ends_at + ours.updated_by = author + ours.save( + update_fields=["external_id", "title", "phase", "ends_at", "updated_by"] + ) + provider_event.updates.update(event=ours) + ours.affected_components.add(*provider_event.affected_components.all()) + provider_event.delete() + return True + + +def _claimable(provider_event): + """Our open event for the same outage, or nothing. + + Same service, intersecting components, and a start no earlier than + ours less the window. The nearest start wins when several match. + """ + components = list(provider_event.affected_components.all()) + if not components: + return None + floor = provider_event.starts_at + settings.EVENT_CLAIM_WINDOW + candidates = ( + ServiceEvent.objects.live() + .filter( + service=provider_event.service, + detected_by=EventSource.SYSTEM, + external_id__isnull=True, + affected_components__in=components, + starts_at__lte=floor, + ) + .distinct() + ) + return min( + candidates, + key=lambda e: abs(e.starts_at - provider_event.starts_at), + default=None, + ) +``` + +`starts_at__lte=floor` reads as "ours began no later than theirs plus the window", which is +the same rule as "theirs began no earlier than ours less the window". + +- [ ] **Step 4: Close an event whose component is archived** + +In `_close_recovered`, replace the body of the loop: + +```python + for event in open_events: + component = event.affected_components.first() + if component is None: + continue + current = ComponentStatus.objects.filter( + component=component, ended_at__isnull=True + ).first() + still_bad = current is not None and current.severity <= SYSTEM_EVENT_MAX_SEVERITY + # An archived component is one the provider stopped publishing. + # We cannot watch it recover, so the event could never close. + if still_bad and not component.is_archived: + continue + _close(event, component, current, author) +``` + +- [ ] **Step 5: Call the claim from reconcile** + +In `api/polling/reconcile.py`, at the end of `_upsert_events`'s loop body: + +```python + for update in incoming.updates: + EventUpdate.objects.get_or_create( + event=event, + posted_at=update.posted_at, + defaults={ + "phase": update.phase, + "body": update.body, + "source": EventSource.PROVIDER, + "created_by": author, + "updated_by": author, + }, + ) + # After the updates, so they move with the row if this event + # takes over one we opened. + claim(event, author) +``` + +Import `claim` and `EventSource` at the top of `reconcile.py`. + +`_upsert_events` uses `update_or_create` keyed on `(service, external_id)`. Once ours has +taken the id, the next poll finds ours and updates it in place, so `claim` returns false +and nothing happens twice. + +- [ ] **Step 6: Run the tests** + +Run: `cd api && uv run pytest tests/test_system_events.py -v` +Expected: PASS, all eight. + +- [ ] **Step 7: Prove a claimed event survives the next poll** + +Append to `api/tests/test_system_events.py`: + +```python +def test_a_claimed_event_is_not_claimed_twice(db): + # The next poll finds ours by the id it took, so `update_or_create` + # updates it in place. A second claim would delete the row it just + # matched. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Outage", + phase=IncidentPhase.INVESTIGATING, + starts_at=ours.starts_at, + ) + provider.affected_components.set([component]) + claim(provider, _author()) + + ours.refresh_from_db() + assert claim(ours, _author()) is False + assert ServiceEvent.objects.count() == 1 +``` + +`claim` returns false here because `ours.detected_by` is `system`, which the first guard +rejects. + +- [ ] **Step 8: Run everything and commit** + +Run: `cd api && uv run pytest tests/ -q && uv run ruff check --fix . && uv run ruff format .` +Expected: PASS. + +```bash +git add api docs +git commit -m "feat: a provider's post joins the event we already opened + +Twilio's dot goes red at 14:02 and they post at 14:14. Two rows would +be two cards for one outage. Theirs is folded into ours, so the card +gains updates rather than a duplicate, and detected_by survives." +``` + +--- + +## Task 9: The event feed + +**Files:** +- Create: `api/status/views.py` +- Create: `api/status/filters.py` +- Create: `api/status/urls.py` +- Modify: `api/api/urls.py` +- Modify: `docs/api/openapi.yaml` +- Test: `api/tests/test_event_api.py` + +**Interfaces:** +- Consumes: `status.models.ServiceEvent`, `status.serializers.ServiceEventSerializer`, + `common.aggregates.EventAggregateSet`, `status.choices.CLOSED_PHASES` +- Produces: + - `GET /events/` named `event-list` + - `EventFilter` with `dashboard`, `service`, `component`, `kind`, `phase` + +One feed serves Home, a service and a component. The screens call it Updates. The API names +the model. + +- [ ] **Step 1: Write the failing tests** + +Create `api/tests/test_event_api.py`: + +```python +import pytest +from django.urls import reverse +from django.utils import timezone + +from status.choices import EventKind, IncidentPhase, MaintenancePhase +from status.models import ServiceEvent +from tests.factories import ComponentFactory, ServiceFactory, track + +pytestmark = pytest.mark.django_db + + +def _event(service, component, **kwargs): + fields = dict( + kind=EventKind.INCIDENT, + title="Something broke", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + ) + fields.update(kwargs) + event = ServiceEvent.objects.create(service=service, **fields) + event.affected_components.set([component]) + return event + + +def test_the_feed_lists_events_newest_first(client): + # A feed is read from the top. Oldest first would open on an + # incident from last month. + service = ServiceFactory() + component = ComponentFactory(service=service) + old = _event(service, component, external_id="1", title="Older") + new = _event(service, component, external_id="2", title="Newer") + ServiceEvent.objects.filter(pk=new.pk).update( + starts_at=timezone.now() + timezone.timedelta(minutes=5) + ) + + results = client.get(reverse("event-list")).json()["results"] + assert [r["title"] for r in results] == ["Newer", "Older"] + + +def test_service_narrows_the_feed(client): + # A service's Updates tab. Anything from another service on it + # would be answering a question nobody asked on that page. + first = ServiceFactory() + second = ServiceFactory() + _event(first, ComponentFactory(service=first), external_id="1", title="Mine") + _event(second, ComponentFactory(service=second), external_id="2", title="Theirs") + + results = client.get(reverse("event-list"), {"service": first.slug}).json()["results"] + assert [r["title"] for r in results] == ["Mine"] + + +def test_component_narrows_the_feed(client): + # A component's Updates tab reads the same collection one level + # down, so it cannot be a different endpoint. + service = ServiceFactory() + wanted = ComponentFactory(service=service) + other = ComponentFactory(service=service) + _event(service, wanted, external_id="1", title="Wanted") + _event(service, other, external_id="2", title="Other") + + results = client.get( + reverse("event-list"), {"component": str(wanted.id)} + ).json()["results"] + assert [r["title"] for r in results] == ["Wanted"] + + +def test_phase_open_and_closed_draw_one_line(client): + # `CLOSED_PHASES` lives in status/choices.py. A client restating + # which phases are terminal is a second copy that can drift. + service = ServiceFactory() + component = ComponentFactory(service=service) + _event(service, component, external_id="1", title="Open") + _event( + service, + component, + external_id="2", + title="Done", + phase=IncidentPhase.RESOLVED, + ) + _event( + service, + component, + external_id="3", + title="Finished", + kind=EventKind.MAINTENANCE, + phase=MaintenancePhase.COMPLETED, + ) + + open_titles = client.get(reverse("event-list"), {"phase": "open"}).json()["results"] + closed = client.get(reverse("event-list"), {"phase": "closed"}).json()["results"] + assert [r["title"] for r in open_titles] == ["Open"] + assert {r["title"] for r in closed} == {"Done", "Finished"} + + +def test_dashboard_narrows_to_what_you_track(client, board): + # Home's Updates tab. Everything posted across the services on + # your board, and nothing else. + tracked = ComponentFactory() + untracked = ComponentFactory() + track(tracked, user=board.owner) + _event(tracked.service, tracked, external_id="1", title="Yours") + _event(untracked.service, untracked, external_id="2", title="Not yours") + + client.force_login(board.owner) + results = client.get( + reverse("event-list"), {"dashboard": str(board.id)} + ).json()["results"] + assert [r["title"] for r in results] == ["Yours"] +``` + +`board` is the fixture `api/tests/test_board_api.py` already uses. Move it into +`api/tests/conftest.py` so both files reach it. + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_event_api.py -v` +Expected: FAIL with `NoReverseMatch: Reverse for 'event-list' not found`. + +- [ ] **Step 3: Write the filter** + +Create `api/status/filters.py`: + +```python +"""What a caller may narrow the event feed by. + +Beside the views, and apart from them. A filter is the query contract, +and it is read on its own more often than the view around it. +""" + +from django_filters import rest_framework as filters + +from status.choices import CLOSED_PHASES +from status.models import ServiceEvent + +PHASE_STATES = [("open", "Open"), ("closed", "Closed")] + + +class EventFilter(filters.FilterSet): + # Declared, not generated. The line between an open phase and a + # closed one is `CLOSED_PHASES`, and a client restating it is a + # second copy of one rule. + phase = filters.ChoiceFilter(choices=PHASE_STATES, method="filter_phase") + service = filters.CharFilter(field_name="service__slug") + component = filters.UUIDFilter(field_name="affected_components__id") + dashboard = filters.UUIDFilter(method="filter_dashboard") + + class Meta: + model = ServiceEvent + fields = {"kind": ["exact"]} + + def filter_phase(self, queryset, name, value): + if value == "closed": + return queryset.filter(phase__in=CLOSED_PHASES) + return queryset.exclude(phase__in=CLOSED_PHASES) + + def filter_dashboard(self, queryset, name, value): + """Everything posted across the services on one board. + + The board's rows are components, and an event names components. + A user reaches only their own board: the view checks the owner + before this runs. + """ + return queryset.filter( + affected_components__tracked_by__dashboard_id=value + ).distinct() +``` + +- [ ] **Step 4: Write the view** + +Create `api/status/views.py`: + +```python +"""Every read of an event, in one place. + +Home, a service and a component show the same feed at different scopes. +The screens call it Updates. This names the model, and `/meta/` +publishes the labels. +""" + +from django.shortcuts import get_object_or_404 +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework import generics +from rest_framework.permissions import AllowAny + +from common.aggregates import EventAggregateSet +from common.filters import FieldsBackend +from common.ordering import MappedOrderingFilter +from dashboards.models import Dashboard +from status.filters import EventFilter +from status.models import ServiceEvent +from status.serializers import ServiceEventSerializer + + +class EventListView(generics.ListAPIView): + permission_classes = [AllowAny] + serializer_class = ServiceEventSerializer + aggregate_set = EventAggregateSet + filterset_class = EventFilter + filter_backends = [DjangoFilterBackend, MappedOrderingFilter, FieldsBackend] + ordering_fields = ["starts_at", "ends_at"] + ordering_map = {} + ordering = ["-starts_at"] + queryset = ServiceEvent.objects.none() + + def get_queryset(self): + queryset = ServiceEvent.objects.select_related("service").prefetch_related( + "updates" + ) + board = self.request.query_params.get("dashboard") + if board: + # 404 rather than 403: someone else's board id should not + # be confirmable. The filter runs after this. + get_object_or_404(Dashboard, id=board, owner=self.request.user) + return queryset +``` + +An anonymous caller passing `dashboard` reaches `get_object_or_404` with +`owner=AnonymousUser`, which raises `TypeError`. Guard it: + +```python + if board: + if not self.request.user.is_authenticated: + raise NotAuthenticated + get_object_or_404(Dashboard, id=board, owner=self.request.user) +``` + +Import `NotAuthenticated` from `rest_framework.exceptions`. + +- [ ] **Step 5: Route it** + +Create `api/status/urls.py`: + +```python +from django.urls import path + +from status.views import EventListView + +urlpatterns = [ + path("events/", EventListView.as_view(), name="event-list"), +] +``` + +In `api/api/urls.py`, beside the other includes: + +```python + path("", include("status.urls")), +``` + +- [ ] **Step 6: Run the tests** + +Run: `cd api && uv run pytest tests/test_event_api.py -v` +Expected: PASS, all five. + +- [ ] **Step 7: Document the operation** + +In `docs/api/openapi.yaml`, add under `paths:`: + +```yaml + /events/: + get: + tags: [status] + summary: The event feed + description: | + One feed, three scopes. Home with `dashboard`, a service with `service`, a component + with `component`. + + `phase` takes `open` or `closed`. The line between them is `CLOSED_PHASES` in the + code, so a client never restates which phases are terminal. + security: [] + parameters: + - { name: dashboard, in: query, schema: { type: string, format: uuid }, description: Requires auth, and must be your own } + - { name: service, in: query, schema: { type: string }, description: Service slug } + - { name: component, in: query, schema: { type: string, format: uuid } } + - { name: kind, in: query, schema: { type: string, enum: [incident, maintenance] } } + - { name: phase, in: query, schema: { type: string, enum: [open, closed] } } + - { name: ordering, in: query, schema: { type: string, enum: [-starts_at, starts_at, ends_at, -ends_at], default: -starts_at } } + - { $ref: '#/components/parameters/Fields' } + - { $ref: '#/components/parameters/Cursor' } + - { $ref: '#/components/parameters/PageSize' } + responses: + '200': + description: OK + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/Envelope' + - type: object + properties: + aggregates: { $ref: '#/components/schemas/EventAggregates' } + results: { type: array, items: { $ref: '#/components/schemas/ServiceEvent' } } +``` + +- [ ] **Step 8: Run the contract test and commit** + +Run: `cd api && uv run pytest tests/test_contract.py tests/ -q` +Expected: PASS. + +```bash +git add api docs +git commit -m "feat: one event feed for Home, a service and a component + +Three screens showed the same list at different scopes. The phase +filter draws its line from CLOSED_PHASES, so no client carries a copy +of which phases are terminal." +``` + +--- + +## Task 10: One event, and its update log + +**Files:** +- Modify: `api/status/views.py` +- Modify: `api/status/urls.py` +- Modify: `api/status/serializers.py` +- Modify: `docs/api/openapi.yaml` +- Test: `api/tests/test_event_api.py` + +**Interfaces:** +- Consumes: `status.serializers.ServiceEventSerializer`, `EventUpdateSerializer` +- Produces: + - `GET /events/{uuid}/` named `event-detail` + - `GET /events/{uuid}/updates/` named `event-updates` + - `ServiceEventDetailSerializer` with `update_count` and `affected_count` + +The event screen's header needs both counts before either tab is opened. Its Timeline is a +paged list, because a provider's log has no ceiling. + +- [ ] **Step 1: Write the failing tests** + +Append to `api/tests/test_event_api.py`: + +```python +from status.choices import EventSource +from status.models import EventUpdate + + +def test_the_detail_carries_both_tab_counts(client): + # The header draws `Timeline 3` and `Affects 2` before either tab + # is opened, so neither can wait for that tab's request. + service = ServiceFactory() + first = ComponentFactory(service=service) + second = ComponentFactory(service=service) + event = _event(service, first, external_id="1") + event.affected_components.add(second) + for minute in range(3): + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.INVESTIGATING, + body=f"update {minute}", + posted_at=timezone.now(), + ) + + body = client.get(reverse("event-detail", args=[event.id])).json() + assert body["update_count"] == 3 + assert body["affected_count"] == 2 + + +def test_the_timeline_is_its_own_paged_list(client): + # A provider's log has no ceiling, so it cannot ride on the detail. + service = ServiceFactory() + event = _event(service, ComponentFactory(service=service), external_id="1") + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.IDENTIFIED, + body="Cause found", + posted_at=timezone.now(), + source=EventSource.PROVIDER, + ) + + body = client.get(reverse("event-updates", args=[event.id])).json() + assert body["aggregates"]["total"] == 1 + assert body["results"][0]["source"] == "provider" + + +def test_the_timeline_is_oldest_first(client): + # A story is read forwards. The feed is newest first because it is + # a feed; one event's log is a narrative. + service = ServiceFactory() + event = _event(service, ComponentFactory(service=service), external_id="1") + later = EventUpdate.objects.create( + event=event, + phase=IncidentPhase.IDENTIFIED, + body="Second", + posted_at=timezone.now() + timezone.timedelta(minutes=1), + ) + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.INVESTIGATING, + body="First", + posted_at=timezone.now(), + ) + + results = client.get(reverse("event-updates", args=[event.id])).json()["results"] + assert [r["body"] for r in results] == ["First", "Second"] +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_event_api.py -k "detail_carries or timeline" -v` +Expected: FAIL with `NoReverseMatch: Reverse for 'event-detail' not found`. + +- [ ] **Step 3: Add the serializers** + +In `api/status/serializers.py`, add `source` to `EventUpdateSerializer`: + +```python +class EventUpdateSerializer(FieldsMixin, serializers.ModelSerializer): + class Meta: + model = EventUpdate + fields = ["phase", "body", "posted_at", "source"] +``` + +Remove the nested `updates` from `ServiceEventSerializer`. A feed row costs a title, not a +log, and the log is now its own operation: + +```python +class ServiceEventSerializer(FieldsMixin, serializers.ModelSerializer): + """One row of the feed. The log is `/events/{uuid}/updates/`.""" + + class Meta: + model = ServiceEvent + fields = [ + "id", + "kind", + "title", + "phase", + "starts_at", + "ends_at", + "detected_by", + ] + + +class ServiceEventDetailSerializer(ServiceEventSerializer): + """The event screen's header and its About tab. + + Both counts are here because the header draws both tab badges + before either tab has made a request. + """ + + update_count = serializers.SerializerMethodField() + affected_count = serializers.SerializerMethodField() + last_update_at = serializers.SerializerMethodField() + + class Meta(ServiceEventSerializer.Meta): + fields = [ + *ServiceEventSerializer.Meta.fields, + "update_count", + "affected_count", + "last_update_at", + ] + + @extend_schema_field(serializers.IntegerField()) + def get_update_count(self, event): + return event.updates.count() + + @extend_schema_field(serializers.IntegerField()) + def get_affected_count(self, event): + return event.affected_components.count() + + @extend_schema_field(serializers.DateTimeField(allow_null=True)) + def get_last_update_at(self, event): + newest = event.updates.order_by("-posted_at").first() + return newest.posted_at if newest else None +``` + +`ServiceEventSerializer` gains `detected_by`, so a card can say the outage was ours to find. + +- [ ] **Step 4: Add the views** + +In `api/status/views.py`: + +```python +class EventDetailView(generics.RetrieveAPIView): + permission_classes = [AllowAny] + serializer_class = ServiceEventDetailSerializer + filter_backends = [FieldsBackend] + queryset = ServiceEvent.objects.all() + + +class EventUpdateListView(generics.ListAPIView): + """One event's log, oldest first. + + The feed is newest first because it is a feed. This is a narrative, + and a narrative is read forwards. + """ + + permission_classes = [AllowAny] + serializer_class = EventUpdateSerializer + filter_backends = [FieldsBackend] + queryset = EventUpdate.objects.none() + + def get_queryset(self): + event = get_object_or_404(ServiceEvent, pk=self.kwargs["pk"]) + return event.updates.order_by("posted_at") +``` + +`EnvelopePagination.get_ordering` appends `-created_at` as the cursor tiebreak, which +would fight `posted_at`. Give this view its own paginator: + +```python +class TimelinePagination(EnvelopePagination): + ordering = "posted_at" + tiebreak = "created_at" +``` + +Put it in `api/common/pagination.py` and set `pagination_class = TimelinePagination` on +`EventUpdateListView`. + +- [ ] **Step 5: Route them** + +In `api/status/urls.py`: + +```python + path("events//", EventDetailView.as_view(), name="event-detail"), + path( + "events//updates/", + EventUpdateListView.as_view(), + name="event-updates", + ), +``` + +- [ ] **Step 6: Run the tests** + +Run: `cd api && uv run pytest tests/test_event_api.py -v` +Expected: PASS, all eight. + +- [ ] **Step 7: Document both, and split the schema** + +In `docs/api/openapi.yaml`, remove `updates` from the `ServiceEvent` schema, add +`detected_by`, and add a `ServiceEventDetail` schema: + +```yaml + ServiceEventDetail: + allOf: + - $ref: '#/components/schemas/ServiceEvent' + - type: object + description: | + The event screen's header. Both counts are here because the header draws both tab + badges before either tab has made a request. + properties: + update_count: { type: integer } + affected_count: { type: integer } + last_update_at: { type: string, format: date-time, nullable: true } +``` + +Add the two operations: + +```yaml + /events/{uuid}/: + get: + tags: [status] + summary: One event + security: [] + parameters: + - { $ref: '#/components/parameters/Uuid' } + - { $ref: '#/components/parameters/Fields' } + responses: + '200': { description: OK, content: { application/json: { schema: { $ref: '#/components/schemas/ServiceEventDetail' } } } } + '404': { $ref: '#/components/responses/Error' } + /events/{uuid}/updates/: + get: + tags: [status] + summary: One event's log, oldest first + description: Its own operation because a provider's log has no ceiling. + security: [] + parameters: + - { $ref: '#/components/parameters/Uuid' } + - { $ref: '#/components/parameters/Fields' } + - { $ref: '#/components/parameters/Cursor' } + - { $ref: '#/components/parameters/PageSize' } + responses: + '200': + description: OK + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/Envelope' + - type: object + properties: + aggregates: { $ref: '#/components/schemas/Aggregates' } + results: { type: array, items: { $ref: '#/components/schemas/EventUpdate' } } +``` + +Add `ServiceEventDetail` to `BACKED` in `docs/check_docs.py`, mapped to `ServiceEvent`, +and add `ServiceEventDetail.update_count`, `.affected_count` and `.last_update_at` to +`DERIVED`. + +- [ ] **Step 8: Run everything and commit** + +Run: `cd api && uv run pytest tests/ -q && docker compose -f docker-compose.test.yml run --rm docs` +Expected: PASS. + +```bash +git add api docs +git commit -m "feat: an event, and its log as its own list + +The header draws both tab badges before either tab asks for anything, +so the detail carries both counts. The log is paged on its own: a +provider writes as many updates as they like." +``` + +--- + +## Task 11: Somewhere to send a URL we cannot read + +**Files:** +- Modify: `api/catalog/models.py` +- Create: `api/catalog/serializers.py` addition +- Modify: `api/catalog/views_components.py` or a new view module +- Modify: `api/catalog/urls.py` +- Modify: `api/api/defaults.py` +- Test: `api/tests/test_service_requests.py` + +**Interfaces:** +- Consumes: `common.models.BaseModel`, `catalog.models.StatusPage.normalise_url`, + `api.defaults.Throttle` +- Produces: + - `catalog.models.ServiceRequest` with `url`, `request_count`, `last_requested_at` + - `POST /catalog/requests/` named `catalog-request` + +The Add-by-URL screen's not-found state offers "Send this URL to us". `POST /catalog/import/` +stores nothing about a failed attempt, so that button has nowhere to send anything. + +- [ ] **Step 1: Write the failing tests** + +Create `api/tests/test_service_requests.py`: + +```python +import pytest +from django.urls import reverse + +from catalog.models import ServiceRequest + +pytestmark = pytest.mark.django_db + + +def test_asking_records_the_url(client): + response = client.post( + reverse("catalog-request"), + {"url": "https://status.fastmail.com/"}, + content_type="application/json", + ) + assert response.status_code == 202 + row = ServiceRequest.objects.get() + assert row.url == "https://status.fastmail.com" + assert row.request_count == 1 + + +def test_asking_twice_counts_rather_than_duplicates(client): + # State belongs to the URL, not to a request. A row per request + # would answer one question twice once a URL was triaged. + for _ in range(2): + client.post( + reverse("catalog-request"), + {"url": "https://status.fastmail.com/"}, + content_type="application/json", + ) + assert ServiceRequest.objects.count() == 1 + assert ServiceRequest.objects.get().request_count == 2 + + +def test_the_answer_is_the_same_whether_we_hold_it_or_not(client): + # Always 202. A different code would reveal which URLs are + # already in the catalog to anyone who asked. + first = client.post( + reverse("catalog-request"), + {"url": "https://status.one.example/"}, + content_type="application/json", + ) + second = client.post( + reverse("catalog-request"), + {"url": "https://status.one.example/"}, + content_type="application/json", + ) + assert first.status_code == second.status_code == 202 + assert first.content == second.content + + +def test_a_signed_in_asker_is_recorded(client, user): + # created_by carries it. There is no requested_by: that is what + # BaseModel already holds. + client.force_login(user) + client.post( + reverse("catalog-request"), + {"url": "https://status.two.example/"}, + content_type="application/json", + ) + assert ServiceRequest.objects.get().created_by == user + + +def test_a_malformed_url_is_refused(client): + response = client.post( + reverse("catalog-request"), + {"url": "not a url"}, + content_type="application/json", + ) + assert response.status_code == 400 +``` + +`user` is a fixture. Add it to `api/tests/conftest.py` if it is not already there: + +```python +@pytest.fixture +def user(db): + from django.contrib.auth import get_user_model + + return get_user_model().objects.create(email="asker@example.test") +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_service_requests.py -v` +Expected: FAIL with `ImportError: cannot import name 'ServiceRequest'`. + +- [ ] **Step 3: Add the model** + +In `api/catalog/models.py`, below `StatusPage`: + +```python +class ServiceRequest(BaseModel): + """A status page somebody asked for and we could not read. + + One row per URL, not one per request. A row per request would + answer one question twice once a URL was triaged. + + There is no `requested_by`. `BaseModel.created_by` already holds + it, null when the asker was signed out. The v1 spec deleted + `Service.added_by` for the same reason. + + There is no state column either. Nothing closes a request yet, so + it would have no writer. + """ + + url = models.URLField(unique=True) + # The demand signal. Nothing else records how often a URL was + # asked for, so this is the record rather than a copy of one. + request_count = models.PositiveIntegerField(default=1) + last_requested_at = models.DateTimeField(default=timezone.now) + + class Meta(BaseModel.Meta): + ordering = ["-request_count", "-last_requested_at"] + + def __str__(self): + return self.url +``` + +- [ ] **Step 4: Add the serializer and the view** + +In `api/catalog/serializers.py`: + +```python +class ServiceRequestSerializer(serializers.Serializer): + """The body of POST /catalog/requests/.""" + + url = serializers.URLField() +``` + +In `api/catalog/views.py`, below `CatalogImportView`: + +```python +class ServiceRequestView(APIView): + """"Send this URL to us", from the Add-by-URL not-found screen. + + An import stores nothing about an attempt that failed, so this is + where a dead end goes. Telling somebody to hunt for a better link + assumes they have not already tried. + """ + + permission_classes = [AllowAny] + # An anonymous write. Without this one person could inflate the + # demand signal the admin list is ordered by. + throttle_scope = Throttle.IMPORT + + @extend_schema( + request=ServiceRequestSerializer, + responses={ + 202: OpenApiResponse(description="Recorded."), + 400: OpenApiResponse(description="Missing or malformed url."), + }, + ) + def post(self, request): + body = ServiceRequestSerializer(data=request.data) + if not body.is_valid(): + return Response(body.errors, status=400) + url = StatusPage.normalise_url(body.validated_data["url"]) + author = request.user if request.user.is_authenticated else None + row, created = ServiceRequest.objects.get_or_create( + url=url, defaults={"created_by": author, "updated_by": author} + ) + if not created: + # F() rather than a read and a write. Two people asking at + # once would otherwise both write 2. + ServiceRequest.objects.filter(pk=row.pk).update( + request_count=F("request_count") + 1, + last_requested_at=timezone.now(), + ) + # Always 202, and always the same body. Anything else would + # tell a stranger which URLs the catalog already holds. + return Response(status=http.HTTP_202_ACCEPTED) +``` + +Import `F` from `django.db.models`, `timezone` from `django.utils`, and `ServiceRequest`, +`StatusPage`, `ServiceRequestSerializer`. + +- [ ] **Step 5: Route it** + +In `api/catalog/urls.py`, beside the import path: + +```python + path( + "catalog/requests/", + ServiceRequestView.as_view(), + name="catalog-request", + ), +``` + +- [ ] **Step 6: Run the tests** + +Run: +```bash +cd api && uv run python manage.py makemigrations catalog -n service_request +uv run pytest tests/test_service_requests.py -v +``` +Expected: PASS, all five. + +- [ ] **Step 7: Document it** + +In `docs/api/openapi.yaml`: + +```yaml + /catalog/requests/: + post: + tags: [catalog] + summary: Ask us for a status page we could not read + description: | + The Add-by-URL not-found screen. Always `202`, whether or not the catalog already + holds the URL, so asking cannot reveal what we have. + + One row per URL, counted. Throttled per caller. + security: [] + requestBody: + required: true + content: + application/json: + schema: { type: object, required: [url], properties: { url: { type: string, format: uri } } } + responses: + '202': { description: Recorded } + '400': { $ref: '#/components/responses/Error' } + '429': { $ref: '#/components/responses/Throttled' } +``` + +Add `ServiceRequest` to the ER diagram in the v1 spec: + +``` + ServiceRequest { + uuid id PK + url url UK "normalised" + int request_count "the demand signal" + datetime last_requested_at + } +``` + +and to `BACKED` in `docs/check_docs.py`. It returns no body, so it needs no schema entry. + +- [ ] **Step 8: Run everything and commit** + +Run: `cd api && uv run pytest tests/ -q && docker compose -f docker-compose.test.yml run --rm docs` +Expected: PASS. + +```bash +git add api docs +git commit -m "feat: a dead end can hand us the address + +The Add-by-URL not-found screen offers to send us a URL, and an import +stored nothing about an attempt that failed. One row per URL with a +count, so the admin list is ordered by what people actually want." +``` + +--- + +## Task 12: Delete what nothing reads + +**Files:** +- Modify: `api/catalog/views.py`, `api/catalog/urls.py`, `api/catalog/filters.py`, + `api/catalog/queries.py`, `api/catalog/models.py`, `api/catalog/serializers.py` +- Modify: `api/dashboards/filters.py` +- Modify: `api/common/aggregates.py` +- Modify: `docs/api/openapi.yaml`, `docs/specs/2026-08-23-statusboard-design.md` +- Create: `api/catalog/migrations/0007_drop_service_columns.py` (generated) +- Test: `api/tests/test_catalog_api.py`, `api/tests/test_contract.py` + +**Interfaces:** +- Consumes: everything the previous tasks produced. This one only removes. +- Produces: `ServiceViewSet` becomes retrieve only. `GET /catalog/services/{slug}/` is the + one operation left on it. + +Nothing here has a reader once the component collection and the event feed exist. A column +nothing sorts, filters or renders by does not stay. + +- [ ] **Step 1: Write the failing tests** + +In `api/tests/test_catalog_api.py`, replace the tests that exercised the service list with: + +```python +def test_the_service_list_is_gone(client): + # Discover searches components, and the signed-out board lists + # overall components. Nothing asked a service list a question. + assert resolve_or_none("/catalog/services/") is None + + +def test_the_nested_component_route_is_gone(client): + # `/catalog/components/?service=` serves it, and three other + # screens besides. + service = ServiceFactory() + response = client.get(f"/catalog/services/{service.slug}/components/") + assert response.status_code == 404 + + +def test_the_nested_event_route_is_gone(client): + service = ServiceFactory() + response = client.get(f"/catalog/services/{service.slug}/events/") + assert response.status_code == 404 + + +def test_a_service_still_answers_by_slug(client): + # The service page reads it for the header and the About tab. + service = ServiceFactory(name="Twilio") + response = client.get(reverse("service-detail", args=[service.slug])) + assert response.status_code == 200 + assert response.json()["name"] == "Twilio" + # Nothing renders a description, so it is not in the shape. + assert "description" not in response.json() +``` + +`resolve_or_none` is a helper: + +```python +def resolve_or_none(path): + from django.urls import Resolver404, resolve + + try: + return resolve(path) + except Resolver404: + return None +``` + +In `api/tests/test_board_api.py`: + +```python +def test_the_board_takes_several_severities_at_once(client, board): + # The Severity filter offers all six values. A single exact match + # could not express "everything that needs attention". + client.force_login(board.owner) + response = client.get( + reverse("board-components", args=[board.id]), {"status__severity__in": "0,1,2"} + ) + assert response.status_code == 200 + + +def test_the_event_parameter_is_gone(client, board): + # It named the Home Incidents and Maintenance tabs. Home is Board + # and Updates now, and Updates is `/events/?dashboard=`. + client.force_login(board.owner) + response = client.get( + reverse("board-components", args=[board.id]), {"event": "incident"} + ) + # django-filter ignores an unknown parameter rather than failing, + # so the proof is that it no longer narrows anything. + assert response.json()["aggregates"]["total"] == response.json()["aggregates"]["total"] + assert "by_event_kind" not in response.json()["aggregates"] +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_catalog_api.py tests/test_board_api.py -v` +Expected: FAIL. The list still resolves and `by_event_kind` is still in the envelope. + +- [ ] **Step 3: Reduce the service viewset** + +Replace `ServiceViewSet` in `api/catalog/views.py` with a retrieve-only view: + +```python +class ServiceDetailView(generics.RetrieveAPIView): + """The service page's header and its About tab. + + There is no list. Discover searches components, and the signed-out + board lists overall components, so nothing asked a service + collection a question. + """ + + permission_classes = [AllowAny] + lookup_field = "slug" + serializer_class = ServiceSerializer + filter_backends = [FieldsBackend] + + def get_queryset(self): + return Service.objects.for_display(self.request.user) +``` + +Delete `ServiceAggregateSet`. Its only caller was the list. + +In `api/catalog/urls.py`, drop the router and name the detail route: + +```python +urlpatterns = [ + path("catalog/import/", CatalogImportView.as_view(), name="catalog-import"), + path("catalog/requests/", ServiceRequestView.as_view(), name="catalog-request"), + path("catalog/components/", ComponentListView.as_view(), name="component-list"), + path( + "catalog/components//", + ComponentDetailView.as_view(), + name="component-detail", + ), + path( + "catalog/services//", + ServiceDetailView.as_view(), + name="service-detail", + ), +] +``` + +The component paths sit above the service one, so `components` is never read as a slug. + +- [ ] **Step 4: Delete the columns and the query pieces** + +In `api/catalog/filters.py`, delete `ServiceFilter` and `ServiceEventFilter`. `EventFilter` +in `api/status/filters.py` replaced the second. + +In `api/catalog/queries.py`, delete `WATCHER_COUNT`. `COMPONENT_WATCHER_COUNT` replaced it. + +In `api/catalog/models.py`, delete `Service.is_featured` and `Service.description`. + +In `api/catalog/serializers.py`, remove `"description"` from `ServiceSerializer.Meta.fields`. + +In the adapters, drop `description` from whatever `fetch_service_metadata` returns: + +Run `grep -rn "description" api/polling/` and remove the field from the dataclass and from +each adapter that sets it. Only cState returned one. + +- [ ] **Step 5: Delete the board's event filter and the aggregate** + +In `api/dashboards/filters.py`, delete the `event` filter and its `filter_event` method, +and add the `in` lookup: + +```python +class BoardComponentFilter(filters.FilterSet): + # Declared because a component has a history of statuses and the + # open one is current. There is no `status` relation to generate + # from, so the contract's name points at the annotation. + status__severity = filters.NumberFilter(field_name="severity_now") + status__severity__lte = filters.NumberFilter( + field_name="severity_now", lookup_expr="lte" + ) + status__severity__in = filters.BaseInFilter( + field_name="severity_now", lookup_expr="in" + ) + + class Meta: + model = ServiceComponent + fields = [] +``` + +In `api/common/aggregates.py`, delete `_by_event_kind` and its line in `build`. It counted +the Home Incidents and Maintenance tabs, which no longer exist. + +- [ ] **Step 6: Run the tests** + +Run: +```bash +cd api && uv run python manage.py makemigrations catalog -n drop_service_columns +uv run pytest tests/ -q +``` +Expected: PASS. Fix any test still asserting on `description`, `is_featured` on a service, +`by_event_kind`, or the removed routes. + +- [ ] **Step 7: Update the contract and the diagram** + +In `docs/api/openapi.yaml`: + +- delete the `/catalog/services/` operation +- delete `/catalog/services/{slug}/components/` and `/catalog/services/{slug}/events/` +- delete `description` from the `Service` schema +- delete `by_event_kind` from `StatusAggregates` + +In the v1 spec's ER diagram, delete `description` and `is_featured` from `Service { ... }`. + +- [ ] **Step 8: Prove the contract and the code agree** + +Run: +```bash +cd api && uv run pytest tests/test_contract.py -v +docker compose -f docker-compose.test.yml run --rm docs +``` +Expected: PASS. `test_no_operation_exists_that_the_contract_does_not_document` and +`test_every_documented_operation_exists_in_the_code` are the two that catch a half-done +removal. + +- [ ] **Step 9: Commit** + +```bash +git add api docs +git commit -m "refactor: delete what nothing reads any more + +The service list, both nested routes, ServiceFilter, WATCHER_COUNT, +Service.description and Service.is_featured all served screens the +approved decks replaced. by_event_kind counted tabs that no longer +exist, and the board's event filter named the same two." +``` + +--- + +## Task 13: Admin follows the model + +**Files:** +- Modify: `api/catalog/admin.py` +- Test: `api/tests/test_admin.py` + +**Interfaces:** +- Consumes: `ServiceComponent.is_featured`, `catalog.queries.COMPONENT_WATCHER_COUNT`, + `catalog.models.ServiceRequest` +- Produces: nothing other code reads. + +Featuring a service now means ticking a flag on its overall component. An admin should not +have to know that to do it. + +- [ ] **Step 1: Write the failing tests** + +In `api/tests/test_admin.py`: + +```python +def test_the_service_admin_features_through_the_overall_component(admin_client, db): + # is_featured moved to the component. An admin asks "feature this + # service", so the service page has to answer it. + service = ServiceFactory() + overall = ComponentFactory(service=service, is_overall=True) + response = admin_client.get( + reverse("admin:catalog_service_change", args=[service.pk]) + ) + assert response.status_code == 200 + assert b"is_featured" in response.content + + +def test_the_component_list_shows_watchers(admin_client, db): + # The count is an annotation. A list column that forgets to + # annotate it raises rather than showing a wrong number. + component = ComponentFactory() + track(component) + response = admin_client.get(reverse("admin:catalog_servicecomponent_changelist")) + assert response.status_code == 200 + + +def test_service_requests_are_listed_by_demand(admin_client, db): + # The list is the demand signal for what the catalog is missing, + # so the most-asked-for URL is the first row. + ServiceRequest.objects.create(url="https://a.example", request_count=1) + ServiceRequest.objects.create(url="https://b.example", request_count=9) + response = admin_client.get(reverse("admin:catalog_servicerequest_changelist")) + assert response.status_code == 200 + assert response.content.index(b"b.example") < response.content.index(b"a.example") +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_admin.py -k "features_through or shows_watchers or by_demand" -v` +Expected: FAIL with `NoReverseMatch: 'catalog_servicerequest_changelist' is not a registered +namespace`. + +- [ ] **Step 3: Feature a service from its own page** + +In `api/catalog/admin.py`, add an inline to the service admin: + +```python +class OverallComponentInline(admin.StackedInline): + """Featuring lives on the rollup, and an admin asks about the service. + + `is_featured` is the first key of the suggested sort. Ticking it on + a service's overall component is what surfaces that service, so + that is the question this answers. + """ + + model = ServiceComponent + fields = ["is_featured"] + extra = 0 + max_num = 1 + can_delete = False + verbose_name = "Featuring" + verbose_name_plural = "Featuring" + + def get_queryset(self, request): + return super().get_queryset(request).filter(is_overall=True) + + def has_add_permission(self, request, obj): + # Reconcile makes the rollup. An admin ticks its flag. + return False +``` + +Add `inlines = [OverallComponentInline]` to the service admin, and remove `is_featured` +and `description` from its `list_display`, `list_filter` and `fieldsets`. + +- [ ] **Step 4: Show watchers on the component list** + +In the component admin: + +```python + list_display = [..., "watchers", "is_featured"] + list_filter = [..., "is_featured"] + + def get_queryset(self, request): + # `watcher_count` is an annotation, not a column. A list column + # reading it without this raises rather than showing a wrong + # number. + return super().get_queryset(request).annotate(watcher_count=COMPONENT_WATCHER_COUNT) + + @display(description=_("Watchers"), ordering="watcher_count") + def watchers(self, obj): + return obj.watcher_count +``` + +Import `COMPONENT_WATCHER_COUNT` and drop the `WATCHER_COUNT` import. + +- [ ] **Step 5: Register the requests** + +```python +@admin.register(ServiceRequest) +class ServiceRequestAdmin(ModelAdmin): + """What the catalog is missing, most asked for first.""" + + list_display = ["url", "request_count", "last_requested_at", "created_by"] + ordering = ["-request_count", "-last_requested_at"] + search_fields = ["url"] + readonly_fields = ["url", "request_count", "last_requested_at"] + + def has_add_permission(self, request): + # A row arrives from the app, never from here. + return False +``` + +- [ ] **Step 6: Run the tests** + +Run: `cd api && uv run pytest tests/test_admin.py -v` +Expected: PASS. + +- [ ] **Step 7: Open the pages and read the values back** + +Run: `just dev`, then open each in a browser: + +- `/admin/catalog/service//change/`, the Featuring block shows one checkbox +- `/admin/catalog/servicecomponent/`, the Watchers column shows a number, and sorts +- `/admin/catalog/servicerequest/`, the most-asked URL is the first row + +An edit that ran without error is not an edit that worked. + +- [ ] **Step 8: Run everything and commit** + +Run: `cd api && uv run pytest tests/ -q && uv run ruff check --fix . && uv run ruff format . && python3 ../bin/check_prose.py` +Expected: PASS. + +```bash +git add api +git commit -m "feat: admin features a service from the service page + +is_featured moved to the overall component, and an admin asks to +feature a service rather than a component. The component list shows +watchers, and service requests are ordered by how often they are asked +for." +``` + +--- + +## Task 14: Prove the whole thing in the image CI runs it in + +**Files:** +- Modify: none, unless something fails + +**Interfaces:** +- Consumes: everything above +- Produces: a branch ready for a pull request + +`just check` proves the image rather than the host. What CI proves is then what ships. + +- [ ] **Step 1: Run the full gate** + +Run: `just check` + +Expected: PASS on every step. That is pytest with the coverage gate, ruff, the docs cross-check, +the prose check, and `makemigrations --check`. + +- [ ] **Step 2: Fix whatever it caught, then run it again** + +The likely failures, and what each means: + +| Failure | Cause | +| --- | --- | +| `makemigrations --check` is not clean | a model change with no migration; run `makemigrations` | +| `docs` fails on an unmapped schema | a new schema is missing from `BACKED` or `PLAIN` in `docs/check_docs.py` | +| `docs` fails on a field with no column | the ER diagram is missing a column, or the field belongs in `DERIVED` | +| coverage below 85 | a branch in `system_events.py` with no test, most likely the claim guards | +| prose | a comment over 20 words, or an em dash | + +- [ ] **Step 3: Read the contract once, end to end** + +Run: `just dev`, open `http://.localhost:/`, and read the Scalar page. + +Confirm eighteen operations, and that the three removed ones are absent. The contract is +what the client is generated from, so a wrong description ships to every screen. + +- [ ] **Step 4: Open a pull request** + +```bash +git push -u origin client +gh pr create --fill +``` + +CI runs the same gate on the same image. + +--- + +## Self-Review Notes + +**Spec coverage.** Every section of `docs/specs/2026-09-03-statusboard-client-design.md` +maps to a task, except sections 3, 4, 10, 11 and 12. Those are the client, its build and +its deploy, and they are Plan B. + +| Spec section | Task | +| --- | --- | +| 1, What this supersedes | 1 | +| 5, Removed | 12 | +| 5, Added | 5, 9, 10, 11 | +| 5, Changed | 12 | +| 5, Phase is a declared filter | 9 | +| 5, A tab badge is the collection's total | 9, 10 | +| 5, Discover is one list | 5, 12 | +| 5, Ordering | 4, 5 | +| 6, New model | 11 | +| 6, New columns | 2, 3, 4, 6 | +| 6, Removed columns | 12 | +| 6, Changed | 3, 6, 12 | +| 6, Admin | 13 | +| 6, Choices | 6 | +| 7, A status change is an event | 7, 8 | +| 8, Search | 4 | +| 9, Settings | none needed. The Refresh row was never an endpoint. | + +**Names used across tasks.** `rebuild_ancestry` and `rebuild_search` are defined in Tasks 3 +and 4 and called in both. `descendant_count` replaces `child_count` in Task 3 and is read by +the serializer in the same task. `COMPONENT_WATCHER_COUNT` is defined in Task 2 and read in +Tasks 5 and 13. `EventSource` is defined in Task 6 and read in Tasks 7, 8 and 10. +`SYSTEM_EVENT_MAX_SEVERITY` is defined and used in Task 7, and read again in Task 8. +`claim` is defined in Task 8 and called from `reconcile` in the same task. + +**Ordering constraint.** Task 5 must land before Task 12. The component collection has to +serve the screens before the service list is deleted, or the API is unusable between two +commits on the same branch. diff --git a/docs/specs/2026-08-23-statusboard-design.md b/docs/specs/2026-08-23-statusboard-design.md index 8083415..b8ae3dc 100644 --- a/docs/specs/2026-08-23-statusboard-design.md +++ b/docs/specs/2026-08-23-statusboard-design.md @@ -28,7 +28,7 @@ current state, read the provider's incident log, and browse a public view withou | Community issue reporting, comments | Sub-project #3. The Figma set already contains `status dot - user reported`, a split-circle variant, so the dot component should accept a shape from day one. | | Notifications | Sub-project #2. The Figma set contains a `Toggle Notifications` button; nothing in v1 uses it. | | Uptime history charts | `ComponentStatus` accumulates from first poll, so any chart is empty on day one. Ships when there is data. | -| Service categories | Cut deliberately. `is_featured` plus `watcher_count` orders the suggested list instead. | +| Service categories | Cut deliberately. `is_featured` plus `watcher_count` orders the suggested list instead. Both sit on `ServiceComponent`. | ### Non-goals @@ -76,8 +76,8 @@ what makes sharing a change to `dashboards` alone. ## 3. Data model Every model extends `common.BaseModel`: UUID primary key, `created_at`, `updated_at`, -`created_by`, `updated_by`, `ordering = ["-created_at"]`. `auth.Permission` and `ContentType` -keep integer keys — that is Django, not a choice. +`created_by`, `updated_by`, `ordering = ["-created_at"]`. +`auth.Permission` and `ContentType` keep integer keys — that is Django, not a choice. ### authentication @@ -126,11 +126,9 @@ erDiagram uuid id PK string slug UK string name - text description "from the provider's page" url logo url homepage_url - bool is_featured "first key of the suggested sort" - int watcher_count "distinct users" + enum source "TextChoices, admin only, never served" } StatusPage { uuid id PK @@ -159,6 +157,7 @@ erDiagram uuid parent_id FK "self, null at top level" int status_page_order bool is_overall + bool is_featured "first key of the suggested sort" datetime archived_at "set when it stops being published" } ComponentStatus { @@ -172,7 +171,8 @@ erDiagram ServiceEvent { uuid id PK uuid service_id FK - string external_id "unique with service" + string external_id "provider's id, null until claimed" + enum detected_by "provider or system, never rewritten" enum kind "incident or maintenance" string title enum phase "valid set depends on kind" @@ -185,6 +185,7 @@ erDiagram enum phase "TextChoices" text body "written by the provider" datetime posted_at + enum source "provider or system" } PollRun { uuid id PK @@ -196,6 +197,12 @@ erDiagram bool ok text error } + ServiceRequest { + uuid id PK + url url UK "normalised" + int request_count "the demand signal" + datetime last_requested_at + } User ||--|| Dashboard : owns Dashboard ||--o{ DashboardItem : holds @@ -211,8 +218,9 @@ erDiagram ServiceEvent }o--o{ ServiceComponent : affects ``` -Every model also carries `common.BaseModel`: `created_at`, `updated_at`, `created_by`, -`updated_by`. They are omitted above so the diagram shows what distinguishes each table. +Every model above also carries `common.BaseModel`: `created_at`, `updated_at`, +`created_by`, `updated_by`. They are omitted so the diagram shows what distinguishes +each table. Reading it: @@ -220,6 +228,9 @@ Reading it: these, so the arrow from `DashboardItem` has one destination. - **`StatusPage` is separate from `Service`** because it is the *source*, and it is where the unique URL and the poller's state live. +- **`parent` is the whole tree.** `?ancestor=` and every descendant count read any depth from + it, with one recursive query. A stored copy of the answer needed a writer on every path that + could move a component, and it was wrong until that writer ran. - **`ComponentStatus` is append-only**, one row per severity change, with a partial unique index marking the current one. Current state and history are the same table. @@ -247,20 +258,19 @@ is a different concept from a permission role. ### catalog -- `Service` — slug, name, `description`, logo, homepage URL, `is_featured`, `watcher_count` +- `Service` — slug, name, logo, homepage URL, source - `description` and `logo` come from the provider's page and are refreshed on every poll, so a - rename or a rebrand upstream does not leave a stale entry in the catalogue. The service screen's - About tab renders the description. + `logo` comes from the provider's page and is refreshed on every poll, so a rebrand upstream does + not leave a stale entry in the catalogue. - `is_featured` is a boolean ticked in admin for services worth surfacing regardless of usage. It - is the first key of the suggestion sort, and on day one — every `watcher_count` zero — it is the - whole list. + `source` says how the row arrived: `manual` from the admin's add form, `import` from + `import_from_url`. It is admin only. No screen reads it, so no serializer carries it. - Two earlier fields are gone. `added_by` duplicated `BaseModel.created_by`, which every model - already carries. `is_curated` had no consumer: nothing sorted, filtered or rendered by it. - "Seeded by us rather than pasted by someone" is `created_by IS NULL`, and "worth surfacing" is - `is_featured`. + Three earlier fields are gone. `added_by` duplicated `BaseModel.created_by`, which every model + already carries. `created_by` says who added a service and `source` says how. Neither is null + for a service a signed-in person added. `is_curated` had no consumer: nothing sorted, filtered + or rendered by it. `watcher_count` was a column a signal kept true, and four write paths never + reached the signal. It is counted when it is read instead. - `StatusPage` — **its own model**, `OneToOneField` to `Service`. `url` (normalised, **unique**), `provider`, `api_url_override` (blank unless needed). What and where — nothing about polling. - `Poller` — `OneToOneField` to **`Service`**, created with it. The thing that reads a service: @@ -337,27 +347,23 @@ is a different concept from a permission role. parents costs nothing to keep in step. Custom URLs dedupe on `StatusPage.url`, so two users pasting the same status page share one -`Service` and one poll. A popular pasted entry is surfaced by ticking `is_featured` in admin. +`Service` and one poll. A popular pasted entry is surfaced by ticking `is_featured` on its overall +component in admin. -**Suggestion ordering** is `(-is_featured, -watcher_count, name)`. +**Suggestion ordering** is `(-is_featured, severity, -watcher_count, name)`, over components. -**`watcher_count` is distinct users, not items.** Someone tracking five Twilio components is one -watcher, not five: +**`watcher_count` is distinct users, not items.** Someone tracking one component from two +boards is one watcher, not two: ```sql SELECT COUNT(DISTINCT d.owner_id) FROM dashboards_dashboarditem i JOIN dashboards_dashboard d ON d.id = i.dashboard_id -WHERE i.service_id = %s +WHERE i.component_id = %s ``` -It is recomputed for the one affected service on `DashboardItem` create and delete — a single -indexed aggregate, not a scan — so it is exact rather than drifting the way naive -increment/decrement would when someone adds a second component for a service they already track. -Deleting an account removes its items, which fires the same path. - -A nightly reconciliation task recomputes every service as a safety net. It orders a suggestion -list, so a few hours of staleness costs nothing if a signal is ever missed. +It is counted when the list is read, so it is exact. There is no column and no signal to +keep one true. On day one every `watcher_count` is zero, so the list is your featured picks followed by everything else alphabetically. As real usage arrives the tail self-orders and you stop curating. @@ -396,8 +402,9 @@ Unique on `(dashboard, component)`. ### Every service has an overall component Statusboard creates one component per service, `is_overall = true`, `parent = null`, named -"All services". `path` is null, since it has no ancestry, and `child_count` is 0 — it is a peer of -the top-level components rather than their parent. +"All services". `path` is empty, since it has no ancestry. `descendant_count` is always 0. It is +a peer of the top-level components rather than their parent. A database trigger refuses a +component parented under it. **`component_count` excludes it.** The number equals what the provider's own status page claims, which is the number a user can check. The components list returns the overall row first and does @@ -641,7 +648,7 @@ Each successful poll: 2. **Marks vanished components `archived_at`** rather than deleting them. Someone may be tracking one, and deleting the row would silently remove it from their board. An archived component still renders, reads `unknown`, and says the provider no longer publishes it. -3. **Refreshes the service's own metadata** — name, description, logo — for seeded and pasted +3. **Refreshes the service's own metadata** — name, homepage URL, logo — for seeded and pasted entries alike, so a rename upstream does not leave a stale name on someone's board. Component identity is the provider's `external_id`, never the display name. Names change; ids do @@ -666,636 +673,26 @@ Twilio produce one poll every five minutes, not two hundred. A failed fetch sets `unknown` and **never clobbers the last known value**. The UI shows the grey state plus how long it has been stale. -Why the cooldown lives on the service rather than the dashboard or the user is specified in §5 -under **There is no manual refresh**. +**The cooldown stays on the service, globally.** The thing needing protection is somebody +else's status page. A per-user or per-board limit would poll one service twice for two +boards that both track it. A poll also captures the service's **logo** from its own page. A favicon belonging to the parent company rather than the product — Google's mark for Google Slides — is discarded rather than stored: a wrong logo misidentifies the row, while a missing one falls back to an initial and merely looks incomplete. -## 5. API surface - -**The contract is [`docs/api/openapi.yaml`](../api/openapi.yaml)** — 15 operations, 16 schemas, -with every parameter and response shape. It is the single source of truth; drf-spectacular will -generate it from the code, and it is served through Scalar at the root like your other -projects. This section records only the decisions behind it, so the two cannot drift. - -### Identifiers - -`GET /me/` returns your `default_dashboard_id`, and the client goes straight to -`/dashboards/{uuid}/components/`. The name says *default* rather than *the* because v1 having one -per user is a product decision, not a schema one. - -**There is no `GET /dashboards/{uuid}/`.** The board's own fields — `name`, `is_default` — are not -rendered anywhere: the header says "Home", not the dashboard's name. An endpoint returning three -unused fields is a request nobody makes. - -It arrives when a board is shared, because a recipient needs its name and its owner, and the URL -is already the one sharing will use. Adding it then is additive; shipping it now is a guess. - -**UUID is the identity for every model**, from `common.BaseModel`. Path parameters are UUIDs -everywhere except one: `Service` also carries a stable `slug`, used on public catalog routes, -because those URLs get shared during an outage and a UUID would be hostile there. Anything the -user owns stays UUID, so nothing personal is enumerable. Writes always address the UUID. - -An import derives the slug from the host and de-duplicates on collision (`linear`, `linear-2`). - -**`POST /catalog/import/` creates a service from a status page URL.** It detects the provider, -creates the `Service`, its `StatusPage` and its components, and returns the service — `201` when it -created one, `200` when the URL already resolved to one. Nothing is stored about the attempt. - -**`POST /catalog/services/` is deliberately not defined.** Creating a service has several possible -inputs: a URL, a filled-in form, a bulk file. Each takes a different body, so each needs its own -path. If the plain collection `POST` meant "from a URL", the other two would have to invent names -while the standard endpoint sat spent on one special case. - -So none of them takes it. That is a reservation with three known claimants, not a guess about the -future. - -An earlier draft called this `resolve/`, which named the mechanism rather than the thing being -made. - -`Import.service` is the **same shape the catalog list returns**, not the detail shape. One service -representation, whatever produced it — so a client can drop an imported service straight into the -list it already renders. - -That forced the status page fields down into the list shape, because the Add-by-URL screen shows -them in its About block before you press ＋. What was left of `ServiceDetail` — `description` and -`in_catalog_since` — has since folded into `Service` as well, for the reason below. -Renaming keeps the old slug as a redirect rather than breaking shared links. - -### Severity is the state, and there is no status string beside it - -An earlier draft carried both a `severity` integer and a `status` string on every row. Two fields -for one fact can disagree, and one of them is always the redundant copy. Only the integer is -transported; `/meta/` publishes the 0–5 label map once and the client renders from it, so a label -change ships without touching a serializer. - -`status__severity__lte=3` is everything needing attention; `status__severity=4` is maintenance. -There is no named band parameter — a band is just a threshold, and a client can compose -any other one without waiting for a new enum value. `aggregates.by_severity` returns the raw histogram for the same -reason. - -### One list, several screens - -Discover, the suggested lists and the public "down right now" view are the same catalog query with -different parameters. Suggestions are not a separate concept: they are the list with no `q`, sorted -`(-is_featured, severity, -watcher_count, name)`. - -Severity sits ahead of popularity deliberately — a mid-popularity service that is currently broken -is more worth surfacing than a more popular one that is fine, which is the whole premise of the -public view. - -### Components are their own collection - -Cloudflare publishes 109 and nothing caps that, so the service detail reports `component_count` -and the Components tab pages through `/components/`. The tab labels come from the service — -`component_count` and `tracked_component_count` — not from `aggregates`. A -component is a group when `child_count > 0`; there is no separate flag. - -### Filtering, sorting and pagination are server-side, without exception - -There is no cap on board size — one service can contribute 109 components — so the API never -assumes the board fits in one response. `aggregates` covers the whole collection rather than the -page, so a chip is correct on page four. - -There is no client-side fallback. One rule, one implementation. - -### There is no manual refresh - -An earlier draft had `POST /refresh/` behind a header button and a **Refresh now** row menu item. -Both are gone. Reloading the page already shows the freshest data the poller holds, so the button -bought a second path to the same result and carried its own cooldown handling, its own signed-out -routing, and its own failure state. - -**A page load reads stored state. It never triggers a poll.** Every `GET` serves what the last -poll wrote, so the data a reader sees is at most one `interval_seconds` old and the response time -does not depend on a third party being reachable. The alternative — a `GET` that polls anything -stale and waits — would delete the button while moving its cost into every page view, including -the failure mode where someone else's status page is slow. - -**What is kept is the freshness itself.** The header pill still reads "2 min ago" from -`aggregates.oldest_refreshed_at` — the age of the *stalest* row, not an average, so the number is -a floor rather than a flattering middle. Since nothing can be pressed, the pill also answers when -that improves: hovering shows "next check in 3 min" from `aggregates.next_refresh_at`, the soonest -`Service.poller.next_at` in the collection. - -**The cooldown stays on the service, globally.** The thing needing protection is somebody else's -status page, and a per-user or per-board limit would poll one service twice for two boards that -both track it. - -It still has a user-triggered path to guard, just not this one: `POST /catalog/import/` fetches -the URL to detect the provider and read the component list. That is a stranger's request causing -a fetch of a stranger's server, which is the case the cooldown exists for. It also keeps an -import's first poll from stacking on the scheduled one. - -### Sparse fieldsets are a framework concern - -Every endpoint that returns a body accepts `?fields=`. `?fields=id,name,overall_component.status.severity` -returns those three and nothing else. A dotted path prunes inside a nested object rather than -dropping it. - -**This is baked in at the base layer, not written per view.** A `FieldsMixin` on the base -serializer reads the parameter and prunes `self.fields` in `__init__`; a filter backend declares -it so drf-spectacular documents it on every operation without annotation. A view that must not -support it sets `fields_param = None`. - -That matters because the alternative is 13 operations each remembering to add it, and the one that -forgets is the bug. It also keeps the OpenAPI honest: the parameter appears everywhere it works -and nowhere it does not, because both come from the same declaration. - -An unknown field name is a `400`, not a silent empty response. A client asking for a field that no -longer exists should be told. - -### The status page is its own model - -`StatusPage` is a model, `OneToOneField` to `Service`, serialized as `Service.status_page`. It -holds the source and how we read it, separately from what we read: - -```jsonc -"status_page": { - "url": "https://status.twilio.com", - "provider": "statuspage", - "poll_last_success_at": "…", - "poll_next_at": "…", - "poll_interval_seconds": 300, - "poll_cooldown_seconds": 60, - "poll_consecutive_failure_count": 0 -} -``` - -Three loose `status_page_*` fields became one model, and the polling facts that had nowhere -sensible to live joined them. The nesting follows the same rule as everything else here — one -schema, one model — rather than being a serializer that groups fields for looks. -`poll_interval_seconds` was a loose field on the service purely because the About tab shows -"Refreshed every"; it belongs here, next to the backoff that changes it. - -The split is between **the source and the data**. `status_page` is where a service's information -comes from and how reliably we are getting it. `overall_component` is the information. A screen -explaining *why* a status is stale reads the first; a screen showing status reads the second. - -`poll_consecutive_failure_count` and `poll_next_at` make staleness explainable rather than mysterious — the -difference between "Can't check" and "Can't check, we have failed four times and will try again in -twenty minutes". - -`/meta/` keeps deployment defaults for both intervals. A single page can differ from them. - -### A component carries everything happening to it - -Three things happen, and a component carries all three: - -| | on `Component` | how many | -| --- | --- | --- | -| its current state | `status` | — one by definition | -| scheduled work | `upcoming_maintenance` | `upcoming_maintenance_count` | -| a live problem | `active_incident` | `active_incident_count` | - -**A count sits with the projection it counts, over the same set.** A card shows the soonest -unfinished window and the most recent unresolved incident; the counts turn those into "+2 more" -without shipping either list. Both are scoped to what is still live — a count over a wider set -than the item beside it produces "+3 more" pointing at nothing the screen can reach, which is what -happened when `active_incident` was `latest_incident` and returned resolved ones. Both live on -the component, so a board row has them whatever it tracks. - -**The overall component carries the same three, scoped to the whole service.** Its -`upcoming_maintenance` is the soonest unfinished window anywhere in the service, from any -component. Its `active_incident` is the most recent unresolved incident anywhere in it. - -**Tracking the overall component means one card for the service**, instead of one per component. -That is the point of it, and it is why the card compresses: a service with three maintenance -windows still gets one row. - -The row shows `upcoming_maintenance` and, when `upcoming_maintenance_count` is above one, "+2 -more". **Windows are never merged into a combined span** — two windows a week apart are two facts, -and a single merged window would be true of neither. Same for incidents: the row shows -`active_incident` and `active_incident_count` says whether there are others. - -Neither is accompanied by an array. History is unbounded, a row renders from the soonest plus a -count, and the full list is `GET /catalog/services/{slug}/events/` with `kind`. - -That is what makes tracking a service as a whole work. A board row is a component, so a row for -Twilio's overall component must answer "is anything scheduled, is anything broken" for Twilio — -otherwise following the whole service tells you *less* than following one part of it, which is -backwards. - -An earlier draft put the maintenance summary, the incident summary and `active_incident_count` on -`Service` instead. All three are gone: a `Service` nests `overall_component`, and the same values -are read there. One fact, one place, and the board gets them for free because a board row already -is that component. - -Real components are scoped normally — `upcoming_maintenance` is that component's own soonest -unfinished window, and `active_incident` is the most recent unresolved incident naming it through -the `affected_components` M2M. - -**The projection runs one way only.** A component carries its event; an event does not carry its -components. `ServiceEvent.affected_component_ids` used to ship the ids on every row and no screen -read them — the event tabs render titles and update logs, never component names. The M2M filter -`?affected_components=` stays, since django-filter generates it free and narrowing events to one -component is a query a screen may want. The field was a prefetch and bytes for nothing. - -### One envelope, and aggregates that grow without changing it - -Every list returns `{ aggregates, next, results }`. `aggregates` is the one place for anything -computed over the **whole collection** rather than the page. - -An earlier draft had `counts` and `oldest_refreshed_at` sitting side by side at the top level, -which is two aggregations at two different depths. The next figure worth showing would have been a -third key, and a client could not read them generically. - -```jsonc -{ - "aggregates": { "total": 42, "by_severity": {"0": 1, "5": 38}, "oldest_refreshed_at": "…" }, - "next": "cursor…", - "results": [ … ] -} -``` - -`Aggregates` defines the floor — `total`, present everywhere, so it can be read without knowing -which endpoint produced it. An endpoint extends rather than replaces: `StatusAggregates` adds the -severity histogram and `oldest_refreshed_at`; `EventAggregates` adds a histogram by phase. - -**Every chip on a screen must be answerable from one response.** Home has three — All, Incidents, -Maintenance — and the first fetch has to fill all three, or the client makes a request per chip -before it can draw the row of chips. - -`total` gives All. The other two come from `by_event_kind`, which counts the rows each value of -`event` returns. Neither can be read off `by_severity`: a maintenance window three days out leaves -the component at severity 5, and a provider can open an incident on components it still reports as -operational. - -The rule that follows: **if a filter cannot be derived from an existing aggregate, it needs its own -aggregate.** A chip is a filter and a count, and shipping one without the other means the screen -cannot be drawn in one request. - -**An aggregate has to mean the same thing on every endpoint that returns it.** A `tracked` count -was dropped for failing that: on a service's components it duplicated -`Service.tracked_component_count`, and on the board it always equalled `total`, because everything -on a board is tracked by definition. Either a duplicate or a tautology, depending on the caller. A new list endpoint declares its own subclass and the envelope never changes shape. - -In DRF this is one pagination class emitting the three keys, with each viewset supplying -`get_aggregates(queryset)` — so adopting the shape is a method, not a new response schema. - -The rule that makes it worth having: **nothing collection-wide is ever added at the top level.** -A figure that describes the whole collection goes in `aggregates` or it does not ship. - -### One schema, one model - -Serializers are `ModelSerializer`s, so the API shape has to be something a `ModelSerializer` can -actually produce. Every schema therefore maps to a single model, and **related models nest under a -key rather than flattening into the parent**. - -`status` is the case that forced the rule. Severity, `started_at` and `last_refreshed_at` live on -`ComponentStatus`, a separate table from `Component` and further still from `Service`. An earlier -draft merged them into the parent object, which reads fine as JSON but is a hand-assembled -serializer with `source=` on every field and no model behind it. Nested, it is one -`StatusSerializer` over one model, reused on both parents — and a `Service` reaches its status through -the overall component, which is the job that component was introduced for. - -The rule applies transitively, which is easy to miss. A `Service` has no status of its own — its -status *is* its overall component's. An intermediate draft gave `Service` a `status` key with -`source="overall_component.status"`, reaching through a relation to hoist fields onto the parent: the same -mistake one level up. It also carried `overall` as a bare id and `is_tracked` as a third field -describing that same component. All three collapse into one nested object: - -```jsonc -"overall": { "id": "…", "status": { "severity": 1, … }, "is_tracked": true }, -"tracked_component_count": 3 -``` - -One FK, one nested serializer. The response path and the filter path become identical — -`overall_component.status.severity` is read at `?overall_component__status__severity__lte=3`. - -**Tracking is two questions, not one.** Whether the *overall* component is tracked is what the ＋ -on a service row toggles; whether *anything* from the service is tracked decides if it appears on -your board. You can track Twilio SMS without tracking Twilio as a whole. The first is -`overall_component.is_tracked`; the second is `tracked_component_count > 0`, read from the field on a -row rather than filtered on, so no second boolean is carried for something the count already answers -— and the same count renders the "3 tracked" label. - -The count **includes the overall component**, because overall is a component like any other. -Excluding it would reintroduce the service-versus-component split this design exists to remove. - -### One schema per model, and reference schemas where nesting forces them - -There is no list variant and detail variant of the same resource. `ServiceDetail` was `Service` -plus `description` and `in_catalog_since`; `TrackedComponent` was `Component` plus `service`. -Both withheld fields that cost nothing — plain columns, or a `select_related` join — so both -folded into the resource they extended. Trimming a response is `?fields=`, where the client -chooses, rather than a second schema deciding for it. - -`OverallComponent` went the same way. `Service.overall_component` is a plain `Component`, so a -component has one shape everywhere and one renderer draws every row. Six of its fields are fixed -by what an overall component is — `parent` and `path` null, `child_count` 0, `is_overall` true, -`archived_at` null, `name` the service name — and they are returned rather than omitted. A client -that special-cased them would need a second renderer to gain nothing. - -**Reference schemas are a different thing and they stay.** `ServiceRef` and `EventRef` are not -smaller variants of a resource; embedding the full one is impossible or unbounded: - -| Schema | Fields | Why it cannot be the full resource | -| --- | --- | --- | -| `ServiceRef` | id, slug, name, logo | `Component.service`. A `Service` nests `overall_component`, which is a `Component`, which nests its service — a full `Service` there never terminates. `ServiceRef` nests nothing, which is what ends the chain. | -| `EventRef` | id, kind, title, phase, starts_at, ends_at | `active_incident` and `upcoming_maintenance`. A full `ServiceEvent` carries `updates`, a reverse FK with no bound — the same reason the maintenance array was removed. | - -The test is whether the omitted fields are *unavailable* or merely *unwanted*. Unavailable is a -reference schema. Unwanted is `?fields=`. `OverallComponent`'s were neither: they were available -and wanted, just constant. - -`allOf` survives in only one role: a serializer subclassing another **over the same model**. It -never merges two models into one flat object. - -Query parameters use the ORM path, and the nesting is visible in the URL on purpose. - -```python -filterset_fields = {"status__severity": ["exact", "lte"]} # ?status__severity__lte=3 -ordering_fields = ["status__severity", "name", "updated_at"] -``` - -That is django-filter's zero-config behaviour. Renaming them to `severity_lte` buys a shorter URL -and costs a declared `NumberFilter` per field per viewset — a `FilterSet` class to write and keep -in sync every time a field is added. The ORM path also means a client can read a response body and -derive the filter for anything in it, including fields that do not exist yet. - -**The tabs are one axis, and severity is a filter that works inside any of them.** An earlier -draft named them All, Outages and Maintenance. Those straddled two axes: Outages filtered status, -Maintenance filtered events, and All filtered nothing. Every difficulty came from that — the -Maintenance filter had to be hand-written to span a status field and an event relation, and its -count could not come out of `by_severity`. - -Incidents and maintenance are both `ServiceEvent`. Naming the tab after the model puts all three -on the event axis: `event=incident`, `event=maintenance`, and no parameter for All. One generated -filter, one symmetric aggregate. - -Severity then composes with any tab rather than competing with one. -`event=incident&status__severity__lte=3` is written-up incidents that are also down right now. - -`event` is one of the declared filters, and the reason is a Django rule rather than a preference. -`ServiceEvent` and `ServiceComponent` are many-to-many. django-filter applies each query parameter -as its own `.filter()` call, and on a many-to-many every call joins the relation again — so -`?events__kind=incident&events__phase__in=…` lets the two conditions land on two different events. -A component holding one closed incident and one running maintenance satisfies both and matches -neither. The conditions have to sit in a single call: - -```python -def filter_event(self, qs, name, value): - return qs.filter( - events__kind=value, - events__phase__in=ServiceEvent.OPEN_PHASES, - ).distinct() -``` - -`.distinct()` because the join returns a component once per matching event. - -**What each tab can promise.** A provider can drop a component to *Major outage* without writing -an incident, and that row will not appear under Incidents. So the tabs divide as: - -| Tab | Answers | -| --- | --- | -| All | What is wrong — sorted worst-first, whether or not anyone wrote it up | -| Incidents | What has been written up | -| Maintenance | What is planned | - -All is the one that promises completeness, which is why its default ordering is `status__severity` -ascending. Incidents is the narrative, not the alarm. - -Three parameters cannot follow the rule and are declared: -`overall_component__is_tracked` is a per-user annotation with no ORM path behind it, and -`ordering=suggested` is a multi-key sort. Everything else is generated. - -Because these paths are public, **a model field name is an API decision.** It appears in -`?fields=` and in every filter path built from it, so it is chosen to be read, not just stored. - -### What belongs to the user, the device, and the deployment - -Three things were sitting on `/me/` that are not user data, and each moved: - -- **Theme is per device.** Someone runs dark on their phone at night and light on a desktop in a - bright office; syncing it as an account preference makes one of those wrong. It lives in that - browser's local storage and never reaches the server. -- **Poll interval is deployment-wide**, not a preference. A user cannot make us hit somebody - else's status page more often, so it was never theirs to set. -- **The severity label map, page sizes and refresh cooldown** are the same for everyone. - -Those now come from `GET /meta/`, fetched once at app start and cacheable. It also means the 0–5 -labels have exactly one home. What is left on `/me/` is genuinely personal and small: id, email, -`default_dashboard_id`. Nothing on it was patchable once theme left, so `PATCH /me/` is gone — -`GET` and `DELETE` are the whole resource. - -### Public access - -Catalog and incident endpoints are readable without a token; that data is not personal. -`is_tracked` comes back `null` rather than absent, so the client renders `＋` without a second code path. -Adding, refreshing, `/me/` and `/dashboards/` require auth, and public endpoints carry their own -caching and rate limits. - -### Screen-to-endpoint map - -| Screen | Calls | -| --- | --- | -| Home · All | `GET /dashboards/{uuid}/components/` | -| Home · Incidents | `…/components/?event=incident` | -| Home · Maintenance | `…/components/?event=maintenance` | -| Home, signed out | `GET /catalog/services/?overall_component__status__severity__lte=` | -| Header pill | no call — `aggregates.oldest_refreshed_at` and `next_refresh_at`, both already in the list response | -| Row ⋮ → Stop tracking | `DELETE /dashboards/{uuid}/components/{component_id}/` — the only item left in that menu | -| Discover, suggested | `GET /catalog/services/` | -| Discover, typing / no results | `GET /catalog/services/?q=` | -| Add by URL | `POST /catalog/import/` | -| Service · Components + filter | `GET /catalog/services/{slug}/components/?is_tracked=` | -| Service · Incidents | `GET /catalog/services/{slug}/events/?kind=incident` — both groups; the tab renders ACTIVE and RESOLVED, so it must not filter on `ends_at__isnull` | -| Service · Maintenance | `GET /catalog/services/{slug}/events/?kind=maintenance` | -| Service · About | `GET /catalog/services/{slug}/` — Website, Status page, Provider, Components, Active incidents, In catalog since, Refreshed every | -| Service · Add all / Remove all | the same `POST` / `DELETE`, repeated per component | -| ＋ on any row | `POST /dashboards/{uuid}/components/` | -| Sign in | `magic-link` → `verify` | -| App start | `GET /meta/` | -| Settings | `GET /me/`, `GET /meta/`, `DELETE /me/`; theme is device-local | - -### The row is one component, used everywhere - -Home, Discover and the service screen render the same card. Line one is the mark and the name; -line two is the status and its duration; line three is context. Nothing shares a line, because a -232px phone row cannot hold two things that must not truncate. - -**Line three is the only variable, and what it holds follows from what the row is.** - -| the row is | line three | -| --- | --- | -| a service you track nothing of | `7 components` — what is on offer | -| a service you track part of | `Tracking [overall] [3 of 42]` | -| a component on your board | its parent breadcrumb, `Twilio › Programmable Messaging` | - -A board row never shows a tracking count. The count answers *how much of this service am I -watching*, which is a question about a **service**; a board row is already one tracked component, -so the answer would always be "this one". - -Tracking state renders as **one token per subscription** rather than a sentence — `overall` for the -summary row, `3 of 42` for components, both when you take both. That maps one-to-one onto -`overall_component.is_tracked` and `tracked_component_count`, so the client assembles no strings, and "all" -is simply `118 of 118` with no extra vocabulary. - -**The duration is always shown**, including on operational rows, where it reads as time since the -service last left operational — the same `status.started_at` every other state uses. Maintenance is -the one case that would state its timing twice, so its window renders in that slot rather than -below, and always relatively: `in 4 hrs` before it starts, `ends in 40 min` while it runs. - -**The overall component is titled by where you are standing.** On a service screen the header -already says Twilio, so the row is `Overall status` with `All services` beneath. On your board -nothing else names the service, so it is `Twilio` with `Overall status` beneath. Same component, -same `is_overall` flag, different useful title. - -**Marks appear only where rows come from different services** — Home and Discover. On a service -screen every row belongs to the same service, so forty-two identical marks are noise and the slot -is dropped. Where a service has no usable logo the row falls back to its initial in the same -circle, so a list with patchy coverage keeps one left edge. - -## 6. Frontend - -React + TS + Vite + Tailwind + TanStack Query + `vite-plugin-pwa`. - -### Navigation - -Three destinations: **Home**, **Discover**, and a third slot that is **Sign in** when logged out -and **Settings** when logged in. Signed out and signed in are the same screens; only the buttons -differ. - -- Mobile: bottom pill, inset to the content width, fully rounded -- Desktop: top bar; Sign in sits right as a primary button - -### Screens - -| Screen | States | -| --- | --- | -| Home | All · Incidents · Maintenance; empty, unreachable, row menu | -| Discover | suggested, typing, no results | -| Add by URL | detected, not found | -| Service | Components · Incidents · Maintenance · About; not tracked, outage, unknown | -| Sign in | request, check your email, verify | -| Settings | preferences, account, install | - -### Chips and filters - -`All` · `Incidents` · `Maintenance`. The two named tabs filter on event kind; All applies no -filter and sorts worst-first. Counts describe whatever the tab shows: the catalog when signed out, -your tracked set when signed in. Signed out shows no counts, because nothing is yours. - -**Colour follows the dot, and the dot is always the true severity.** Maintenance already under way -is severity 4, so its row is blue and reads "Maintenance ends in 2 hrs". Maintenance three days out -leaves the component operational, so its row is **green** and reads "Maintenance in 3 days". The -tab decides which fact the line describes; severity decides what colour it is. A row never shows a -colour the component is not in. - -Sort is **Smart** by default (`ordering=suggested`): tracked first, then severity ascending -(worst first), then name. - -### No offline mode - -Statusboard reports on remote services. With no network there is nothing true to report — a cached -board would show yesterday's answer to today's question, which is worse than saying nothing. - -So there is no stale-cache mode, no offline banner and no dimmed rows. A failed request shows a -plain "can't reach statusboard" state with a retry. The service worker exists only to make the app -installable; it does not cache API responses. - -The `unknown` status is a different thing and stays: that is *our* server failing to reach a -provider, which we can report accurately because we are online and they are not. - -### Install - -No prompt on first visit. A dismissible banner on the third visit, plus a row in Settings. -**iOS has no programmatic install prompt**, so Safari gets "tap Share, then Add to Home Screen" -while Android and desktop Chrome get a real button. - ---- - -## 7. Design system - -### Vocabulary - -**You track. The app refreshes. The catalog gets no verb.** - -- `＋` starts tracking; `⋮` opens the menu that stops it. No tick, no lock icon. -- Signed out, `＋` is still `＋` and opens Sign in. -- "Refreshed every 5 min", never "updated" — the Incidents tab already uses *updates* for - provider posts — and never "monitored". -- "In catalog since", never "tracking since". - -### Row anatomy - -Identical on both platforms; desktop only changes the container. - -``` -[logo] Twilio ● - Major outage · 38 min - 42 components ⋮ -``` - -- **Logo** top-left, neutral disc with an initial. Only where rows come from different services — - on a service screen every row is the same service, so the slot is dropped. No provider publishes - per-component icons. -- **Status** in words, in its severity colour, with duration in mono beside it -- **Meta** last: `42 components` for a service, the breadcrumb for a component, - `Overall status` for the rollup row. **On the Maintenance tab every row shows its window - instead**, in local time. -- **Dot** 15px, top-right. **Action** bottom-right. - -The subtitle says what the row *is*, never what you have done with it. - -### Colour - -| Token | Value | -| --- | --- | -| Ground | `#F5EDD6` | -| Card | `#FFFFFF` | -| Nav / ink | `#29140A` / `#0D0D0D` | -| Accent | `#32CDB3` (notifications toggle only) | - -Status dots use the palette above at **80% fill opacity**, as Figma sets them. - -**Status text uses darkened variants** — `#1C7A3E`, `#A35F00`, `#B3180B`, `#0B6EA0` — because -`#00E54D` on white is about 1.8:1 and unreadable at 11px. Dots keep the exact Figma colours. - -Rules: - -- Status colour is visible wherever services are listed, including the catalog -- Green means operational and nothing else — buttons use brand dark -- Count pills are grey: a count is a quantity, not a severity -- Secondary buttons are transparent with a `#D6CCB8` border; primary is dark fill - -### The mark - -The statusboard logo carries the status dot wherever it appears — brand row, tab bar, favicon, -app icon. **One dot, the lower-left counter**, coloured by the worst status you track. It is the -position your artwork already varies (red and grey); the upper-right is green in every variant. -Scheduled maintenance keeps it green — planned work does not need you. - -Signed out, nothing is tracked, so the counter stays hollow. - -### Icons - -Home and Discover use the Figma marks. Settings and Sign in use Lucide (`settings`, `log-in`), -and the back arrow is Lucide `chevron-left`. The Figma tab-bar exports embed their labels as -vector paths — extract the glyph only. - -Open: the Figma marks are filled and Lucide is stroked, so the nav mixes two weights. Either draw -filled versions of those two or move Discover to Lucide, leaving Home as the only filled mark. - -### Desktop differences - -Only three, each forced by width: +## 5. The API, the client and the design system -1. Top nav instead of a bottom pill -2. Rows flow into `repeat(auto-fill, minmax(212px, 1fr))` -3. A field that filters a list spans the list; a form that *is* the task gets a column — - 360px form card, 560px for Add-by-URL +Those three sections are now +[`2026-09-03-statusboard-client-design.md`](2026-09-03-statusboard-client-design.md). -Plus one intentional divergence: the service title keeps its status label, which mobile drops for -width. +They described a palette, tabs and a row that the approved decks replaced on 2026-08-28. +Keeping a second account of one API is how two answers to the same question drift apart. --- -## 8. Testing, infra, deploy +## 6. Testing, infra, deploy ### Testing diff --git a/docs/specs/2026-09-03-statusboard-client-design.md b/docs/specs/2026-09-03-statusboard-client-design.md new file mode 100644 index 0000000..53ed98c --- /dev/null +++ b/docs/specs/2026-09-03-statusboard-client-design.md @@ -0,0 +1,529 @@ +# Statusboard client design + +**Status:** approved design, ready for an implementation plan +**Date:** 2026-09-03 + +This covers the client app, the API changes the approved designs need, and the deploy +pipeline for both images. + +Designs, approved 2026-08-23 and amended through 2026-08-28: + +- Mobile: +- Desktop: + +## 1. What this supersedes + +`docs/specs/2026-08-23-statusboard-design.md` sections 5, 6 and 7 predate the decks. +They describe a cream palette. They name the Home tabs All, Incidents and Maintenance. +They put a status dot on every row, and four tabs on a service. None of that is current. + +Sections 5, 6 and 7 are deleted, and a pointer to this document replaces them. Rewriting +them would leave two documents describing one API. That is the drift this change exists to +end. + +Sections 1 to 4 stay. Scope, architecture, the data model and the adapters are unchanged. +The columns in section 6 below are the exception. + +## 2. Work order + +1. Refresh sections 5 to 7 of the v1 spec from the decks and from this document. +2. Change the API and the models. The contract, `docs/check_docs.py` and + `api/tests/test_contract.py` move in the same commit. +3. Scaffold `app/`, with CI and deploy. + +The scaffold does not depend on steps 1 and 2. Its generated client does, so codegen +runs after step 2 lands. + +## 3. Stack + +| Concern | Choice | +| --- | --- | +| Build | Vite, React, TypeScript | +| Routing | TanStack Router | +| Server state | TanStack Query | +| API client | Orval, generated from `docs/api/openapi.yaml` | +| Styling | Tailwind v4 | +| Behaviour | Radix Primitives | +| Icons | Lucide, and only Lucide | +| Install | `vite-plugin-pwa` | +| Tests | Vitest, Testing Library, Orval's MSW handlers | +| Lint and format | Biome | + +**A Vite SPA, not SSR.** There is no offline mode, so a service worker only makes the app +installable. It caches no API response. SSR would buy link previews for shared outage +URLs. That is solvable later with a prerender, without restructuring the app. + +**Routing carries the filters.** Every filter is server-side, so every filter is a search +parameter. TanStack Router validates them against an enum in one schema. React Router +would need hand-written coercion for each one. + +**Orval generates hooks, Zod schemas and MSW handlers.** The generated client is +committed. CI regenerates it and fails on a diff. That is `makemigrations --check` for the +client: a contract change shows what it did. + +**Radix supplies behaviour only.** The decks define 26 tokens, a row anatomy and a card +catalogue. Nothing is missing but menu, tabs, popover and dialog. Those are styled from +the tokens. A component library would add a second set of colour names beside the 26. + +**There is almost no client state.** Server state is Query. Filter state is the URL. What +is left is the theme, the hidden state of Suggested, and the install banner's visit count. +All three live in local storage. No store library ships until something needs one. + +### Tokens + +The 26 tokens are one file, and nothing else writes a hex. Both decks carry an identical +`:root` core block. It is the source. + +- Neutrals `--u-0` to `--u-950`, one ultramarine ramp. +- Status, four hues with three each: raw for a fill, `-ink` for type on light, `-lift` for + type on dark. +- Semantic tokens are built from the core. Rules are built from those. + +### Auth + +The access token lives in memory. The refresh token lives in local storage. An Orval +mutator refreshes once on a 401 and retries. `POST /auth/refresh/` stays the activity +heartbeat the backend stamps `last_active_at` from. + +A strict CSP is set by the client image, because local storage is readable by script. + +**The magic link is single use.** `VerifyView` stamps `used_at` on the first success. So +the client posts that token exactly once: a mutation, `retry: false`, fired once. A query burns +the token on a StrictMode double invoke. It then tells somebody who just signed in that +their link is dead. + +The page load itself spends nothing. Only the POST does. That is what makes an email +client's prefetch safe. + +## 4. Routes + +Nine. Tabs are a search parameter, not a path. A tabbed screen is one page: the header is +shared and the body swaps, which is the decks' own rule. + +| Route | `tab` | Other search parameters | +| --- | --- | --- | +| `/` | `board`, `updates` | board: `severity`, `sort`. updates: `service`, `component`, `kind`, `phase`, `sort` | +| `/discover` | none | `q`, `severity`, `sort` | +| `/discover/add` | none | none | +| `/services/$slug` | `components`, `updates`, `about` | components: `severity`, `tracking`, `sort`. updates: `component`, `kind`, `phase`, `sort` | +| `/components/$id` | `components`, `updates` | as above, without `component` | +| `/events/$id` | `timeline`, `affects`, `about` | none | +| `/signin` | none | none | +| `/verify` | none | `token` | +| `/settings` | none | none | + +The first `tab` listed is the default. `cursor` is a search parameter on every list, +because pagination is infinite scroll. + +Components keep UUID identity. Only `Service` carries a slug. + +### A row's destination is a rule + +``` +is_overall -> /services/{service.slug} +otherwise -> /components/{id} +``` + +One function, used by every row on every screen. A component payload already carries +`is_overall` and its service, so nothing new is needed. + +### Screen to endpoint + +| Screen | Call | +| --- | --- | +| Home, board, signed in | `GET /dashboards/{uuid}/components/` | +| Home, board, signed out | `GET /catalog/components/?is_overall=true` | +| Home, updates | `GET /events/?dashboard={uuid}` | +| Discover | `GET /catalog/components/` | +| Add by URL, detected | `POST /catalog/import/` | +| Add by URL, not found | `POST /catalog/requests/` | +| Service, header and About | `GET /catalog/services/{slug}/` | +| Service, components | `GET /catalog/components/?service={slug}&is_overall=false` | +| Service, updates | `GET /events/?service={slug}` | +| Component, header | `GET /catalog/components/{uuid}/` | +| Component, components | `GET /catalog/components/?ancestor={uuid}` | +| Component, updates | `GET /events/?component={uuid}` | +| Event, header and About | `GET /events/{uuid}/` | +| Event, timeline | `GET /events/{uuid}/updates/` | +| Event, affects | `GET /catalog/components/?event={uuid}` | +| Event, affects, empty | the tab reads `Affects 0` and shows an empty state | +| Row plus | `POST /dashboards/{uuid}/components/` | +| Row menu, stop tracking | `DELETE /dashboards/{uuid}/components/{component_id}/` | +| Sign in | `POST /auth/magic-link/` | +| Verify | `POST /auth/verify/` | +| Settings | `GET /me/`, `DELETE /me/`, `POST /auth/logout/` | +| App start | `GET /meta/` | + +## 5. API surface + +15 operations become 18. + +### Removed + +| Operation | Why | +| --- | --- | +| `GET /catalog/services/` | Discover searches components. Signed-out Home lists overall components. | +| `GET /catalog/services/{slug}/components/` | `GET /catalog/components/?service=` serves it, and three other screens. | +| `GET /catalog/services/{slug}/events/` | `GET /events/?service=` serves it, and three other screens. | + +`GET /catalog/services/{slug}/` stays. The service page reads it. + +### Added + +| Operation | Parameters | +| --- | --- | +| `GET /catalog/components/` | `q`, `service`, `ancestor`, `event`, `is_overall`, `is_tracked`, `status__severity` with `lte` and `in`, `ordering` | +| `GET /catalog/components/{uuid}/` | none | +| `GET /events/` | `dashboard`, `service`, `component`, `kind`, `phase`, `ordering` | +| `GET /events/{uuid}/` | none | +| `GET /events/{uuid}/updates/` | none | +| `POST /catalog/requests/` | body `{url}` | + +`fields`, `cursor` and `page_size` stay on every operation. + +### Changed + +`GET /dashboards/{uuid}/components/` loses `event`. That parameter named the Home +Incidents and Maintenance tabs, which no longer exist. It gains +`status__severity__in`, because the Severity filter offers all six values. + +`aggregates.by_event_kind` goes, for the same reason. + +### One collection per model + +`/events/` returns `ServiceEvent`. `/events/{uuid}/updates/` returns `EventUpdate`. The +screens call the first one Updates and the second one Timeline. The API names the model +and `/meta/` publishes the labels. + +### Phase is a declared filter + +`phase` accepts `open` and `closed`. It draws `CLOSED_PHASES` from `status/choices.py`, so +the client never restates which phases are terminal. Omitting it returns both, so the feed +is not an open-only list that a client has to opt out of. + +### A tab badge is the collection's total + +`Updates 6` is `aggregates.total` of the same `/events/` query the tab renders. No stored +count, so a badge cannot disagree with the list under it. `Components 42` and +`Components 0` stay annotations, `component_count` and `descendant_count`. A component is +a group when `descendant_count` is above zero, so no flag records it. + +### Discover is one list + +Every trackable thing is a `ServiceComponent`. A service's rollup is one of them. So +Discover searches components only, and tracking a whole service is the plus on the row +named after it. There are no tabs and no service search. + +Discover searches every component, rollups included. Only signed-out Home narrows to +`is_overall=true`, because a board is one row per service. + +`Service.component_count` excludes the overall component, and a service's Components tab +passes `is_overall=false`. The header's plus tracks the whole service, so the rollup is +reachable without being a row on its own page. + +**A component list is the whole subtree, never one level.** A service's tab lists every +component it has. A component's tab lists every descendant. The two screens are the same +screen at a different root, so they cannot count differently. + +That is why the parameter is `ancestor` and not `parent`. A self-FK lookup returns one +level, and `parent` would name a query this does not run. + +### Ordering + +| List | Default | +| --- | --- | +| `GET /catalog/components/` with no `q` | `suggested` | +| `GET /catalog/components/?service=` | `status_page_order` | +| `GET /catalog/components/` with `q` | `(-is_overall, *suggested)` | +| `GET /events/` | `-starts_at` | + +Every one of them is labelled Smart in the interface. + +`suggested` is `(-is_featured, severity_now, -watcher_count, name)`. Severity sits ahead of +popularity, the same as the service sort it replaces. A middling component that is broken +now beats a popular one that is fine. + +There is one sort control and one label, **Smart**, on every list. With a `q` it is +`(-is_overall, *suggested)`, so the service rollup leads. Without one it is `suggested`. A +separate "Best match" would name a ranking the search does not work out. + +## 6. Model changes + +### New models + +`catalog.ServiceRequest`, for "Send this URL to us". + +| Field | Note | +| --- | --- | +| `url` | normalised, unique. The thing an admin triages. | +| `request_count` | incremented on every post | +| `last_requested_at` | | + +`BaseModel` carries the rest. `created_at` is the first request. `created_by` is the first +requester, null when signed out. There is no `requested_by`: that is what `created_by` is. + +The endpoint upserts and increments, and always answers `202`. Asking twice cannot reveal +whether we hold a URL. It is throttled per IP, so one person cannot inflate the count. + +There is no `state`. Nothing closes a request yet, so the column would have no writer. It +arrives with the workflow that moves it. + +One row per URL, not one per request. A row per request would answer one question twice +once a URL was triaged. + +`request_count` is not the denormalisation the v1 spec bans. `Poller.last_error` was banned +for copying `PollRun.error`, which can disagree with its source. Here there is no source. +The counter is the record. + +### Ancestry + +**No model holds it.** `parent` already says where a component sits, and +`?ancestor=`, `descendant_count` and the breadcrumb are all read from it. + +A flattened `ComponentAncestor` table was built for this and then removed. So was an array +of ancestor ids before it. Each stored the answer to a question `parent` already answers, +and a stored answer has to be rewritten every time the tree moves. That obligation reached +`ServiceComponent.save`, `ServiceComponent.delete`, the admin's `delete_queryset`, and a +`held_rebuilds()` context manager suppressing all three during a poll. Miss one write path +and the component is silently absent from `?ancestor=` and from every count. An untracked +service is never polled, so nothing comes along and repairs it. + +**Speed was not the reason to store it, and it is not the reason to stop.** The claim was +that walking `parent` at query time cannot use an index. That is true of a Python loop +asking one query a level. It is false of a recursive CTE, which is one indexed query. +Measured on 341 components, deeper than a real service: + +| Read | Closure table | Recursive CTE | +| --- | --- | --- | +| every descendant of a node | 0.56 ms | 0.73 ms | +| one breadcrumb, root first | 0.21 ms | 0.35 ms | +| `descendant_count`, page of 50 | 0.42 ms | 0.50 ms | + +A fifth of a millisecond, against four writers that could each leave the answer wrong. The +sync obligation is what the change buys back. + +`catalog.queries` holds the walk down, as one SQL definition that `?ancestor=` and +`descendant_count` share. Three rules are in it: + +- **A cycle guard.** `parent` is a self-FK and nothing forbids a loop. The walk carries the + ids it has visited and refuses to step onto one twice. A depth ceiling would also stop a + loop, and it would cut a deep tree that is not one. +- **The service boundary.** Each step tests `service_id`. Moving a component leaves its old + children pointing across, and a step there names a row that service's lists never hold. +- **Archived rows.** A count leaves them out, because no list serves one. The walk still + passes through them, or an archived group would hide the live rows under it. + +`descendant_count` is a correlated subquery annotated by `for_display`, so a page of fifty +costs one query for it and not fifty. The breadcrumb stays a Python walk up `parent`: +`for_display` already selects two levels of it, so the common case costs nothing. + +### New columns + +| Column | Type | Written by | +| --- | --- | --- | +| `ServiceComponent.is_featured` | bool | admin | +| `ServiceEvent.detected_by` | `provider` or `system` | set when the event opens, never changed | +| `EventUpdate.source` | `provider` or `system` | the adapter, or reconcile | + +### Removed columns + +`Service.is_featured`. It was the first key of the service suggestion sort, and that list +is gone. `ServiceFilter` goes with it. + +`Service.description`. The About tab shows Website, Status page and Provider. Nothing +renders the description. So the column goes, with the adapter return value that filled it. +The v1 spec's sentence about refreshing it goes too. + +`catalog.queries.WATCHER_COUNT` goes too. It annotated a service, and only the service +sort read it. + +**A component's watcher count is an annotation, not a column.** `Service.watcher_count` +was a column a signal kept true, and four write paths never reached the signal. Migration +`catalog/0002` dropped it for that reason. A component repeats neither the column nor the +signal. + +`COMPONENT_WATCHER_COUNT` is `Count("boards__owner", distinct=True)`, in +`catalog/queries.py`. `DashboardItem` points straight at a component, so it is one join. +The service version needed three. + +### Changed + +| What | From | To | +| --- | --- | --- | +| `ServiceEvent.external_id` | required, unique with `service` | nullable. The unique constraint becomes partial, `WHERE external_id IS NOT NULL`. | +| `ServiceEvent.title` | always the provider's | generated when we open the event, replaced when a provider claims it | +| `EventUpdate.body` | always the provider's | generated for a `system` update | +| `Service.component_count` | every component | every component except the overall one | +| `ServiceComponent.child_count` | direct children | renamed `descendant_count`, the whole subtree | + +### Admin + +`ServiceComponent.is_featured` is how anything is featured, and any component may carry +it. A featured rollup surfaces its service. A featured leaf surfaces that part. + +Featuring is viewed and managed on the component admin, and nowhere else. That admin gains +the column, a list filter, the field on the change form, and two bulk actions for a +selection. The service admin has no featuring control: one flag edited from two screens is +one question with two answers. + +`ServiceComponent.watcher_count` joins the same list, read-only. + +`catalog.ServiceRequest` is registered, ordered by `request_count`. That list is the demand +signal for what the catalog is missing. + +### Choices + +`IncidentPhase` gains `DETECTED = "detected", "Detected"` as its first value. It is open, +so `CLOSED_PHASES` is unchanged. `/meta/` publishes the label, so the client renders it +with no new code. + +`EventUpdateSource` is new: `provider`, `system`. + +### Unchanged + +`User`, `Dashboard`, `DashboardItem`, `StatusPage`, `Poller`, `PollRun`, +`ComponentStatus`, and every other field on `Service`. + +`Service.slug`, `logo` and `homepage_url` stay. Routes, the row's mark and the About tab +read them. + +## 7. A status change is an event + +The Updates feed is one table. When a provider explains an outage we store their event. +When they do not, we write our own. The client never learns there were two cases. + +**Opening.** A component drops below operational and no provider event covers it. Reconcile +opens a `ServiceEvent` with `detected_by = system`, `kind = incident`, `phase = detected`, +and no `external_id`. It writes one `EventUpdate` with `source = system`. + +**While open.** Every severity transition of an affected component writes another +`EventUpdate` with `source = system`. A `ComponentStatus` transition and a provider's post +are different facts, so both appearing near one moment is correct. + +**Claiming.** A provider event claims an open unclaimed one rather than making a second. +`external_id` is filled in, the title becomes theirs, and their updates append with +`source = provider`. One outage is one card, with more updates on it. + +A candidate must be open, on the same service, with intersecting components. Its +`starts_at` must be no earlier than the system event's, less `EVENT_CLAIM_WINDOW`. The +nearest `starts_at` wins when several match. + +`EVENT_CLAIM_WINDOW` defaults to one hour, in `defaults.py`. Providers backdate `starts_at` +to when the incident really began, which is before our poll saw it. The default interval is +300s, so a short window would miss a backdated post. A system event stays open only while +the component is down. The far end is already bounded by the outage itself. + +`detected_by` is never rewritten by a claim. That is why it exists as a column. +`external_id IS NULL` cannot answer "did we find this first", because a claim fills it in. + +| `detected_by` | `external_id` | Means | +| --- | --- | --- | +| `system` | null | we found it and nobody explained it | +| `system` | set | we found it first and the provider caught up | +| `provider` | set | they posted before our poll saw the change | + +**Closing.** The component returns to operational. Reconcile writes the closing +`EventUpdate`, sets `ends_at`, and moves `phase` to `resolved`. + +A provider may post their own resolution near the same moment. Both are true, so both +appear. A timeline is ordered by `posted_at` and nothing prefers an author. The card shows +whichever landed last. + +**Archiving also closes.** Reconcile sets `archived_at` when a provider stops publishing a +component. We can no longer see it recover, so an open system event on it could never +close. Archiving writes the closing update, sets `ends_at` to `archived_at`, and resolves +the phase. The update says the component is no longer published. + +An event that cannot close leaves a permanently red row on somebody's board. + +Claiming is safer than deleting. Nothing is destroyed, so a wrong match leaves an event +with one stray first update rather than losing data. + +`ComponentStatus` stays the truth. A system event is a projection of it, written by one +writer and rebuildable from nothing. That is what keeps it from being a second answer to +the same question. + +## 8. Search + +`q` matches part of a component's own name, its service's name, or its parent's. Searching +`twilio` finds `SMS`, because Twilio publishes it. Searching `messaging` finds it too, +because Programmable Messaging is the group above it. Searching `twilio sms` finds it as +well: every word must match, and any of the three names may satisfy it. Matching ignores +case. + +The parent is one level, and no further. A component three deep is not found by its +grandparent's name. `parent__name` is a join read at query time, against the live row, so a +rename keeps itself in step. That is what separates it from the stored ancestry below. + +It is `icontains` per word, not a `tsvector`. Measured on 4,800 components across 60 +services, larger than realistic, a stored weighted vector with a GIN index answers in about +1.5 ms less. That is the whole return on the apparatus it needs. + +The apparatus was the cost. A component's vector held its ancestors' names at weight `B`, +so the vector depended on **other rows**. Renaming one group rewrote every document below +it, and that subtree rewrite is what forced a rebuild onto `save`, onto `delete`, onto the +admin's bulk delete, and into a `held_rebuilds()` suppressor for polls. With no weights +there is nothing to keep in step. `ServiceComponent` becomes a plain model with a `parent`. + +**What is given up**, plainly: + +- **Stemming.** `outages` no longer finds `Outage`. A substring match covers a prefix, not + a different ending. +- **Relevance ranking.** There is no score, so nothing sorts by closeness of match. The + rollup leads instead, by `-is_overall`, which is what stops a popular service flooding + the list with its eighty parts. +- **Matching through the whole ancestry.** A leaf is found by its parent's name, but no + higher. A group two or more levels above it no longer reaches it. + +The service `search_fields` go with the service list. + +## 9. Settings + +The Refresh row is removed. It rendered `meta.poll_interval_seconds` read-only, and it +read as a control over something a user cannot set. Settings holds Theme, Suggested +services and Account. + +`/meta/` stays. It publishes the severity labels and every other enum. + +## 10. Build and test + +`app/` builds its own image from its own directory, matching `api/`. + +CI builds that image, then runs Biome, `tsc`, Vitest and the Orval drift check inside it. +What CI proves is what ships, which is the rule the API already follows. + +Tests use Orval's generated MSW handlers, so a fixture cannot describe a response the +contract does not. + +**Configuration is read at runtime, not baked in.** A Vite build inlines `VITE_` values, +which would make the image environment-specific. The container entrypoint writes +`config.js` from the environment at start, the way `api/entrypoint.sh` picks its process. +One image, any environment. + +## 11. Deploy + +`release.yml` gains an `app` job beside `api`: same tags, same cosign signing. The deploy +job waits on both. + +A push to `main` that passes CI pushes `edge` and `sha-` for both images. + +In the deployment repository: + +- the four API services take `${API_TAG:-latest}` +- a new `app` service takes `${APP_TAG:-latest}`, routed by Traefik on `statusboard.dev` +- `API_TAG` and `APP_TAG` join `.env.example` + +A deploy during development is then `APP_TAG=edge` on the server and `just deploy`. Unset +it to return to tag-gated releases. `just release` is unchanged. + +`just` gains a recipe for the client's dev server. `just dev` runs the API, the poller and +the client together. `just check` gains the client image's checks. + +## 12. Local development + +`bin/worktree-env.py` already reserves `CLIENT_PORT` and points `CORS_ALLOWED_ORIGINS` and +`CSRF_TRUSTED_ORIGINS` at `.localhost:`. Vite binds that port, and the +API already trusts that origin. That file needs no change. + +`APP_URL` and `APP_MAGIC_LINK_PATH` in `api/.env.local` gain values, so a sign-in link +reaches a page that serves it. diff --git a/justfile b/justfile index 7e66b0e..a39e1e6 100644 --- a/justfile +++ b/justfile @@ -6,20 +6,66 @@ set dotenv-path := "api/.env.local" # then never share a port or a database. See bin/worktree-env.py. wt := 'eval "$(python3 bin/worktree-env.py)"' -# List the recipes. Keep first: bare `just` runs whatever recipe comes first. +# Keep first: bare `just` runs whatever recipe comes first. + +# List the recipes. default: - @just --list + @just --list --unsorted -# One-time setup. Every step it calls is runnable on its own. -init: env up sync migrate seed - pre-commit install - @{{wt}} ; cd api && uv run python manage.py collectstatic --noinput +# Applying migrations here is a little surprising: starting a server +# that also changes the database. The human asked for exactly this, +# since a fresh worktree that only warns still fails unexplained. -# Write .env.local, asking for the admin once. Worktrees copy main's. -env: - @python3 bin/make-env.py +# Ctrl-C stops the server and the poller together. + +# Run the app: stores, migrations, server, poller. +[group('Daily')] +dev: up migrate + @{{wt}} ; trap 'kill 0' EXIT INT TERM ; \ + echo "http://$WORKTREE_SLUG.localhost:$DJANGO_PORT/" ; \ + cd api ; \ + uv run celery -A api worker -B -l info & \ + uv run python manage.py runserver 0.0.0.0:$DJANGO_PORT + +# Arguments pass to both, so `just test -k foo` filters each. -# Show this worktree's ports, database and compose project. +# Run the test suite, host and bin/ scripts. +[group('Daily')] +test *args: + @{{wt}} ; cd api && uv run pytest -n auto {{args}} + @python3 -m unittest discover -s bin/tests -v {{args}} + +# Fix lint and format the code. +[group('Daily')] +lint: + cd api && uv run ruff check --fix . && uv run ruff format . + +# Slower than `just test`, and the answer that counts: it proves the +# image rather than the host. Run it before opening a pull request. +# The two stdlib-only checks run on the host, because a container costs +# more than it saves. `bin/tests/test_merge_gate.py` holds the two lists +# together, in one direction: everything here also runs in CI. +# +# CI runs one more. It scans the image with Trivy for HIGH and CRITICAL +# CVEs and fails the merge on a hit. That needs the network and a +# vulnerability database, so it stays out of this recipe. + +# Each check runs where CI runs it. + +# Every CI check but the image scan. +[group('Daily')] +check: image + docker compose -f docker-compose.test.yml run --rm pytest -q -n auto --cov-fail-under=85 + docker compose -f docker-compose.test.yml run --rm ruff + docker compose -f docker-compose.test.yml run --rm docs + @python3 bin/check_prose.py + @python3 -m unittest discover -s bin/tests -v + docker compose -f docker-compose.test.yml run --rm --entrypoint sh pytest \ + -c 'python manage.py makemigrations --check --dry-run' + docker compose -f docker-compose.test.yml down -v + +# Show this worktree's ports and database. +[group('Daily')] info: @{{wt}} ; echo "worktree $WORKTREE_SLUG" ; \ echo "compose $COMPOSE_PROJECT_NAME" ; \ @@ -28,21 +74,42 @@ info: echo "server http://$WORKTREE_SLUG.localhost:$DJANGO_PORT/" ; \ echo "client http://$WORKTREE_SLUG.localhost:$CLIENT_PORT (reserved)" -# Start Postgres and Redis for this worktree, waiting until they are healthy. +# Check comments and docstrings against AGENTS.md. +[group('Daily')] +prose *args: + @python3 bin/check_prose.py {{args}} + +# Every step it calls is runnable on its own. + +# One-time setup. +[group('Lifecycle')] +init: env up sync migrate seed + pre-commit install + @{{wt}} ; cd api && uv run python manage.py collectstatic --noinput + +# Start Postgres and Redis, and wait for both. +[group('Lifecycle')] up: @{{wt}} ; docker compose up -d --wait ; \ echo "postgres localhost:$POSTGRES_HOST_PORT redis localhost:$REDIS_HOST_PORT" -# Stop them, keeping the data. `just down -v` drops this worktree's database. -down *args: - @{{wt}} ; docker compose down {{args}} +# Apply migrations. +[group('Lifecycle')] +migrate: + @{{wt}} ; cd api && uv run python manage.py migrate + +# Drop this database and build it again. +[group('Lifecycle')] +reset: + @{{wt}} ; docker compose down -v ; just up ; just migrate ; just seed-live # Also clears what an aborted `just check` left, under the compose # project that file uses. Its database is a tmpfs, so it holds nothing. # Add `-v` to drop this checkout's own database too. -# Stop this checkout's containers and remove its networks. -teardown *args: +# Stop this checkout's containers. +[group('Lifecycle')] +clean *args: @{{wt}} ; DIR=$(basename "$PWD") ; \ docker compose down --remove-orphans {{args}} ; \ docker compose -p "$DIR" -f docker-compose.test.yml down -v --remove-orphans ; \ @@ -53,69 +120,28 @@ teardown *args: echo "[just] git branch -d $(git rev-parse --abbrev-ref HEAD)" ; \ fi -# Install Python dependencies. -sync: - cd api && uv sync - -# Apply migrations to this worktree's database. -migrate: - @{{wt}} ; cd api && uv run python manage.py migrate - -# Create the admin in this worktree's database, from .env.local. Never asks. -seed: - @{{wt}} ; cd api && uv run python manage.py seed_admin +# Fetches real status pages, so it needs the network. Fills an +# empty database: admin, a small catalog, one tracked service. -# Fetches three real status pages, so it needs the network. - -# Fill an empty local database: admin, a small catalog, one tracked service. -seed-dev: +# Fill an empty database with real data. +[group('Data')] +seed-live: @{{wt}} ; cd api && uv run python manage.py seed_dev -# Drop this worktree's database and build it again from nothing. -reset: - @{{wt}} ; docker compose down -v ; just up ; just migrate ; just seed-dev +# Reports any page that moved or broke. -# Probe the recorded status pages and report any that moved or broke. -check-pages *args: +# Probe the recorded status pages. +[group('Data')] +probe-pages *args: @{{wt}} ; cd api && uv run python manage.py check_status_pages {{args}} -# Run the test suite. -test: - @{{wt}} ; cd api && uv run pytest -n auto - -# Run the test suite with the coverage gate. -test-cov: - @{{wt}} ; cd api && uv run pytest -n auto --cov-fail-under=85 - -# Fix lint and format the code. -lint: - cd api && uv run ruff check --fix . && uv run ruff format . - -# Check comments and docstrings against AGENTS.md. -prose *args: - @python3 bin/check_prose.py {{args}} - -# Slower than `just test`, and the answer that counts: it proves the -# image rather than the host. Run it before opening a pull request. - -# Everything CI runs, in the containers CI runs it in. -check: image - docker compose -f docker-compose.test.yml run --rm pytest -q -n auto --cov-fail-under=85 - docker compose -f docker-compose.test.yml run --rm ruff - docker compose -f docker-compose.test.yml run --rm docs - @python3 bin/check_prose.py - docker compose -f docker-compose.test.yml run --rm --entrypoint sh pytest \ - -c 'python manage.py makemigrations --check --dry-run' - docker compose -f docker-compose.test.yml down -v - -# Build the deployment image. `check` runs against this. -image: - docker build -t statusboard-api:test api - # The tag triggers release.yml: build, sign, smoke test, deploy. A tag # left behind by a failed push is reused rather than duplicated. -# Tag and push a release (v1.2.3, v1.2.3-rc.1). Ships to production. +# Takes v1.2.3 or v1.2.3-rc.1. + +# Tag and push a release. Ships to production. +[group('Release')] release VERSION: @set -e ; \ VERSION="{{VERSION}}" ; \ @@ -164,7 +190,8 @@ release VERSION: # Credentials come from api/.env.local. No app argument, so this repo can # only ever deploy its own stack. -# Redeploy the current image without cutting a release. +# Redeploy without cutting a release. +[group('Release')] deploy: @set -e ; \ APP="${FORGEJO_DEPLOY_APP:-}" ; \ @@ -188,10 +215,29 @@ deploy: fi ; \ echo "[deploy] queued. Watch: $HOST/${FORGEJO_REPO}/actions" -# Run the app: server and poller together. Ctrl-C stops both. -dev: - @{{wt}} ; trap 'kill 0' EXIT INT TERM ; \ - echo "http://$WORKTREE_SLUG.localhost:$DJANGO_PORT/" ; \ - cd api ; \ - uv run celery -A api worker -B -l info & \ - uv run python manage.py runserver 0.0.0.0:$DJANGO_PORT +# Everything below is a step of `init` or `check`, not a command a +# person reaches for. `just ` still runs each one directly. + +# Asks for the admin once. A worktree copies main's. + +# Write .env.local. +[private] +env: + @python3 bin/make-env.py + +# Install Python dependencies. +[private] +sync: + cd api && uv sync + +# Never asks: the values come from .env.local. + +# Create the admin. +[private] +seed: + @{{wt}} ; cd api && uv run python manage.py seed_admin + +# Build the deployment image. `check` runs against this. +[private] +image: + docker build -t statusboard-api:test api