From 206e99caa6820e8754de4e7f6576a09bee3d5fed Mon Sep 17 00:00:00 2001 From: Tim Geoghegan Date: Mon, 19 Oct 2026 13:58:17 -0400 Subject: [PATCH] clarify batch aggregation scrubbing Adds some doccomments on items related to scrubbed batch aggregations, as well as tidying up comments in the DB schema so they don't scroll off the right side of one's editor window. --- aggregator_core/src/datastore/models.rs | 22 ++++++++++---- db/00000000000001_initial_schema.up.sql | 39 ++++++++++++++++++------- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/aggregator_core/src/datastore/models.rs b/aggregator_core/src/datastore/models.rs index 844abfec5..beea07f5e 100644 --- a/aggregator_core/src/datastore/models.rs +++ b/aggregator_core/src/datastore/models.rs @@ -1717,18 +1717,30 @@ impl> Eq { } -// ReportAggregationStateCode exists alongside the public ReportAggregationState because there is no -// apparent way to denote a Postgres enum literal without deriving FromSql/ToSql on a Rust enum -// type, but it is not possible to derive FromSql/ToSql on a non-C-style enum. +/// BatchAggregationStateCode corresponds to the `batch_aggregation_state` enum. +/// +/// It exists alongside the public BatchAggregationStateCode because there is no apparent way to +/// denote a Postgres enum literal without deriving FromSql/ToSql on a Rust enum type, but it is not +/// possible to derive FromSql/ToSql on a non-C-style enum. #[derive(Debug, Clone, Copy, PartialEq, Eq, FromSql, ToSql)] #[postgres(name = "batch_aggregation_state")] pub(super) enum BatchAggregationStateCode { - /// This batch aggregation has not been collected & permits further aggregation. + /// This batch aggregation has not been collected. + /// + /// Report shares may still be aggregated into it. #[postgres(name = "AGGREGATING")] Aggregating, - /// This batch aggregation has been collected & no longer permits aggregation. + /// This batch aggregation has been collected. + /// + /// It contains a batch aggregate share which may be merged into an aggregate share to service a + /// collection job or aggregate share request. No more report shares may be aggregated into it. #[postgres(name = "COLLECTED")] Collected, + /// This batch aggregation has been scrubbed. + /// + /// Report shares have been aggregated into it and its batch aggregate has been merged into an + /// aggregate share. It has been scrubbed of data to save space. No more report shares may be + /// aggregated into it. #[postgres(name = "SCRUBBED")] Scrubbed, } diff --git a/db/00000000000001_initial_schema.up.sql b/db/00000000000001_initial_schema.up.sql index 9fc91009e..3e24014cb 100644 --- a/db/00000000000001_initial_schema.up.sql +++ b/db/00000000000001_initial_schema.up.sql @@ -375,23 +375,40 @@ CREATE TYPE BATCH_AGGREGATION_STATE AS ENUM( -- Information on aggregation for a single batch. This information may be incremental if the VDAF -- supports incremental aggregation. Each batch's aggregation is sharded via the `ord` column. CREATE TABLE batch_aggregations( - id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, -- artificial ID, internal-only - task_id BIGINT NOT NULL, -- the task ID - batch_identifier BYTEA NOT NULL, -- encoded batch mode-specific batch identifier (corresponds to identifier in BatchSelector) + -- artificial ID, internal-only + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + -- the task ID + task_id BIGINT NOT NULL, + -- encoded batch mode-specific batch identifier (corresponds to identifier in BatchSelector) + batch_identifier BYTEA NOT NULL, -- batch interval, as an INT8RANGE, in time precision units. -- Populated only for time-interval tasks. Will always match batch_identifier. batch_interval INT8RANGE, - aggregation_param BYTEA NOT NULL, -- the aggregation parameter (opaque VDAF message) - ord BIGINT NOT NULL, -- the index of this batch aggregation shard, over (task ID, batch_identifier, aggregation_param). + -- the aggregation parameter (opaque VDAF message) + aggregation_param BYTEA NOT NULL, + -- the index of this batch aggregation shard, over (task ID, batch_identifier, + -- aggregation_param). + ord BIGINT NOT NULL, -- The minimal interval, in time precision units, containing all of client timestamps included -- in this batch aggregation. client_timestamp_interval INT8RANGE NOT NULL, - state BATCH_AGGREGATION_STATE NOT NULL, -- the current state of this batch aggregation - aggregate_share BYTEA, -- the (possibly-incremental) aggregate share; populated unless report_count is 0 or the batch aggregation has been scrubbed. - report_count BIGINT, -- the (possibly-incremental) client report count; populated unless the batch aggregation has been scrubbed. - checksum BYTEA, -- the (possibly-incremental) checksum; populated unless the batch aggregation has been scrubbed. - aggregation_jobs_created BIGINT, -- the number of aggregation jobs that have been created for this batch; populated unless the batch aggregation has been scrubbed. - aggregation_jobs_terminated BIGINT, -- the number of aggregation jobs that have been terminated (finished or abandoned) for this batch; populated unless the batch aggregation has been scrubbed. + -- the current state of this batch aggregation + state BATCH_AGGREGATION_STATE NOT NULL, + -- the (possibly-incremental) aggregate share; populated unless report_count is 0 or the batch + -- aggregation has been scrubbed. + aggregate_share BYTEA, + -- the (possibly-incremental) client report count; populated unless the batch aggregation has + -- been scrubbed. + report_count BIGINT, + -- the (possibly-incremental) checksum; populated unless the batch aggregation has been + -- scrubbed. + checksum BYTEA, + -- the number of aggregation jobs that have been created for this batch; populated unless the + -- batch aggregation has been scrubbed. + aggregation_jobs_created BIGINT, + -- the number of aggregation jobs that have been terminated (finished or abandoned) for this + -- batch; populated unless the batch aggregation has been scrubbed. + aggregation_jobs_terminated BIGINT, -- creation/update records created_at TIMESTAMP WITH TIME ZONE NOT NULL, -- when the row was created