diff --git a/.gitattributes b/.gitattributes
index 8def7a98f..90bb428de 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -68,6 +68,7 @@
/src/package-sources/npmjs/canary/npmjs-package-canary.ts linguist-generated
/src/package-sources/npmjs/known-versions-migration.ts linguist-generated
/src/package-sources/npmjs/npm-js-follower.ts linguist-generated
+/src/package-sources/npmjs/packument-processor.ts linguist-generated
/src/package-sources/npmjs/re-stage-package-version.ts linguist-generated
/src/package-sources/npmjs/stage-and-notify.ts linguist-generated
/src/spdx-license.ts linguist-generated
diff --git a/.gitignore b/.gitignore
index ec6ca9dda..0bd84d979 100644
--- a/.gitignore
+++ b/.gitignore
@@ -82,6 +82,7 @@ lib/__tests__/devapp/cdk.out
!/test/tsconfig.json
!/src/package-sources/npmjs/stage-and-notify.ts
!/src/package-sources/npmjs/re-stage-package-version.ts
+!/src/package-sources/npmjs/packument-processor.ts
!/src/package-sources/npmjs/npm-js-follower.ts
!/src/package-sources/npmjs/known-versions-migration.ts
!/src/package-sources/npmjs/canary/npmjs-package-canary.ts
diff --git a/.prettierignore b/.prettierignore
index c9902bb09..b96b1c5cd 100644
--- a/.prettierignore
+++ b/.prettierignore
@@ -1,6 +1,7 @@
# ~~ Generated by projen. To modify, edit .projenrc.ts and run "yarn projen".
src/package-sources/npmjs/stage-and-notify.ts
src/package-sources/npmjs/re-stage-package-version.ts
+src/package-sources/npmjs/packument-processor.ts
src/package-sources/npmjs/npm-js-follower.ts
src/package-sources/npmjs/known-versions-migration.ts
src/package-sources/npmjs/canary/npmjs-package-canary.ts
diff --git a/.projen/files.json b/.projen/files.json
index a8dae9bd6..cdb725291 100644
--- a/.projen/files.json
+++ b/.projen/files.json
@@ -57,6 +57,7 @@
"src/package-sources/npmjs/canary/npmjs-package-canary.ts",
"src/package-sources/npmjs/known-versions-migration.ts",
"src/package-sources/npmjs/npm-js-follower.ts",
+ "src/package-sources/npmjs/packument-processor.ts",
"src/package-sources/npmjs/re-stage-package-version.ts",
"src/package-sources/npmjs/stage-and-notify.ts",
"src/spdx-license.ts",
diff --git a/.projen/tasks.json b/.projen/tasks.json
index b5296b1f6..57a730631 100644
--- a/.projen/tasks.json
+++ b/.projen/tasks.json
@@ -79,6 +79,9 @@
{
"spawn": "bundle:re-stage-package-version"
},
+ {
+ "spawn": "bundle:packument-processor"
+ },
{
"spawn": "bundle:npm-js-follower"
},
@@ -389,7 +392,7 @@
"description": "Continuously bundle all AWS Lambda functions",
"steps": [
{
- "exec": "esbuild --bundle src/package-sources/npmjs/stage-and-notify.lambda.ts src/package-sources/npmjs/re-stage-package-version.lambda.ts src/package-sources/npmjs/npm-js-follower.lambda.ts src/package-sources/npmjs/known-versions-migration.lambda.ts src/package-sources/npmjs/canary/npmjs-package-canary.lambda.ts src/package-sources/codeartifact/code-artifact-forwarder.lambda.ts src/overview-dashboard/sqs-dlq-stats-widget-function.lambda.ts src/monitoring/http-get-function.lambda.ts src/monitored-certificate/certificate-monitor.lambda.ts src/backend/version-tracker/version-tracker.lambda.ts src/backend/release-notes/release-notes-trigger.lambda.ts src/backend/release-notes/get-messages-from-worker-queue.lambda.ts src/backend/release-notes/generate-release-notes.lambda.ts src/backend/package-stats/package-stats-processor.lambda.ts src/backend/package-stats/package-stats-chunker.lambda.ts src/backend/package-stats/package-stats-aggregator.lambda.ts src/backend/orchestration/redrive-state-machine.lambda.ts src/backend/orchestration/read-uninstallable-report.lambda.ts src/backend/orchestration/needs-catalog-update.lambda.ts src/backend/inventory/package-versions-table-widget-function.lambda.ts src/backend/inventory/canary.lambda.ts src/backend/ingestion/re-ingest.lambda.ts src/backend/ingestion/ingestion.lambda.ts src/backend/feed-builder/update-feed.lambda.ts src/backend/ecs-task-monitor/monitor.lambda.ts src/backend/deny-list/prune-queue-handler.lambda.ts src/backend/deny-list/prune-handler.lambda.ts src/backend/catalog-builder/catalog-builder.lambda.ts src/__tests__/backend/deny-list/mocks/prune-test.lambda.ts src/__tests__/backend/deny-list/mocks/client-test.lambda.ts src/__tests__/backend/deny-list/mocks/catalog-builder-mock.lambda.ts --target=\"node22 --platform=\"node\" --outbase=\"src\" --outdir=\"lib\" --entry-names=\"[dir]/[name].bundle/index\" --external:aws-sdk --sourcemap --watch --tsconfig=tsconfig.json"
+ "exec": "esbuild --bundle src/package-sources/npmjs/stage-and-notify.lambda.ts src/package-sources/npmjs/re-stage-package-version.lambda.ts src/package-sources/npmjs/packument-processor.lambda.ts src/package-sources/npmjs/npm-js-follower.lambda.ts src/package-sources/npmjs/known-versions-migration.lambda.ts src/package-sources/npmjs/canary/npmjs-package-canary.lambda.ts src/package-sources/codeartifact/code-artifact-forwarder.lambda.ts src/overview-dashboard/sqs-dlq-stats-widget-function.lambda.ts src/monitoring/http-get-function.lambda.ts src/monitored-certificate/certificate-monitor.lambda.ts src/backend/version-tracker/version-tracker.lambda.ts src/backend/release-notes/release-notes-trigger.lambda.ts src/backend/release-notes/get-messages-from-worker-queue.lambda.ts src/backend/release-notes/generate-release-notes.lambda.ts src/backend/package-stats/package-stats-processor.lambda.ts src/backend/package-stats/package-stats-chunker.lambda.ts src/backend/package-stats/package-stats-aggregator.lambda.ts src/backend/orchestration/redrive-state-machine.lambda.ts src/backend/orchestration/read-uninstallable-report.lambda.ts src/backend/orchestration/needs-catalog-update.lambda.ts src/backend/inventory/package-versions-table-widget-function.lambda.ts src/backend/inventory/canary.lambda.ts src/backend/ingestion/re-ingest.lambda.ts src/backend/ingestion/ingestion.lambda.ts src/backend/feed-builder/update-feed.lambda.ts src/backend/ecs-task-monitor/monitor.lambda.ts src/backend/deny-list/prune-queue-handler.lambda.ts src/backend/deny-list/prune-handler.lambda.ts src/backend/catalog-builder/catalog-builder.lambda.ts src/__tests__/backend/deny-list/mocks/prune-test.lambda.ts src/__tests__/backend/deny-list/mocks/client-test.lambda.ts src/__tests__/backend/deny-list/mocks/catalog-builder-mock.lambda.ts --target=\"node22 --platform=\"node\" --outbase=\"src\" --outdir=\"lib\" --entry-names=\"[dir]/[name].bundle/index\" --external:aws-sdk --sourcemap --watch --tsconfig=tsconfig.json"
}
]
},
@@ -537,6 +540,24 @@
}
]
},
+ "bundle:packument-processor": {
+ "name": "bundle:packument-processor",
+ "description": "Create an AWS Lambda bundle from src/package-sources/npmjs/packument-processor.lambda.ts",
+ "steps": [
+ {
+ "exec": "esbuild --bundle src/package-sources/npmjs/packument-processor.lambda.ts --target=\"node22\" --platform=\"node\" --outfile=\"lib/package-sources/npmjs/packument-processor.lambda.bundle/index.js\" --external:aws-sdk --sourcemap --tsconfig=tsconfig.json"
+ }
+ ]
+ },
+ "bundle:packument-processor:watch": {
+ "name": "bundle:packument-processor:watch",
+ "description": "Continuously update an AWS Lambda bundle from src/package-sources/npmjs/packument-processor.lambda.ts",
+ "steps": [
+ {
+ "exec": "esbuild --bundle src/package-sources/npmjs/packument-processor.lambda.ts --target=\"node22\" --platform=\"node\" --outfile=\"lib/package-sources/npmjs/packument-processor.lambda.bundle/index.js\" --external:aws-sdk --sourcemap --tsconfig=tsconfig.json --watch"
+ }
+ ]
+ },
"bundle:prune-handler": {
"name": "bundle:prune-handler",
"description": "Create an AWS Lambda bundle from src/backend/deny-list/prune-handler.lambda.ts",
@@ -861,6 +882,9 @@
{
"spawn": "bundle:re-stage-package-version"
},
+ {
+ "spawn": "bundle:packument-processor"
+ },
{
"spawn": "bundle:npm-js-follower"
},
diff --git a/API.md b/API.md
index 25f5a18fd..052b32075 100644
--- a/API.md
+++ b/API.md
@@ -2913,13 +2913,17 @@ new sources.NpmJs(props?: NpmJsProps)
| bind | Binds the package source to a scope and target queue. |
| metricBatchProcessingTime | The average time it took to process a changes batch. |
| metricChangeCount | The total count of changes that were processed. |
-| metricLaggyPackumentGiveUps | The number of laggy packuments the follower gave up on: the registry never served the revision announced by the `_changes` feed within the maximum retry age. |
-| metricLaggyPackuments | The number of packages for which the registry packument is still behind the revision announced by the `_changes` feed. |
+| metricLaggyPackumentGiveUps | The number of laggy packuments given up on: the registry never served the revision announced by the `_changes` feed within the maximum retry age. |
+| metricLaggyPackumentLag | How long laggy packuments took to catch up (or until they were given up on), measured from when the follower first saw them. |
+| metricLaggyPackuments | The number of requests in the packument queue, for laggy and missing packuments. |
| metricLaggyPackumentsRecovered | The number of laggy packuments for which the registry caught up with the revision announced by the `_changes` feed. |
| metricLastSeq | The last sequence number that was processed. |
| metricLateChangeCount | The number of change entries discovered by a scan that were inserted into the `_changes` feed behind a position the follower had already read past. |
| metricLateChangeLag | For each late change entry, the time elapsed between the moment the follower first read past the entry's sequence number and the moment the entry was discovered. |
| metricMetadataFetchFailures | The number of change entries whose registry metadata could not be fetched (after transient-error retries). |
+| metricMissingPackumentGiveUps | The number of missing packuments given up on: the registry never served a packument for the package within the maximum retry age. |
+| metricMissingPackuments | The number of change entries for which the registry had no packument at all (HTTP 404). |
+| metricMissingPackumentsRecovered | The number of missing packuments the registry has since served. |
| metricNpmJsChangeAge | *No description.* |
| metricPackageVersionAge | The age of the oldest package version that was processed. |
| metricPackageVersionCount | The total count of package versions that were inspected. |
@@ -2983,7 +2987,7 @@ The total count of changes that were processed.
public metricLaggyPackumentGiveUps(opts?: MetricOptions): Metric
```
-The number of laggy packuments the follower gave up on: the registry never served the revision announced by the `_changes` feed within the maximum retry age.
+The number of laggy packuments given up on: the registry never served the revision announced by the `_changes` feed within the maximum retry age.
A version announced by the feed may be missing until
the affected package publishes again.
@@ -2994,17 +2998,30 @@ the affected package publishes again.
---
+##### `metricLaggyPackumentLag`
+
+```typescript
+public metricLaggyPackumentLag(opts?: MetricOptions): Metric
+```
+
+How long laggy packuments took to catch up (or until they were given up on), measured from when the follower first saw them.
+
+###### `opts`Optional
+
+- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions
+
+---
+
##### `metricLaggyPackuments`
```typescript
public metricLaggyPackuments(opts?: MetricOptions): Metric
```
-The number of packages for which the registry packument is still behind the revision announced by the `_changes` feed.
+The number of requests in the packument queue, for laggy and missing packuments.
-The versions served so far
-have been processed; the follower keeps re-checking for the announced
-revision.
+A package that lagged in several change entries has several
+requests.
###### `opts`Optional
@@ -3091,6 +3108,54 @@ later scan retries them.
---
+##### `metricMissingPackumentGiveUps`
+
+```typescript
+public metricMissingPackumentGiveUps(opts?: MetricOptions): Metric
+```
+
+The number of missing packuments given up on: the registry never served a packument for the package within the maximum retry age.
+
+Usually an old
+change entry for a package that no longer exists.
+
+###### `opts`Optional
+
+- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions
+
+---
+
+##### `metricMissingPackuments`
+
+```typescript
+public metricMissingPackuments(opts?: MetricOptions): Metric
+```
+
+The number of change entries for which the registry had no packument at all (HTTP 404).
+
+Usually a new package that has not replicated to the
+registry yet; the packument processor keeps re-checking for it.
+
+###### `opts`Optional
+
+- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions
+
+---
+
+##### `metricMissingPackumentsRecovered`
+
+```typescript
+public metricMissingPackumentsRecovered(opts?: MetricOptions): Metric
+```
+
+The number of missing packuments the registry has since served.
+
+###### `opts`Optional
+
+- *Type:* aws-cdk-lib.aws_cloudwatch.MetricOptions
+
+---
+
##### `metricNpmJsChangeAge`
```typescript
diff --git a/docs/application-overview.md b/docs/application-overview.md
index 319a94322..225fd8b40 100644
--- a/docs/application-overview.md
+++ b/docs/application-overview.md
@@ -42,9 +42,11 @@ ConstructHub provides two package source implementations: `NpmJs` and
function is invoked, which stages the package tarball into an S3 bucket then
notifies the ConstructHub ingestion SQS queue. The CouchDB follower is
scheduled to run every `5 minutes`, and stores its state (receipts for the
- change entries received, time/sequence checkpoints, and laggy packument
- expectations) in a specific object in the S3 bucket used for staging package
- tarballs.
+ change entries received, and time/sequence checkpoints) in a specific object
+ in the S3 bucket used for staging package tarballs.
+ When the registry serves an older revision of a package than the feed
+ announced, the package is sent to a queue, and a packument processor function
+ checks it again, with growing waits, for about a day.
The package versions it has processed are recorded in a DynamoDB table, so
each version is only processed once.
Existing deployments kept this list in the S3 bucket; it is copied to the
diff --git a/docs/npm-sync-protocol.md b/docs/npm-sync-protocol.md
index 36fc5cc0c..2350d460a 100644
--- a/docs/npm-sync-protocol.md
+++ b/docs/npm-sync-protocol.md
@@ -122,12 +122,12 @@ processing any entry it has no receipt for.
Rolling checkpoints map times to feed positions, and derive the window's starting position.
On top of that baseline, each gap has a dedicated mitigation:
-| Gap | Mitigation | Detection |
-| ---------------- | ------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------- |
-| Late insertion | Every scan re-covers the trailing window; a periodic deep scan covers the full state retention | `LateChangeCount` / `LateChangeLag` metrics; `LateChangeLagHigh` alarm |
-| Laggy packuments | Process every packument the registry serves immediately; record the announced revision and re-check it every run until it appears or ages out | `LaggyPackuments` / `LaggyPackumentsRecovered` / `LaggyPackumentGiveUps` metrics; `LaggyPackumentGiveUps` alarm |
-| Lost events | None possible; the missed versions self-heal on the package's next publish | Package canary (`Canary/SLA-Breached`) covers the probe package |
-| Head regression | The follower discards its position state and reseeds at the new head when the feed head drops below its newest checkpoint | `LastSeq` metric |
+| Gap | Mitigation | Detection |
+| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Late insertion | Every scan re-covers the trailing window; a periodic deep scan covers the full state retention | `LateChangeCount` / `LateChangeLag` metrics; `LateChangeLagHigh` alarm |
+| Laggy packuments | Process every packument the registry serves immediately; queue a re-check of the announced revision (also when the registry has no packument yet), retried with back-off for about 29 hours, then given up on | `LaggyPackuments` / `LaggyPackumentsRecovered` / `LaggyPackumentGiveUps` / `LaggyPackumentLag` and `MissingPackuments` / `MissingPackumentsRecovered` / `MissingPackumentGiveUps` metrics; `LaggyPackumentLagHigh` and `MissingPackumentsHigh` alarms |
+| Lost events | None possible; the missed versions self-heal on the package's next publish | Package canary (`Canary/SLA-Breached`) covers the probe package |
+| Head regression | The follower discards its position state and reseeds at the new head when the feed head drops below its newest checkpoint | `LastSeq` metric |
**The design idea behind all of this is that the feed data are just hints without real guarantees.**
The follower treats every change entry as "this package probably changed around here",
diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md
index 9773cc1d0..69f1590ad 100644
--- a/docs/operator-runbook.md
+++ b/docs/operator-runbook.md
@@ -614,49 +614,92 @@ be increased: adjust `SCAN_WINDOW_MS` in
`src/package-sources/npmjs/constants.lambda-shared.ts` (keeping it below
`STATE_RETENTION_MS`) and deploy the updated construct.
-### `ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps`
+### `ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh`
#### Description
This alarm is only provisioned when the `NpmJs` package source is configured. It
-triggers when the npm registry never served the packument revision announced by
-the CouchDB `_changes` feed for one or more packages, even after the follower
-re-checked for a prolonged period.
-
-All package versions the registry did serve have been processed. However, a
-version announced by the changes feed may be missing from ConstructHub until
-the affected package publishes its next version.
+triggers when the npm registry had no packument (HTTP 404) for an unusually high
+number of packages announced by the CouchDB `_changes` feed, within 3 hours.
+This usually means the registry has an outage, or is far behind the feed.
#### Investigation
The *NpmJs Follower* fetches package metadata from `registry.npmjs.org` for
-each change announced by the CouchDB `_changes` feed. The registry occasionally
-lags behind the feed. When it does, the follower processes the versions that
-were served, records the announced revision as a "laggy packument", and
-re-checks it on every run until the revision appears or the expectation ages
-out. This alarm fires for the latter.
+each change announced by the CouchDB `_changes` feed. Some of these always
+return 404, usually because a new package has not replicated to the registry
+yet (a *missing* packument). The follower sends them to the *Packument Queue*,
+where the *Packument Processor* checks them again with growing waits, for about
+a day. A normal 3 hour window has a couple of hundred of them.
-Review the logs of the *NpmJs Follower* for `Giving up on laggy packument`
-entries; they identify the affected package and the revision that never
-appeared. Verify what the registry currently serves for the package:
+Check the `Missing Packuments` line on the *Feed Reliability* graph of the
+backend dashboard, and pick a few affected packages from the logs of the
+*NpmJs Follower* (`the registry has no packument for it yet`). Verify whether
+the registry serves them now:
```console
-$ curl https://registry.npmjs.org/ | jq ._rev
+$ curl -s -o /dev/null -w '%{http_code}\n' https://registry.npmjs.org/
```
-If the served revision is still older than the expected one, the problem is
+If most still return 404 while the feed keeps announcing them, the problem is
with upstream npmjs.com. Look at `npmjs.com` status updates and announcements.
+If you suspect the problem is with upstream npmjs.com, you can cut them a
+ticket. Log in to , visit the
+[Support Page](https://www.npmjs.com/support) and select
+*There is a problem with the npm registry*.
-If you suspect the problem is with upstream npmjs.com, you can cut them a ticket. Log
-in to , visit the [Support Page](https://www.npmjs.com/support) and
-select *There is a problem with the npm registry*.
+#### Resolution
+
+The packument processor processes the packages once the registry serves them.
+Packages it gives up on are logged (`Giving up on missing packument`) and then
+dropped. They show up again when the package next publishes. To ingest one
+right away, invoke the *ReStagePackageVersion* Lambda function with the package
+name and version.
+
+--------------------------------------------------------------------------------
+
+### `ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh`
+
+#### Description
+
+This alarm is only provisioned when the `NpmJs` package source is configured. It
+triggers when laggy packuments take unusually long to catch up: in a 3 hour
+window, 90% of them took longer than 6 hours. This usually means the registry
+has an outage, or is far behind the feed.
+
+All package versions the registry did serve have been processed. However,
+versions announced by the changes feed may be missing from ConstructHub until
+the registry catches up.
+
+#### Investigation
+
+The registry occasionally serves an older revision of a package than the
+changes feed announced (a *laggy* packument). The follower processes the
+versions that were served, and sends the package to the *Packument Queue*. The
+*Packument Processor* checks it again with growing waits, for about 29 hours,
+and reports how long each laggy packument took to catch up. Most catch up within
+minutes, and nearly all within 6 hours.
+
+Check the *Packument Queue* graph of the backend dashboard for how many
+requests are waiting, and the `Laggy Packument Give-Ups` line on the
+*Feed Reliability* graph for how many were given up on. The logs of the
+*Packument Processor* show `caught up` and `Giving up on laggy packument`
+entries. Verify what the registry serves for a few affected packages:
+
+```console
+$ curl https://registry.npmjs.org/ | jq ._rev
+```
+
+If the served revision is still older than the expected one, the problem is
+with upstream npmjs.com (see the previous alarm for how to reach them).
#### Resolution
-The missed package versions will automatically be discovered once the affected
-packages publish their next version. To ingest a missed version immediately,
-invoke the *ReStagePackageVersion* Lambda function with the package name and
-version from the follower logs.
+The packument processor processes the missed versions once the registry serves
+them. Packages it gives up on are logged and then dropped; there is nothing to
+redrive. The missed versions show up again when the package next publishes. To
+ingest one right away, invoke the *ReStagePackageVersion* Lambda function with
+the package name and version.
--------------------------------------------------------------------------------
@@ -811,6 +854,30 @@ When documentation for a package is not available, it could be due to several re
--------------------------------------------------------------------------------
+### `ConstructHub/Sources/NpmJs/PackumentProcessor/Failures`
+
+#### Description
+
+This alarm is only provisioned when the `NpmJs` package source is configured. It
+triggers when the *Packument Processor* function fails or times out. While it
+fails, the requests in the *Packument Queue* keep using up their attempts, so
+they can be given up on earlier than the usual 29 hours.
+
+#### Investigation
+
+Review the logs of the *Packument Processor* (linked from the backend
+dashboard). Errors while setting up a batch (for example reading the deny list
+or license list) fail the whole batch; errors for a single package only fail
+that request. Time-outs usually mean the npm registry is slow to respond.
+
+#### Resolution
+
+Fix the cause. Requests that were given up on in the meantime are lost. Their
+versions show up again when the packages next publish, or can be ingested
+right away with the *ReStagePackageVersion* Lambda function.
+
+--------------------------------------------------------------------------------
+
### `ConstructHub/Sources/NpmJs/Canary/SLA-Breached`
#### Description
diff --git a/package.json b/package.json
index e3017425b..0026b851c 100644
--- a/package.json
+++ b/package.json
@@ -50,6 +50,8 @@
"bundle:package-stats-processor:watch": "projen bundle:package-stats-processor:watch",
"bundle:package-versions-table-widget-function": "projen bundle:package-versions-table-widget-function",
"bundle:package-versions-table-widget-function:watch": "projen bundle:package-versions-table-widget-function:watch",
+ "bundle:packument-processor": "projen bundle:packument-processor",
+ "bundle:packument-processor:watch": "projen bundle:packument-processor:watch",
"bundle:prune-handler": "projen bundle:prune-handler",
"bundle:prune-handler:watch": "projen bundle:prune-handler:watch",
"bundle:prune-queue-handler": "projen bundle:prune-queue-handler",
diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap
index 6851553ee..2dd5303ba 100644
--- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap
+++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap
@@ -771,6 +771,25 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
+ ")\\n[button:Packument Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ ")\\n[button:Packument Processor](/lambda/home#/functions/",
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
@@ -819,7 +838,32 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
+ "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Missing Packuments","expression":"FILL(m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackuments",{"label":"Missing Packuments","stat":"Sum","visible":false,"id":"m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62"}],[{"label":"Missing Packuments Recovered","expression":"FILL(m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentsRecovered",{"label":"Missing Packuments Recovered","stat":"Sum","visible":false,"id":"m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7"}],[{"label":"Missing Packument Give-Ups","expression":"FILL(m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentGiveUps",{"label":"Missing Packument Give-Ups","stat":"Sum","visible":false,"id":"m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Packument Queue","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Waiting","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Waiting for next attempt","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesDelayed","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Delayed","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -10124,7 +10168,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "13418b75fd3f428b357c909862c8ea4c195836de07299f56105c6e3175416cc5.zip",
+ "S3Key": "93295fa0a1f1d54bc071e064c57c7713c38f4487ba3fcb3d574aa9536b376cd0.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages",
"Environment": {
@@ -10147,6 +10191,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Ref": "ConstructHubLicenseListBucket9334047F",
},
"LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "PACKUMENT_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ },
},
},
"Handler": "index.handler",
@@ -10581,27 +10628,6 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
- "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": {
- "Properties": {
- "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed
-for one or more packages, even after repeated re-checks. All versions the registry did serve
-have been processed, but a version announced by the feed may be missing until the affected
-package publishes again. The follower logs identify the affected packages, which can be
-manually ingested using the ReStagePackageVersion function.
-
-Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
- "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps",
- "ComparisonOperator": "GreaterThanOrEqualToThreshold",
- "EvaluationPeriods": 1,
- "MetricName": "LaggyPackumentGiveUps",
- "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
- "Period": 3600,
- "Statistic": "Sum",
- "Threshold": 1,
- "TreatMissingData": "notBreaching",
- },
- "Type": "AWS::CloudWatch::Alarm",
- },
"ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": {
"Properties": {
"AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed
@@ -10622,6 +10648,25 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerMissingPackumentsHigh0AB93A32": {
+ "Properties": {
+ "AlarmDescription": "The npm registry had no packument for an unusually high number of packages announced by
+the CouchDB changes feed. This usually means the registry has an outage or is far behind
+the feed. The packument processor keeps re-checking these packages.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "MetricName": "MissingPackuments",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 10800,
+ "Statistic": "Sum",
+ "Threshold": 1500,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": {
"Properties": {
"AlarmDescription": {
@@ -10705,7 +10750,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Properties": {
"ExecuteOnHandlerChange": false,
"HandlerArn": {
- "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5",
+ "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681",
},
"InvocationType": "Event",
"ServiceToken": {
@@ -10729,7 +10774,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Properties": {
"ExecuteOnHandlerChange": false,
"HandlerArn": {
- "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5",
+ "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681",
},
"InvocationType": "RequestResponse",
"ServiceToken": {
@@ -10743,7 +10788,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Type": "Custom::Trigger",
"UpdateReplacePolicy": "Delete",
},
- "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5": {
+ "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681": {
"DependsOn": [
"ConstructHubLicenseListAwsCliLayer59592811",
"ConstructHubLicenseListCustomResource323F0FD4",
@@ -10770,7 +10815,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "1e7b1fb4509d2b2ef4b63b0ad82abf8b1e820a8920419db90eff23d9392dbd49.zip",
+ "S3Key": "8d0e24c004f0022d0e6e92031e898d0eb9643d03e18273a9abc71da4fe2820d3.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table",
"Environment": {
@@ -10938,6 +10983,64 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::IAM::Policy",
},
+ "ConstructHubSourcesNpmJsPackumentProcessorFailuresC6FC837A": {
+ "Properties": {
+ "AlarmDescription": {
+ "Fn::Join": [
+ "",
+ [
+ "The packument processor function is failing (errors or time-outs). Requests in the
+packument queue use up their attempts while it fails, and may be given up on early.
+
+Link to the lambda function: /lambda/home#/functions/",
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ ],
+ ],
+ },
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/Failures",
+ "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+ "Dimensions": [
+ {
+ "Name": "FunctionName",
+ "Value": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ },
+ ],
+ "EvaluationPeriods": 2,
+ "MetricName": "Errors",
+ "Namespace": "AWS/Lambda",
+ "Period": 900,
+ "Statistic": "Sum",
+ "Threshold": 1,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
+ "ConstructHubSourcesNpmJsPackumentProcessorLaggyPackumentLagHigh863AFDFC": {
+ "Properties": {
+ "AlarmDescription": "Laggy packuments take unusually long to catch up: the npm registry serves older revisions
+than the CouchDB changes feed announced, for hours. This usually means the registry has an
+outage or is far behind the feed. The packument processor keeps re-checking these packages.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "ExtendedStatistic": "p90",
+ "MetricName": "LaggyPackumentLag",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 10800,
+ "Threshold": 21600000,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsSchedule34031870": {
"DependsOn": [
"ConstructHubSourcesNpmJsFollowerFailures86BCBA0D",
@@ -11209,6 +11312,20 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "sqs:SendMessage",
+ "sqs:GetQueueAttributes",
+ "sqs:GetQueueUrl",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ },
],
"Version": "2012-10-17",
},
@@ -11234,7 +11351,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "ef94880eff6f41104133b16d56770947ee1a674effd8a6ae47b17b3e604ed377.zip",
+ "S3Key": "e3822146d5e137d4bf9e0ac71dbab1cd07c63186f8140df399e0f49b919c8838.zip",
},
"DeadLetterConfig": {
"TargetArn": {
@@ -11560,10 +11677,21 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::S3::BucketPolicy",
},
- "ConstructHubSourcesReStagePackageVersion5E1211D1": {
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
+ "ConstructHubSourcesPackumentProcessorFunction0E5F8177": {
"DependsOn": [
- "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40",
- "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA",
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4",
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
],
"Properties": {
"Architectures": [
@@ -11573,31 +11701,58 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "2bc29d2913a9accb30972e2ebbb8cf405efc9e8b73ea4605c359d99907fa0dbd.zip",
+ "S3Key": "6ae1eb05cba8999261a20927e8c15ffccd256111d9fe1b62c24b9a0ca1ed11e3.zip",
},
- "Description": "Manually re-stage a package version",
+ "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed",
"Environment": {
"Variables": {
+ "AWS_EMF_ENVIRONMENT": "Local",
+ "DENY_LIST_BUCKET_NAME": {
+ "Ref": "ConstructHubDenyListBucket1B3C2C2E",
+ },
+ "DENY_LIST_OBJECT_KEY": "deny-list.json",
"FUNCTION_NAME": {
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
- "REGISTRY_URL": "https://registry.npmjs.org",
+ "KNOWN_VERSIONS_TABLE_NAME": {
+ "Ref": "ConstructHubSourcesKnownVersions1480B28B",
+ },
+ "LICENSE_LIST_BUCKET_NAME": {
+ "Ref": "ConstructHubLicenseListBucket9334047F",
+ },
+ "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "PACKUMENT_DEAD_LETTER_QUEUE_ARN": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "PACKUMENT_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ },
},
},
"Handler": "index.handler",
- "MemorySize": 1024,
+ "MemorySize": 2048,
"Role": {
"Fn::GetAtt": [
- "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA",
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
"Arn",
],
},
"Runtime": "nodejs22.x",
- "Timeout": 10,
+ "Timeout": 300,
+ "TracingConfig": {
+ "Mode": "Active",
+ },
},
"Type": "AWS::Lambda::Function",
},
- "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA": {
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
@@ -11628,7 +11783,310 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::IAM::Role",
},
- "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40": {
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "PolicyDocument": {
+ "Statement": [
+ {
+ "Action": [
+ "xray:PutTraceSegments",
+ "xray:PutTelemetryRecords",
+ ],
+ "Effect": "Allow",
+ "Resource": "*",
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "dynamodb:BatchGetItem",
+ "dynamodb:GetRecords",
+ "dynamodb:GetShardIterator",
+ "dynamodb:Query",
+ "dynamodb:GetItem",
+ "dynamodb:Scan",
+ "dynamodb:ConditionCheckItem",
+ "dynamodb:BatchWriteItem",
+ "dynamodb:PutItem",
+ "dynamodb:UpdateItem",
+ "dynamodb:DeleteItem",
+ "dynamodb:DescribeTable",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesKnownVersions1480B28B",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubDenyListBucket1B3C2C2E",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubDenyListBucket1B3C2C2E",
+ "Arn",
+ ],
+ },
+ "/*",
+ ],
+ ],
+ },
+ ],
+ },
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubLicenseListBucket9334047F",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubLicenseListBucket9334047F",
+ "Arn",
+ ],
+ },
+ "/*",
+ ],
+ ],
+ },
+ ],
+ },
+ {
+ "Action": "lambda:InvokeFunction",
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ "Arn",
+ ],
+ },
+ ":*",
+ ],
+ ],
+ },
+ ],
+ },
+ ],
+ "Version": "2012-10-17",
+ },
+ "PolicyName": "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4",
+ "Roles": [
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
+ },
+ ],
+ },
+ "Type": "AWS::IAM::Policy",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorDLQEED56E2F4AC5AB3E": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "BatchSize": 10,
+ "EventSourceArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "FunctionName": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "FunctionResponseTypes": [
+ "ReportBatchItemFailures",
+ ],
+ },
+ "Type": "AWS::Lambda::EventSourceMapping",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "BatchSize": 10,
+ "EventSourceArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ "FunctionName": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "FunctionResponseTypes": [
+ "ReportBatchItemFailures",
+ ],
+ "ScalingConfig": {
+ "MaximumConcurrency": 2,
+ },
+ },
+ "Type": "AWS::Lambda::EventSourceMapping",
+ },
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ "RedrivePolicy": {
+ "deadLetterTargetArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "maxReceiveCount": 10,
+ },
+ "VisibilityTimeout": 1800,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
+ "ConstructHubSourcesReStagePackageVersion5E1211D1": {
+ "DependsOn": [
+ "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40",
+ "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA",
+ ],
+ "Properties": {
+ "Architectures": [
+ "arm64",
+ ],
+ "Code": {
+ "S3Bucket": {
+ "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
+ },
+ "S3Key": "3667bd11e9ba1e2476b569411bb892b311eda3d4b8dbe5880034486059b18bed.zip",
+ },
+ "Description": "Manually re-stage a package version",
+ "Environment": {
+ "Variables": {
+ "FUNCTION_NAME": {
+ "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ },
+ "REGISTRY_URL": "https://registry.npmjs.org",
+ },
+ },
+ "Handler": "index.handler",
+ "MemorySize": 1024,
+ "Role": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA",
+ "Arn",
+ ],
+ },
+ "Runtime": "nodejs22.x",
+ "Timeout": 10,
+ },
+ "Type": "AWS::Lambda::Function",
+ },
+ "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA": {
+ "Properties": {
+ "AssumeRolePolicyDocument": {
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Effect": "Allow",
+ "Principal": {
+ "Service": "lambda.amazonaws.com",
+ },
+ },
+ ],
+ "Version": "2012-10-17",
+ },
+ "ManagedPolicyArns": [
+ {
+ "Fn::Join": [
+ "",
+ [
+ "arn:",
+ {
+ "Ref": "AWS::Partition",
+ },
+ ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole",
+ ],
+ ],
+ },
+ ],
+ },
+ "Type": "AWS::IAM::Role",
+ },
+ "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40": {
"Properties": {
"PolicyDocument": {
"Statement": [
@@ -15225,6 +15683,25 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
+ ")\\n[button:Packument Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ ")\\n[button:Packument Processor](/lambda/home#/functions/",
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
@@ -15273,7 +15750,32 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
+ "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Missing Packuments","expression":"FILL(m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackuments",{"label":"Missing Packuments","stat":"Sum","visible":false,"id":"m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62"}],[{"label":"Missing Packuments Recovered","expression":"FILL(m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentsRecovered",{"label":"Missing Packuments Recovered","stat":"Sum","visible":false,"id":"m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7"}],[{"label":"Missing Packument Give-Ups","expression":"FILL(m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentGiveUps",{"label":"Missing Packument Give-Ups","stat":"Sum","visible":false,"id":"m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Packument Queue","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Waiting","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Waiting for next attempt","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesDelayed","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Delayed","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -24952,7 +25454,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "13418b75fd3f428b357c909862c8ea4c195836de07299f56105c6e3175416cc5.zip",
+ "S3Key": "93295fa0a1f1d54bc071e064c57c7713c38f4487ba3fcb3d574aa9536b376cd0.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages",
"Environment": {
@@ -24975,6 +25477,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Ref": "ConstructHubLicenseListBucket9334047F",
},
"LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "PACKUMENT_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ },
},
},
"Handler": "index.handler",
@@ -25409,27 +25914,6 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
- "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": {
- "Properties": {
- "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed
-for one or more packages, even after repeated re-checks. All versions the registry did serve
-have been processed, but a version announced by the feed may be missing until the affected
-package publishes again. The follower logs identify the affected packages, which can be
-manually ingested using the ReStagePackageVersion function.
-
-Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
- "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps",
- "ComparisonOperator": "GreaterThanOrEqualToThreshold",
- "EvaluationPeriods": 1,
- "MetricName": "LaggyPackumentGiveUps",
- "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
- "Period": 3600,
- "Statistic": "Sum",
- "Threshold": 1,
- "TreatMissingData": "notBreaching",
- },
- "Type": "AWS::CloudWatch::Alarm",
- },
"ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": {
"Properties": {
"AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed
@@ -25450,6 +25934,25 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerMissingPackumentsHigh0AB93A32": {
+ "Properties": {
+ "AlarmDescription": "The npm registry had no packument for an unusually high number of packages announced by
+the CouchDB changes feed. This usually means the registry has an outage or is far behind
+the feed. The packument processor keeps re-checking these packages.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "MetricName": "MissingPackuments",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 10800,
+ "Statistic": "Sum",
+ "Threshold": 1500,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": {
"Properties": {
"AlarmDescription": {
@@ -25533,7 +26036,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Properties": {
"ExecuteOnHandlerChange": false,
"HandlerArn": {
- "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5",
+ "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681",
},
"InvocationType": "Event",
"ServiceToken": {
@@ -25557,7 +26060,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Properties": {
"ExecuteOnHandlerChange": false,
"HandlerArn": {
- "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5",
+ "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681",
},
"InvocationType": "RequestResponse",
"ServiceToken": {
@@ -25571,7 +26074,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Type": "Custom::Trigger",
"UpdateReplacePolicy": "Delete",
},
- "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5": {
+ "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681": {
"DependsOn": [
"ConstructHubLicenseListAwsCliLayer59592811",
"ConstructHubLicenseListCustomResource323F0FD4",
@@ -25598,7 +26101,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "1e7b1fb4509d2b2ef4b63b0ad82abf8b1e820a8920419db90eff23d9392dbd49.zip",
+ "S3Key": "8d0e24c004f0022d0e6e92031e898d0eb9643d03e18273a9abc71da4fe2820d3.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table",
"Environment": {
@@ -25766,6 +26269,64 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::IAM::Policy",
},
+ "ConstructHubSourcesNpmJsPackumentProcessorFailuresC6FC837A": {
+ "Properties": {
+ "AlarmDescription": {
+ "Fn::Join": [
+ "",
+ [
+ "The packument processor function is failing (errors or time-outs). Requests in the
+packument queue use up their attempts while it fails, and may be given up on early.
+
+Link to the lambda function: /lambda/home#/functions/",
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ ],
+ ],
+ },
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/Failures",
+ "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+ "Dimensions": [
+ {
+ "Name": "FunctionName",
+ "Value": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ },
+ ],
+ "EvaluationPeriods": 2,
+ "MetricName": "Errors",
+ "Namespace": "AWS/Lambda",
+ "Period": 900,
+ "Statistic": "Sum",
+ "Threshold": 1,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
+ "ConstructHubSourcesNpmJsPackumentProcessorLaggyPackumentLagHigh863AFDFC": {
+ "Properties": {
+ "AlarmDescription": "Laggy packuments take unusually long to catch up: the npm registry serves older revisions
+than the CouchDB changes feed announced, for hours. This usually means the registry has an
+outage or is far behind the feed. The packument processor keeps re-checking these packages.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "ExtendedStatistic": "p90",
+ "MetricName": "LaggyPackumentLag",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 10800,
+ "Threshold": 21600000,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsSchedule34031870": {
"DependsOn": [
"ConstructHubSourcesNpmJsFollowerFailures86BCBA0D",
@@ -26037,6 +26598,20 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "sqs:SendMessage",
+ "sqs:GetQueueAttributes",
+ "sqs:GetQueueUrl",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ },
],
"Version": "2012-10-17",
},
@@ -26062,7 +26637,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "ef94880eff6f41104133b16d56770947ee1a674effd8a6ae47b17b3e604ed377.zip",
+ "S3Key": "e3822146d5e137d4bf9e0ac71dbab1cd07c63186f8140df399e0f49b919c8838.zip",
},
"DeadLetterConfig": {
"TargetArn": {
@@ -26388,6 +26963,347 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::S3::BucketPolicy",
},
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
+ "ConstructHubSourcesPackumentProcessorFunction0E5F8177": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4",
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
+ ],
+ "Properties": {
+ "Architectures": [
+ "arm64",
+ ],
+ "Code": {
+ "S3Bucket": {
+ "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
+ },
+ "S3Key": "6ae1eb05cba8999261a20927e8c15ffccd256111d9fe1b62c24b9a0ca1ed11e3.zip",
+ },
+ "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed",
+ "Environment": {
+ "Variables": {
+ "AWS_EMF_ENVIRONMENT": "Local",
+ "DENY_LIST_BUCKET_NAME": {
+ "Ref": "ConstructHubDenyListBucket1B3C2C2E",
+ },
+ "DENY_LIST_OBJECT_KEY": "deny-list.json",
+ "FUNCTION_NAME": {
+ "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ },
+ "KNOWN_VERSIONS_TABLE_NAME": {
+ "Ref": "ConstructHubSourcesKnownVersions1480B28B",
+ },
+ "LICENSE_LIST_BUCKET_NAME": {
+ "Ref": "ConstructHubLicenseListBucket9334047F",
+ },
+ "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "PACKUMENT_DEAD_LETTER_QUEUE_ARN": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "PACKUMENT_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ },
+ },
+ },
+ "Handler": "index.handler",
+ "MemorySize": 2048,
+ "Role": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
+ "Arn",
+ ],
+ },
+ "Runtime": "nodejs22.x",
+ "Timeout": 300,
+ "TracingConfig": {
+ "Mode": "Active",
+ },
+ },
+ "Type": "AWS::Lambda::Function",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "AssumeRolePolicyDocument": {
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Effect": "Allow",
+ "Principal": {
+ "Service": "lambda.amazonaws.com",
+ },
+ },
+ ],
+ "Version": "2012-10-17",
+ },
+ "ManagedPolicyArns": [
+ {
+ "Fn::Join": [
+ "",
+ [
+ "arn:",
+ {
+ "Ref": "AWS::Partition",
+ },
+ ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole",
+ ],
+ ],
+ },
+ ],
+ },
+ "Type": "AWS::IAM::Role",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "PolicyDocument": {
+ "Statement": [
+ {
+ "Action": [
+ "xray:PutTraceSegments",
+ "xray:PutTelemetryRecords",
+ ],
+ "Effect": "Allow",
+ "Resource": "*",
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "dynamodb:BatchGetItem",
+ "dynamodb:GetRecords",
+ "dynamodb:GetShardIterator",
+ "dynamodb:Query",
+ "dynamodb:GetItem",
+ "dynamodb:Scan",
+ "dynamodb:ConditionCheckItem",
+ "dynamodb:BatchWriteItem",
+ "dynamodb:PutItem",
+ "dynamodb:UpdateItem",
+ "dynamodb:DeleteItem",
+ "dynamodb:DescribeTable",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesKnownVersions1480B28B",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubDenyListBucket1B3C2C2E",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubDenyListBucket1B3C2C2E",
+ "Arn",
+ ],
+ },
+ "/*",
+ ],
+ ],
+ },
+ ],
+ },
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubLicenseListBucket9334047F",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubLicenseListBucket9334047F",
+ "Arn",
+ ],
+ },
+ "/*",
+ ],
+ ],
+ },
+ ],
+ },
+ {
+ "Action": "lambda:InvokeFunction",
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ "Arn",
+ ],
+ },
+ ":*",
+ ],
+ ],
+ },
+ ],
+ },
+ ],
+ "Version": "2012-10-17",
+ },
+ "PolicyName": "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4",
+ "Roles": [
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
+ },
+ ],
+ },
+ "Type": "AWS::IAM::Policy",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorDLQEED56E2F4AC5AB3E": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "BatchSize": 10,
+ "EventSourceArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "FunctionName": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "FunctionResponseTypes": [
+ "ReportBatchItemFailures",
+ ],
+ },
+ "Type": "AWS::Lambda::EventSourceMapping",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "BatchSize": 10,
+ "EventSourceArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ "FunctionName": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "FunctionResponseTypes": [
+ "ReportBatchItemFailures",
+ ],
+ "ScalingConfig": {
+ "MaximumConcurrency": 2,
+ },
+ },
+ "Type": "AWS::Lambda::EventSourceMapping",
+ },
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ "RedrivePolicy": {
+ "deadLetterTargetArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "maxReceiveCount": 10,
+ },
+ "VisibilityTimeout": 1800,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
"ConstructHubSourcesReStagePackageVersion5E1211D1": {
"DependsOn": [
"ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40",
@@ -26401,7 +27317,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "2bc29d2913a9accb30972e2ebbb8cf405efc9e8b73ea4605c359d99907fa0dbd.zip",
+ "S3Key": "3667bd11e9ba1e2476b569411bb892b311eda3d4b8dbe5880034486059b18bed.zip",
},
"Description": "Manually re-stage a package version",
"Environment": {
@@ -29991,6 +30907,25 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
+ ")\\n[button:Packument Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ ")\\n[button:Packument Processor](/lambda/home#/functions/",
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
@@ -30039,7 +30974,32 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
+ "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Missing Packuments","expression":"FILL(m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackuments",{"label":"Missing Packuments","stat":"Sum","visible":false,"id":"m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62"}],[{"label":"Missing Packuments Recovered","expression":"FILL(m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentsRecovered",{"label":"Missing Packuments Recovered","stat":"Sum","visible":false,"id":"m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7"}],[{"label":"Missing Packument Give-Ups","expression":"FILL(m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentGiveUps",{"label":"Missing Packument Give-Ups","stat":"Sum","visible":false,"id":"m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Packument Queue","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Waiting","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Waiting for next attempt","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesDelayed","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Delayed","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -39344,7 +40304,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "13418b75fd3f428b357c909862c8ea4c195836de07299f56105c6e3175416cc5.zip",
+ "S3Key": "93295fa0a1f1d54bc071e064c57c7713c38f4487ba3fcb3d574aa9536b376cd0.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages",
"Environment": {
@@ -39367,6 +40327,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Ref": "ConstructHubLicenseListBucket9334047F",
},
"LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "PACKUMENT_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ },
},
},
"Handler": "index.handler",
@@ -39801,27 +40764,6 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
- "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": {
- "Properties": {
- "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed
-for one or more packages, even after repeated re-checks. All versions the registry did serve
-have been processed, but a version announced by the feed may be missing until the affected
-package publishes again. The follower logs identify the affected packages, which can be
-manually ingested using the ReStagePackageVersion function.
-
-Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
- "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps",
- "ComparisonOperator": "GreaterThanOrEqualToThreshold",
- "EvaluationPeriods": 1,
- "MetricName": "LaggyPackumentGiveUps",
- "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
- "Period": 3600,
- "Statistic": "Sum",
- "Threshold": 1,
- "TreatMissingData": "notBreaching",
- },
- "Type": "AWS::CloudWatch::Alarm",
- },
"ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": {
"Properties": {
"AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed
@@ -39842,6 +40784,25 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerMissingPackumentsHigh0AB93A32": {
+ "Properties": {
+ "AlarmDescription": "The npm registry had no packument for an unusually high number of packages announced by
+the CouchDB changes feed. This usually means the registry has an outage or is far behind
+the feed. The packument processor keeps re-checking these packages.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "MetricName": "MissingPackuments",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 10800,
+ "Statistic": "Sum",
+ "Threshold": 1500,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": {
"Properties": {
"AlarmDescription": {
@@ -39925,7 +40886,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Properties": {
"ExecuteOnHandlerChange": false,
"HandlerArn": {
- "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5",
+ "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681",
},
"InvocationType": "Event",
"ServiceToken": {
@@ -39949,7 +40910,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Properties": {
"ExecuteOnHandlerChange": false,
"HandlerArn": {
- "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5",
+ "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681",
},
"InvocationType": "RequestResponse",
"ServiceToken": {
@@ -39963,7 +40924,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Type": "Custom::Trigger",
"UpdateReplacePolicy": "Delete",
},
- "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5": {
+ "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681": {
"DependsOn": [
"ConstructHubLicenseListAwsCliLayer59592811",
"ConstructHubLicenseListCustomResource323F0FD4",
@@ -39990,7 +40951,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "1e7b1fb4509d2b2ef4b63b0ad82abf8b1e820a8920419db90eff23d9392dbd49.zip",
+ "S3Key": "8d0e24c004f0022d0e6e92031e898d0eb9643d03e18273a9abc71da4fe2820d3.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table",
"Environment": {
@@ -40158,6 +41119,64 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::IAM::Policy",
},
+ "ConstructHubSourcesNpmJsPackumentProcessorFailuresC6FC837A": {
+ "Properties": {
+ "AlarmDescription": {
+ "Fn::Join": [
+ "",
+ [
+ "The packument processor function is failing (errors or time-outs). Requests in the
+packument queue use up their attempts while it fails, and may be given up on early.
+
+Link to the lambda function: /lambda/home#/functions/",
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ ],
+ ],
+ },
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/Failures",
+ "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+ "Dimensions": [
+ {
+ "Name": "FunctionName",
+ "Value": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ },
+ ],
+ "EvaluationPeriods": 2,
+ "MetricName": "Errors",
+ "Namespace": "AWS/Lambda",
+ "Period": 900,
+ "Statistic": "Sum",
+ "Threshold": 1,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
+ "ConstructHubSourcesNpmJsPackumentProcessorLaggyPackumentLagHigh863AFDFC": {
+ "Properties": {
+ "AlarmDescription": "Laggy packuments take unusually long to catch up: the npm registry serves older revisions
+than the CouchDB changes feed announced, for hours. This usually means the registry has an
+outage or is far behind the feed. The packument processor keeps re-checking these packages.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "ExtendedStatistic": "p90",
+ "MetricName": "LaggyPackumentLag",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 10800,
+ "Threshold": 21600000,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsSchedule34031870": {
"DependsOn": [
"ConstructHubSourcesNpmJsFollowerFailures86BCBA0D",
@@ -40429,6 +41448,20 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "sqs:SendMessage",
+ "sqs:GetQueueAttributes",
+ "sqs:GetQueueUrl",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ },
],
"Version": "2012-10-17",
},
@@ -40454,7 +41487,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "ef94880eff6f41104133b16d56770947ee1a674effd8a6ae47b17b3e604ed377.zip",
+ "S3Key": "e3822146d5e137d4bf9e0ac71dbab1cd07c63186f8140df399e0f49b919c8838.zip",
},
"DeadLetterConfig": {
"TargetArn": {
@@ -40780,10 +41813,21 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::S3::BucketPolicy",
},
- "ConstructHubSourcesReStagePackageVersion5E1211D1": {
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
+ "ConstructHubSourcesPackumentProcessorFunction0E5F8177": {
"DependsOn": [
- "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40",
- "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA",
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4",
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
],
"Properties": {
"Architectures": [
@@ -40793,31 +41837,58 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "2bc29d2913a9accb30972e2ebbb8cf405efc9e8b73ea4605c359d99907fa0dbd.zip",
+ "S3Key": "6ae1eb05cba8999261a20927e8c15ffccd256111d9fe1b62c24b9a0ca1ed11e3.zip",
},
- "Description": "Manually re-stage a package version",
+ "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed",
"Environment": {
"Variables": {
+ "AWS_EMF_ENVIRONMENT": "Local",
+ "DENY_LIST_BUCKET_NAME": {
+ "Ref": "ConstructHubDenyListBucket1B3C2C2E",
+ },
+ "DENY_LIST_OBJECT_KEY": "deny-list.json",
"FUNCTION_NAME": {
"Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
},
- "REGISTRY_URL": "https://registry.npmjs.org",
+ "KNOWN_VERSIONS_TABLE_NAME": {
+ "Ref": "ConstructHubSourcesKnownVersions1480B28B",
+ },
+ "LICENSE_LIST_BUCKET_NAME": {
+ "Ref": "ConstructHubLicenseListBucket9334047F",
+ },
+ "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "PACKUMENT_DEAD_LETTER_QUEUE_ARN": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "PACKUMENT_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ },
},
},
"Handler": "index.handler",
- "MemorySize": 1024,
+ "MemorySize": 2048,
"Role": {
"Fn::GetAtt": [
- "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA",
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
"Arn",
],
},
"Runtime": "nodejs22.x",
- "Timeout": 10,
+ "Timeout": 300,
+ "TracingConfig": {
+ "Mode": "Active",
+ },
},
"Type": "AWS::Lambda::Function",
},
- "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA": {
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
@@ -40848,7 +41919,310 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::IAM::Role",
},
- "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40": {
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "PolicyDocument": {
+ "Statement": [
+ {
+ "Action": [
+ "xray:PutTraceSegments",
+ "xray:PutTelemetryRecords",
+ ],
+ "Effect": "Allow",
+ "Resource": "*",
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "dynamodb:BatchGetItem",
+ "dynamodb:GetRecords",
+ "dynamodb:GetShardIterator",
+ "dynamodb:Query",
+ "dynamodb:GetItem",
+ "dynamodb:Scan",
+ "dynamodb:ConditionCheckItem",
+ "dynamodb:BatchWriteItem",
+ "dynamodb:PutItem",
+ "dynamodb:UpdateItem",
+ "dynamodb:DeleteItem",
+ "dynamodb:DescribeTable",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesKnownVersions1480B28B",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubDenyListBucket1B3C2C2E",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubDenyListBucket1B3C2C2E",
+ "Arn",
+ ],
+ },
+ "/*",
+ ],
+ ],
+ },
+ ],
+ },
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubLicenseListBucket9334047F",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubLicenseListBucket9334047F",
+ "Arn",
+ ],
+ },
+ "/*",
+ ],
+ ],
+ },
+ ],
+ },
+ {
+ "Action": "lambda:InvokeFunction",
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ "Arn",
+ ],
+ },
+ ":*",
+ ],
+ ],
+ },
+ ],
+ },
+ ],
+ "Version": "2012-10-17",
+ },
+ "PolicyName": "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4",
+ "Roles": [
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
+ },
+ ],
+ },
+ "Type": "AWS::IAM::Policy",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorDLQEED56E2F4AC5AB3E": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "BatchSize": 10,
+ "EventSourceArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "FunctionName": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "FunctionResponseTypes": [
+ "ReportBatchItemFailures",
+ ],
+ },
+ "Type": "AWS::Lambda::EventSourceMapping",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "BatchSize": 10,
+ "EventSourceArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ "FunctionName": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "FunctionResponseTypes": [
+ "ReportBatchItemFailures",
+ ],
+ "ScalingConfig": {
+ "MaximumConcurrency": 2,
+ },
+ },
+ "Type": "AWS::Lambda::EventSourceMapping",
+ },
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ "RedrivePolicy": {
+ "deadLetterTargetArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "maxReceiveCount": 10,
+ },
+ "VisibilityTimeout": 1800,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
+ "ConstructHubSourcesReStagePackageVersion5E1211D1": {
+ "DependsOn": [
+ "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40",
+ "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA",
+ ],
+ "Properties": {
+ "Architectures": [
+ "arm64",
+ ],
+ "Code": {
+ "S3Bucket": {
+ "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
+ },
+ "S3Key": "3667bd11e9ba1e2476b569411bb892b311eda3d4b8dbe5880034486059b18bed.zip",
+ },
+ "Description": "Manually re-stage a package version",
+ "Environment": {
+ "Variables": {
+ "FUNCTION_NAME": {
+ "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ },
+ "REGISTRY_URL": "https://registry.npmjs.org",
+ },
+ },
+ "Handler": "index.handler",
+ "MemorySize": 1024,
+ "Role": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA",
+ "Arn",
+ ],
+ },
+ "Runtime": "nodejs22.x",
+ "Timeout": 10,
+ },
+ "Type": "AWS::Lambda::Function",
+ },
+ "ConstructHubSourcesReStagePackageVersionServiceRoleCDF009BA": {
+ "Properties": {
+ "AssumeRolePolicyDocument": {
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Effect": "Allow",
+ "Principal": {
+ "Service": "lambda.amazonaws.com",
+ },
+ },
+ ],
+ "Version": "2012-10-17",
+ },
+ "ManagedPolicyArns": [
+ {
+ "Fn::Join": [
+ "",
+ [
+ "arn:",
+ {
+ "Ref": "AWS::Partition",
+ },
+ ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole",
+ ],
+ ],
+ },
+ ],
+ },
+ "Type": "AWS::IAM::Role",
+ },
+ "ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40": {
"Properties": {
"PolicyDocument": {
"Statement": [
@@ -44531,6 +45905,25 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
+ ")\\n[button:Packument Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ ")\\n[button:Packument Processor](/lambda/home#/functions/",
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
@@ -44579,7 +45972,32 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
+ "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Missing Packuments","expression":"FILL(m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackuments",{"label":"Missing Packuments","stat":"Sum","visible":false,"id":"m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62"}],[{"label":"Missing Packuments Recovered","expression":"FILL(m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentsRecovered",{"label":"Missing Packuments Recovered","stat":"Sum","visible":false,"id":"m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7"}],[{"label":"Missing Packument Give-Ups","expression":"FILL(m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentGiveUps",{"label":"Missing Packument Give-Ups","stat":"Sum","visible":false,"id":"m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Packument Queue","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Waiting","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Waiting for next attempt","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesDelayed","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Delayed","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -53893,7 +55311,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "13418b75fd3f428b357c909862c8ea4c195836de07299f56105c6e3175416cc5.zip",
+ "S3Key": "93295fa0a1f1d54bc071e064c57c7713c38f4487ba3fcb3d574aa9536b376cd0.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages",
"Environment": {
@@ -53916,6 +55334,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Ref": "ConstructHubLicenseListBucket9334047F",
},
"LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "PACKUMENT_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ },
},
},
"Handler": "index.handler",
@@ -54337,27 +55758,6 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
- "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": {
- "Properties": {
- "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed
-for one or more packages, even after repeated re-checks. All versions the registry did serve
-have been processed, but a version announced by the feed may be missing until the affected
-package publishes again. The follower logs identify the affected packages, which can be
-manually ingested using the ReStagePackageVersion function.
-
-Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
- "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps",
- "ComparisonOperator": "GreaterThanOrEqualToThreshold",
- "EvaluationPeriods": 1,
- "MetricName": "LaggyPackumentGiveUps",
- "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
- "Period": 3600,
- "Statistic": "Sum",
- "Threshold": 1,
- "TreatMissingData": "notBreaching",
- },
- "Type": "AWS::CloudWatch::Alarm",
- },
"ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": {
"Properties": {
"AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed
@@ -54378,6 +55778,25 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerMissingPackumentsHigh0AB93A32": {
+ "Properties": {
+ "AlarmDescription": "The npm registry had no packument for an unusually high number of packages announced by
+the CouchDB changes feed. This usually means the registry has an outage or is far behind
+the feed. The packument processor keeps re-checking these packages.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "MetricName": "MissingPackuments",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 10800,
+ "Statistic": "Sum",
+ "Threshold": 1500,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": {
"Properties": {
"AlarmDescription": {
@@ -54461,7 +55880,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Properties": {
"ExecuteOnHandlerChange": false,
"HandlerArn": {
- "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5",
+ "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681",
},
"InvocationType": "Event",
"ServiceToken": {
@@ -54485,7 +55904,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Properties": {
"ExecuteOnHandlerChange": false,
"HandlerArn": {
- "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5",
+ "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681",
},
"InvocationType": "RequestResponse",
"ServiceToken": {
@@ -54499,7 +55918,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Type": "Custom::Trigger",
"UpdateReplacePolicy": "Delete",
},
- "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5": {
+ "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681": {
"DependsOn": [
"ConstructHubLicenseListAwsCliLayer59592811",
"ConstructHubLicenseListCustomResource323F0FD4",
@@ -54526,7 +55945,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "1e7b1fb4509d2b2ef4b63b0ad82abf8b1e820a8920419db90eff23d9392dbd49.zip",
+ "S3Key": "8d0e24c004f0022d0e6e92031e898d0eb9643d03e18273a9abc71da4fe2820d3.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table",
"Environment": {
@@ -54694,6 +56113,64 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::IAM::Policy",
},
+ "ConstructHubSourcesNpmJsPackumentProcessorFailuresC6FC837A": {
+ "Properties": {
+ "AlarmDescription": {
+ "Fn::Join": [
+ "",
+ [
+ "The packument processor function is failing (errors or time-outs). Requests in the
+packument queue use up their attempts while it fails, and may be given up on early.
+
+Link to the lambda function: /lambda/home#/functions/",
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ ],
+ ],
+ },
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/Failures",
+ "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+ "Dimensions": [
+ {
+ "Name": "FunctionName",
+ "Value": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ },
+ ],
+ "EvaluationPeriods": 2,
+ "MetricName": "Errors",
+ "Namespace": "AWS/Lambda",
+ "Period": 900,
+ "Statistic": "Sum",
+ "Threshold": 1,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
+ "ConstructHubSourcesNpmJsPackumentProcessorLaggyPackumentLagHigh863AFDFC": {
+ "Properties": {
+ "AlarmDescription": "Laggy packuments take unusually long to catch up: the npm registry serves older revisions
+than the CouchDB changes feed announced, for hours. This usually means the registry has an
+outage or is far behind the feed. The packument processor keeps re-checking these packages.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "ExtendedStatistic": "p90",
+ "MetricName": "LaggyPackumentLag",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 10800,
+ "Threshold": 21600000,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsSchedule34031870": {
"DependsOn": [
"ConstructHubSourcesNpmJsFollowerFailures86BCBA0D",
@@ -54965,6 +56442,20 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "sqs:SendMessage",
+ "sqs:GetQueueAttributes",
+ "sqs:GetQueueUrl",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ },
],
"Version": "2012-10-17",
},
@@ -54990,7 +56481,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "ef94880eff6f41104133b16d56770947ee1a674effd8a6ae47b17b3e604ed377.zip",
+ "S3Key": "e3822146d5e137d4bf9e0ac71dbab1cd07c63186f8140df399e0f49b919c8838.zip",
},
"DeadLetterConfig": {
"TargetArn": {
@@ -55316,6 +56807,347 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::S3::BucketPolicy",
},
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
+ "ConstructHubSourcesPackumentProcessorFunction0E5F8177": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4",
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
+ ],
+ "Properties": {
+ "Architectures": [
+ "arm64",
+ ],
+ "Code": {
+ "S3Bucket": {
+ "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
+ },
+ "S3Key": "6ae1eb05cba8999261a20927e8c15ffccd256111d9fe1b62c24b9a0ca1ed11e3.zip",
+ },
+ "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed",
+ "Environment": {
+ "Variables": {
+ "AWS_EMF_ENVIRONMENT": "Local",
+ "DENY_LIST_BUCKET_NAME": {
+ "Ref": "ConstructHubDenyListBucket1B3C2C2E",
+ },
+ "DENY_LIST_OBJECT_KEY": "deny-list.json",
+ "FUNCTION_NAME": {
+ "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ },
+ "KNOWN_VERSIONS_TABLE_NAME": {
+ "Ref": "ConstructHubSourcesKnownVersions1480B28B",
+ },
+ "LICENSE_LIST_BUCKET_NAME": {
+ "Ref": "ConstructHubLicenseListBucket9334047F",
+ },
+ "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "PACKUMENT_DEAD_LETTER_QUEUE_ARN": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "PACKUMENT_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ },
+ },
+ },
+ "Handler": "index.handler",
+ "MemorySize": 2048,
+ "Role": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
+ "Arn",
+ ],
+ },
+ "Runtime": "nodejs22.x",
+ "Timeout": 300,
+ "TracingConfig": {
+ "Mode": "Active",
+ },
+ },
+ "Type": "AWS::Lambda::Function",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "AssumeRolePolicyDocument": {
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Effect": "Allow",
+ "Principal": {
+ "Service": "lambda.amazonaws.com",
+ },
+ },
+ ],
+ "Version": "2012-10-17",
+ },
+ "ManagedPolicyArns": [
+ {
+ "Fn::Join": [
+ "",
+ [
+ "arn:",
+ {
+ "Ref": "AWS::Partition",
+ },
+ ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole",
+ ],
+ ],
+ },
+ ],
+ },
+ "Type": "AWS::IAM::Role",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "PolicyDocument": {
+ "Statement": [
+ {
+ "Action": [
+ "xray:PutTraceSegments",
+ "xray:PutTelemetryRecords",
+ ],
+ "Effect": "Allow",
+ "Resource": "*",
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "dynamodb:BatchGetItem",
+ "dynamodb:GetRecords",
+ "dynamodb:GetShardIterator",
+ "dynamodb:Query",
+ "dynamodb:GetItem",
+ "dynamodb:Scan",
+ "dynamodb:ConditionCheckItem",
+ "dynamodb:BatchWriteItem",
+ "dynamodb:PutItem",
+ "dynamodb:UpdateItem",
+ "dynamodb:DeleteItem",
+ "dynamodb:DescribeTable",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesKnownVersions1480B28B",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubDenyListBucket1B3C2C2E",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubDenyListBucket1B3C2C2E",
+ "Arn",
+ ],
+ },
+ "/*",
+ ],
+ ],
+ },
+ ],
+ },
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubLicenseListBucket9334047F",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubLicenseListBucket9334047F",
+ "Arn",
+ ],
+ },
+ "/*",
+ ],
+ ],
+ },
+ ],
+ },
+ {
+ "Action": "lambda:InvokeFunction",
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ "Arn",
+ ],
+ },
+ ":*",
+ ],
+ ],
+ },
+ ],
+ },
+ ],
+ "Version": "2012-10-17",
+ },
+ "PolicyName": "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4",
+ "Roles": [
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
+ },
+ ],
+ },
+ "Type": "AWS::IAM::Policy",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorDLQEED56E2F4AC5AB3E": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "BatchSize": 10,
+ "EventSourceArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "FunctionName": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "FunctionResponseTypes": [
+ "ReportBatchItemFailures",
+ ],
+ },
+ "Type": "AWS::Lambda::EventSourceMapping",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "BatchSize": 10,
+ "EventSourceArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ "FunctionName": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "FunctionResponseTypes": [
+ "ReportBatchItemFailures",
+ ],
+ "ScalingConfig": {
+ "MaximumConcurrency": 2,
+ },
+ },
+ "Type": "AWS::Lambda::EventSourceMapping",
+ },
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ "RedrivePolicy": {
+ "deadLetterTargetArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "maxReceiveCount": 10,
+ },
+ "VisibilityTimeout": 1800,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
"ConstructHubSourcesReStagePackageVersion5E1211D1": {
"DependsOn": [
"ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40",
@@ -55329,7 +57161,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "2bc29d2913a9accb30972e2ebbb8cf405efc9e8b73ea4605c359d99907fa0dbd.zip",
+ "S3Key": "3667bd11e9ba1e2476b569411bb892b311eda3d4b8dbe5880034486059b18bed.zip",
},
"Description": "Manually re-stage a package version",
"Environment": {
@@ -59256,6 +61088,25 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
+ ")\\n[button:Packument Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ ")\\n[button:Packument Processor](/lambda/home#/functions/",
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
@@ -59304,7 +61155,32 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
+ "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Missing Packuments","expression":"FILL(m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackuments",{"label":"Missing Packuments","stat":"Sum","visible":false,"id":"m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62"}],[{"label":"Missing Packuments Recovered","expression":"FILL(m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentsRecovered",{"label":"Missing Packuments Recovered","stat":"Sum","visible":false,"id":"m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7"}],[{"label":"Missing Packument Give-Ups","expression":"FILL(m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentGiveUps",{"label":"Missing Packument Give-Ups","stat":"Sum","visible":false,"id":"m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Packument Queue","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Waiting","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Waiting for next attempt","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesDelayed","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Delayed","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -68425,7 +70301,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "13418b75fd3f428b357c909862c8ea4c195836de07299f56105c6e3175416cc5.zip",
+ "S3Key": "93295fa0a1f1d54bc071e064c57c7713c38f4487ba3fcb3d574aa9536b376cd0.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages",
"Environment": {
@@ -68448,6 +70324,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
"Ref": "ConstructHubLicenseListBucket9334047F",
},
"LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "PACKUMENT_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ },
},
},
"Handler": "index.handler",
@@ -68882,27 +70761,6 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
- "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": {
- "Properties": {
- "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed
-for one or more packages, even after repeated re-checks. All versions the registry did serve
-have been processed, but a version announced by the feed may be missing until the affected
-package publishes again. The follower logs identify the affected packages, which can be
-manually ingested using the ReStagePackageVersion function.
-
-Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
- "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps",
- "ComparisonOperator": "GreaterThanOrEqualToThreshold",
- "EvaluationPeriods": 1,
- "MetricName": "LaggyPackumentGiveUps",
- "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
- "Period": 3600,
- "Statistic": "Sum",
- "Threshold": 1,
- "TreatMissingData": "notBreaching",
- },
- "Type": "AWS::CloudWatch::Alarm",
- },
"ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": {
"Properties": {
"AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed
@@ -68923,6 +70781,25 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerMissingPackumentsHigh0AB93A32": {
+ "Properties": {
+ "AlarmDescription": "The npm registry had no packument for an unusually high number of packages announced by
+the CouchDB changes feed. This usually means the registry has an outage or is far behind
+the feed. The packument processor keeps re-checking these packages.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "MetricName": "MissingPackuments",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 10800,
+ "Statistic": "Sum",
+ "Threshold": 1500,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": {
"Properties": {
"AlarmDescription": {
@@ -69006,7 +70883,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Properties": {
"ExecuteOnHandlerChange": false,
"HandlerArn": {
- "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5",
+ "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681",
},
"InvocationType": "Event",
"ServiceToken": {
@@ -69030,7 +70907,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Properties": {
"ExecuteOnHandlerChange": false,
"HandlerArn": {
- "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5",
+ "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681",
},
"InvocationType": "RequestResponse",
"ServiceToken": {
@@ -69044,7 +70921,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Type": "Custom::Trigger",
"UpdateReplacePolicy": "Delete",
},
- "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7010a536594a81bc0e746f3dedc3e72bad5": {
+ "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701a803e65028faaeaf3f70c9da5535b681": {
"DependsOn": [
"ConstructHubLicenseListAwsCliLayer59592811",
"ConstructHubLicenseListCustomResource323F0FD4",
@@ -69071,7 +70948,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "1e7b1fb4509d2b2ef4b63b0ad82abf8b1e820a8920419db90eff23d9392dbd49.zip",
+ "S3Key": "8d0e24c004f0022d0e6e92031e898d0eb9643d03e18273a9abc71da4fe2820d3.zip",
},
"Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table",
"Environment": {
@@ -69239,6 +71116,64 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::IAM::Policy",
},
+ "ConstructHubSourcesNpmJsPackumentProcessorFailuresC6FC837A": {
+ "Properties": {
+ "AlarmDescription": {
+ "Fn::Join": [
+ "",
+ [
+ "The packument processor function is failing (errors or time-outs). Requests in the
+packument queue use up their attempts while it fails, and may be given up on early.
+
+Link to the lambda function: /lambda/home#/functions/",
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ ],
+ ],
+ },
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/Failures",
+ "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+ "Dimensions": [
+ {
+ "Name": "FunctionName",
+ "Value": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ },
+ ],
+ "EvaluationPeriods": 2,
+ "MetricName": "Errors",
+ "Namespace": "AWS/Lambda",
+ "Period": 900,
+ "Statistic": "Sum",
+ "Threshold": 1,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
+ "ConstructHubSourcesNpmJsPackumentProcessorLaggyPackumentLagHigh863AFDFC": {
+ "Properties": {
+ "AlarmDescription": "Laggy packuments take unusually long to catch up: the npm registry serves older revisions
+than the CouchDB changes feed announced, for hours. This usually means the registry has an
+outage or is far behind the feed. The packument processor keeps re-checking these packages.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "Test/ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "ExtendedStatistic": "p90",
+ "MetricName": "LaggyPackumentLag",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 10800,
+ "Threshold": 21600000,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsSchedule34031870": {
"DependsOn": [
"ConstructHubSourcesNpmJsFollowerFailures86BCBA0D",
@@ -69510,6 +71445,20 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "sqs:SendMessage",
+ "sqs:GetQueueAttributes",
+ "sqs:GetQueueUrl",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ },
],
"Version": "2012-10-17",
},
@@ -69535,7 +71484,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "ef94880eff6f41104133b16d56770947ee1a674effd8a6ae47b17b3e604ed377.zip",
+ "S3Key": "e3822146d5e137d4bf9e0ac71dbab1cd07c63186f8140df399e0f49b919c8838.zip",
},
"DeadLetterConfig": {
"TargetArn": {
@@ -69861,6 +71810,347 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::S3::BucketPolicy",
},
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
+ "ConstructHubSourcesPackumentProcessorFunction0E5F8177": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4",
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
+ ],
+ "Properties": {
+ "Architectures": [
+ "arm64",
+ ],
+ "Code": {
+ "S3Bucket": {
+ "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
+ },
+ "S3Key": "6ae1eb05cba8999261a20927e8c15ffccd256111d9fe1b62c24b9a0ca1ed11e3.zip",
+ },
+ "Description": "[Test/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed",
+ "Environment": {
+ "Variables": {
+ "AWS_EMF_ENVIRONMENT": "Local",
+ "DENY_LIST_BUCKET_NAME": {
+ "Ref": "ConstructHubDenyListBucket1B3C2C2E",
+ },
+ "DENY_LIST_OBJECT_KEY": "deny-list.json",
+ "FUNCTION_NAME": {
+ "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ },
+ "KNOWN_VERSIONS_TABLE_NAME": {
+ "Ref": "ConstructHubSourcesKnownVersions1480B28B",
+ },
+ "LICENSE_LIST_BUCKET_NAME": {
+ "Ref": "ConstructHubLicenseListBucket9334047F",
+ },
+ "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "PACKUMENT_DEAD_LETTER_QUEUE_ARN": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "PACKUMENT_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ },
+ },
+ },
+ "Handler": "index.handler",
+ "MemorySize": 2048,
+ "Role": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
+ "Arn",
+ ],
+ },
+ "Runtime": "nodejs22.x",
+ "Timeout": 300,
+ "TracingConfig": {
+ "Mode": "Active",
+ },
+ },
+ "Type": "AWS::Lambda::Function",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "AssumeRolePolicyDocument": {
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Effect": "Allow",
+ "Principal": {
+ "Service": "lambda.amazonaws.com",
+ },
+ },
+ ],
+ "Version": "2012-10-17",
+ },
+ "ManagedPolicyArns": [
+ {
+ "Fn::Join": [
+ "",
+ [
+ "arn:",
+ {
+ "Ref": "AWS::Partition",
+ },
+ ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole",
+ ],
+ ],
+ },
+ ],
+ },
+ "Type": "AWS::IAM::Role",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "PolicyDocument": {
+ "Statement": [
+ {
+ "Action": [
+ "xray:PutTraceSegments",
+ "xray:PutTelemetryRecords",
+ ],
+ "Effect": "Allow",
+ "Resource": "*",
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "dynamodb:BatchGetItem",
+ "dynamodb:GetRecords",
+ "dynamodb:GetShardIterator",
+ "dynamodb:Query",
+ "dynamodb:GetItem",
+ "dynamodb:Scan",
+ "dynamodb:ConditionCheckItem",
+ "dynamodb:BatchWriteItem",
+ "dynamodb:PutItem",
+ "dynamodb:UpdateItem",
+ "dynamodb:DeleteItem",
+ "dynamodb:DescribeTable",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesKnownVersions1480B28B",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubDenyListBucket1B3C2C2E",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubDenyListBucket1B3C2C2E",
+ "Arn",
+ ],
+ },
+ "/*",
+ ],
+ ],
+ },
+ ],
+ },
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubLicenseListBucket9334047F",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubLicenseListBucket9334047F",
+ "Arn",
+ ],
+ },
+ "/*",
+ ],
+ ],
+ },
+ ],
+ },
+ {
+ "Action": "lambda:InvokeFunction",
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ "Arn",
+ ],
+ },
+ ":*",
+ ],
+ ],
+ },
+ ],
+ },
+ ],
+ "Version": "2012-10-17",
+ },
+ "PolicyName": "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4",
+ "Roles": [
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
+ },
+ ],
+ },
+ "Type": "AWS::IAM::Policy",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorDLQEED56E2F4AC5AB3E": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "BatchSize": 10,
+ "EventSourceArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "FunctionName": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "FunctionResponseTypes": [
+ "ReportBatchItemFailures",
+ ],
+ },
+ "Type": "AWS::Lambda::EventSourceMapping",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourceTestConstructHubSourcesPackumentProcessorQueue026064FBB2863D38": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "BatchSize": 10,
+ "EventSourceArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ "FunctionName": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "FunctionResponseTypes": [
+ "ReportBatchItemFailures",
+ ],
+ "ScalingConfig": {
+ "MaximumConcurrency": 2,
+ },
+ },
+ "Type": "AWS::Lambda::EventSourceMapping",
+ },
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ "RedrivePolicy": {
+ "deadLetterTargetArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "maxReceiveCount": 10,
+ },
+ "VisibilityTimeout": 1800,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
"ConstructHubSourcesReStagePackageVersion5E1211D1": {
"DependsOn": [
"ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40",
@@ -69874,7 +72164,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "2bc29d2913a9accb30972e2ebbb8cf405efc9e8b73ea4605c359d99907fa0dbd.zip",
+ "S3Key": "3667bd11e9ba1e2476b569411bb892b311eda3d4b8dbe5880034486059b18bed.zip",
},
"Description": "Manually re-stage a package version",
"Environment": {
diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap
index 999fe3486..930aad7ae 100644
--- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap
+++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap
@@ -833,6 +833,25 @@ def sort_filter_rules(json_obj):
"QueueName",
],
},
+ ")\\n[button:Packument Queue](/sqs/v2/home#/queues/https%3A%2F%2Fsqs.",
+ {
+ "Ref": "AWS::Region",
+ },
+ ".amazonaws.com%2F",
+ {
+ "Ref": "AWS::AccountId",
+ },
+ "%2F",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ ")\\n[button:Packument Processor](/lambda/home#/functions/",
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
")\\n[button:Pipeline Trace (Log Analytics)](/cloudwatch/home#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-86400~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2C*20*40log*2C*20*40message*0A*7C*20filter*20*40message*20like*20*2Fconstruct-hub-probe*2F*0A*23*20filter*20*40message*20like*20*2F*3Cversion*3E*2F*20*3C-*20narrow*20down*20to*20a*20stuck*20version*0A*7C*20sort*20*40timestamp*20desc*0A*7C*20limit*20100~source~(~'*2Faws*2Flambda*2F",
{
"Ref": "ConstructHubSourcesNpmJs15A77D2D",
@@ -881,7 +900,32 @@ def sort_filter_rules(json_obj):
{
"Ref": "AWS::Region",
},
- "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
+ "","metrics":[[{"label":"Late Changes","expression":"FILL(m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeCount",{"label":"Late Changes","stat":"Sum","visible":false,"id":"m97b26d9c3912f8122fc8cbc26c10d7e362f10a1b01dd7b6fc615793a0614ffd5"}],[{"label":"Laggy Packuments","expression":"FILL(m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackuments",{"label":"Laggy Packuments","stat":"Maximum","visible":false,"id":"m7542c42c2a42804f45673f1b7a1e208ef9657546e83a7fed66558c0c9a9458fc"}],[{"label":"Laggy Packuments Recovered","expression":"FILL(meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentsRecovered",{"label":"Laggy Packuments Recovered","stat":"Sum","visible":false,"id":"meebc873175632650e3dbd407c0e22f4395c7b945fffbd66f17cc225c7f3c5ab7"}],[{"label":"Laggy Packument Give-Ups","expression":"FILL(m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentGiveUps",{"label":"Laggy Packument Give-Ups","stat":"Sum","visible":false,"id":"m25c8aa905abbcebe79405fb8d308d3b26cc11789e67b7b432396024cc3688665"}],[{"label":"Missing Packuments","expression":"FILL(m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackuments",{"label":"Missing Packuments","stat":"Sum","visible":false,"id":"m01772d2035c66bb758d8af88685f9a5f508aac1982cd26d7b1ed107a5e948e62"}],[{"label":"Missing Packuments Recovered","expression":"FILL(m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentsRecovered",{"label":"Missing Packuments Recovered","stat":"Sum","visible":false,"id":"m65c59efca9f729008affb3924f59d4dc66e24297ee63442794b259b64a2a85e7"}],[{"label":"Missing Packument Give-Ups","expression":"FILL(m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MissingPackumentGiveUps",{"label":"Missing Packument Give-Ups","stat":"Sum","visible":false,"id":"m853f955d255e0171792576a2bb4142132ad90c73ede7d119e6bdd8c383111e72"}],[{"label":"Metadata Fetch Failures","expression":"FILL(mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb, 0)"}],["ConstructHub/PackageSource/NpmJs/Follower","MetadataFetchFailures",{"label":"Metadata Fetch Failures","stat":"Sum","visible":false,"id":"mb4ba2ab8081f9474c7f13c3726c05a8d8243b0432c76e4ce05db82f5f80bf5bb"}],["ConstructHub/PackageSource/NpmJs/Follower","LateChangeLag",{"label":"Late Change Lag (max)","stat":"Maximum","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Scan Window","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":12,"y":71,"properties":{"view":"timeSeries","title":"Packument Queue","region":"",
+ {
+ "Ref": "AWS::Region",
+ },
+ "","metrics":[["AWS/SQS","ApproximateNumberOfMessagesVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Waiting","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesNotVisible","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Waiting for next attempt","stat":"Maximum"}],["AWS/SQS","ApproximateNumberOfMessagesDelayed","QueueName","",
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "QueueName",
+ ],
+ },
+ "",{"label":"Delayed","stat":"Maximum"}],["ConstructHub/PackageSource/NpmJs/Follower","LaggyPackumentLag",{"label":"Laggy Packument Lag (p90)","stat":"p90","yAxis":"right"}]],"annotations":{"horizontal":[{"color":"#ff0000","label":"Alarm","value":21600000,"yAxis":"right"}]},"yAxis":{"left":{"min":0},"right":{"label":"Milliseconds","min":0,"showUnits":false}},"period":300}},{"type":"metric","width":12,"height":6,"x":0,"y":77,"properties":{"view":"timeSeries","title":"Stager Dead-Letter Queue","region":"",
{
"Ref": "AWS::Region",
},
@@ -11399,7 +11443,7 @@ Direct link to the function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "13418b75fd3f428b357c909862c8ea4c195836de07299f56105c6e3175416cc5.zip",
+ "S3Key": "93295fa0a1f1d54bc071e064c57c7713c38f4487ba3fcb3d574aa9536b376cd0.zip",
},
"Description": "[dev/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages",
"Environment": {
@@ -11422,6 +11466,9 @@ Direct link to the function: /lambda/home#/functions/",
"Ref": "ConstructHubLicenseListBucket9334047F",
},
"LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "PACKUMENT_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ },
},
},
"Handler": "index.handler",
@@ -11853,27 +11900,6 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::CloudWatch::Alarm",
},
- "ConstructHubSourcesNpmJsFollowerLaggyPackumentGiveUps9CFECACE": {
- "Properties": {
- "AlarmDescription": "The npm registry never served the packument revision announced by the CouchDB changes feed
-for one or more packages, even after repeated re-checks. All versions the registry did serve
-have been processed, but a version announced by the feed may be missing until the affected
-package publishes again. The follower logs identify the affected packages, which can be
-manually ingested using the ReStagePackageVersion function.
-
-Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
- "AlarmName": "dev/ConstructHub/Sources/NpmJs/Follower/LaggyPackumentGiveUps",
- "ComparisonOperator": "GreaterThanOrEqualToThreshold",
- "EvaluationPeriods": 1,
- "MetricName": "LaggyPackumentGiveUps",
- "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
- "Period": 3600,
- "Statistic": "Sum",
- "Threshold": 1,
- "TreatMissingData": "notBreaching",
- },
- "Type": "AWS::CloudWatch::Alarm",
- },
"ConstructHubSourcesNpmJsFollowerLateChangeLagHigh74C200AA": {
"Properties": {
"AlarmDescription": "The NpmJs follower is discovering changes that were inserted into the CouchDB changes feed
@@ -11894,6 +11920,25 @@ Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo
},
"Type": "AWS::CloudWatch::Alarm",
},
+ "ConstructHubSourcesNpmJsFollowerMissingPackumentsHigh0AB93A32": {
+ "Properties": {
+ "AlarmDescription": "The npm registry had no packument for an unusually high number of packages announced by
+the CouchDB changes feed. This usually means the registry has an outage or is far behind
+the feed. The packument processor keeps re-checking these packages.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "dev/ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "MetricName": "MissingPackuments",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 10800,
+ "Statistic": "Sum",
+ "Threshold": 1500,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsFollowerNoChanges2CB08C93": {
"Properties": {
"AlarmDescription": {
@@ -11974,7 +12019,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Properties": {
"ExecuteOnHandlerChange": false,
"HandlerArn": {
- "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701fcb8d7a0dc1df7ff16564a6fd439e58c",
+ "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7018914ee7ebabff12cadb11438c85d3bec",
},
"InvocationType": "Event",
"ServiceToken": {
@@ -11998,7 +12043,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Properties": {
"ExecuteOnHandlerChange": false,
"HandlerArn": {
- "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701fcb8d7a0dc1df7ff16564a6fd439e58c",
+ "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7018914ee7ebabff12cadb11438c85d3bec",
},
"InvocationType": "RequestResponse",
"ServiceToken": {
@@ -12012,7 +12057,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"Type": "Custom::Trigger",
"UpdateReplacePolicy": "Delete",
},
- "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A701fcb8d7a0dc1df7ff16564a6fd439e58c": {
+ "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7018914ee7ebabff12cadb11438c85d3bec": {
"DependsOn": [
"ConstructHubLicenseListAwsCliLayer59592811",
"ConstructHubLicenseListCustomResource323F0FD4",
@@ -12039,7 +12084,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "1e7b1fb4509d2b2ef4b63b0ad82abf8b1e820a8920419db90eff23d9392dbd49.zip",
+ "S3Key": "8d0e24c004f0022d0e6e92031e898d0eb9643d03e18273a9abc71da4fe2820d3.zip",
},
"Description": "[dev/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table",
"Environment": {
@@ -12207,6 +12252,64 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::IAM::Policy",
},
+ "ConstructHubSourcesNpmJsPackumentProcessorFailuresC6FC837A": {
+ "Properties": {
+ "AlarmDescription": {
+ "Fn::Join": [
+ "",
+ [
+ "The packument processor function is failing (errors or time-outs). Requests in the
+packument queue use up their attempts while it fails, and may be given up on early.
+
+Link to the lambda function: /lambda/home#/functions/",
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ ],
+ ],
+ },
+ "AlarmName": "dev/ConstructHub/Sources/NpmJs/PackumentProcessor/Failures",
+ "ComparisonOperator": "GreaterThanOrEqualToThreshold",
+ "Dimensions": [
+ {
+ "Name": "FunctionName",
+ "Value": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ },
+ ],
+ "EvaluationPeriods": 2,
+ "MetricName": "Errors",
+ "Namespace": "AWS/Lambda",
+ "Period": 900,
+ "Statistic": "Sum",
+ "Threshold": 1,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
+ "ConstructHubSourcesNpmJsPackumentProcessorLaggyPackumentLagHigh863AFDFC": {
+ "Properties": {
+ "AlarmDescription": "Laggy packuments take unusually long to catch up: the npm registry serves older revisions
+than the CouchDB changes feed announced, for hours. This usually means the registry has an
+outage or is far behind the feed. The packument processor keeps re-checking these packages.
+
+Runbook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runbook.md",
+ "AlarmName": "dev/ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh",
+ "ComparisonOperator": "GreaterThanThreshold",
+ "EvaluationPeriods": 1,
+ "ExtendedStatistic": "p90",
+ "MetricName": "LaggyPackumentLag",
+ "Namespace": "ConstructHub/PackageSource/NpmJs/Follower",
+ "Period": 10800,
+ "Threshold": 21600000,
+ "TreatMissingData": "notBreaching",
+ },
+ "Type": "AWS::CloudWatch::Alarm",
+ },
"ConstructHubSourcesNpmJsSchedule34031870": {
"DependsOn": [
"ConstructHubSourcesNpmJsFollowerFailures86BCBA0D",
@@ -12478,6 +12581,20 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
],
},
+ {
+ "Action": [
+ "sqs:SendMessage",
+ "sqs:GetQueueAttributes",
+ "sqs:GetQueueUrl",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ },
],
"Version": "2012-10-17",
},
@@ -12503,7 +12620,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "ef94880eff6f41104133b16d56770947ee1a674effd8a6ae47b17b3e604ed377.zip",
+ "S3Key": "e3822146d5e137d4bf9e0ac71dbab1cd07c63186f8140df399e0f49b919c8838.zip",
},
"DeadLetterConfig": {
"TargetArn": {
@@ -12829,6 +12946,347 @@ Direct link to Lambda function: /lambda/home#/functions/",
},
"Type": "AWS::S3::BucketPolicy",
},
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
+ "ConstructHubSourcesPackumentProcessorFunction0E5F8177": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4",
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
+ ],
+ "Properties": {
+ "Architectures": [
+ "arm64",
+ ],
+ "Code": {
+ "S3Bucket": {
+ "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
+ },
+ "S3Key": "6ae1eb05cba8999261a20927e8c15ffccd256111d9fe1b62c24b9a0ca1ed11e3.zip",
+ },
+ "Description": "[dev/ConstructHub/Sources/PackumentProcessor] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed",
+ "Environment": {
+ "Variables": {
+ "AWS_EMF_ENVIRONMENT": "Local",
+ "DENY_LIST_BUCKET_NAME": {
+ "Ref": "ConstructHubDenyListBucket1B3C2C2E",
+ },
+ "DENY_LIST_OBJECT_KEY": "deny-list.json",
+ "FUNCTION_NAME": {
+ "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ },
+ "KNOWN_VERSIONS_TABLE_NAME": {
+ "Ref": "ConstructHubSourcesKnownVersions1480B28B",
+ },
+ "LICENSE_LIST_BUCKET_NAME": {
+ "Ref": "ConstructHubLicenseListBucket9334047F",
+ },
+ "LICENSE_LIST_OBJECT_KEY": "allowed-licenses.json",
+ "PACKUMENT_DEAD_LETTER_QUEUE_ARN": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "PACKUMENT_QUEUE_URL": {
+ "Ref": "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ },
+ },
+ },
+ "Handler": "index.handler",
+ "MemorySize": 2048,
+ "Role": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
+ "Arn",
+ ],
+ },
+ "Runtime": "nodejs22.x",
+ "Timeout": 300,
+ "TracingConfig": {
+ "Mode": "Active",
+ },
+ },
+ "Type": "AWS::Lambda::Function",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "AssumeRolePolicyDocument": {
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Effect": "Allow",
+ "Principal": {
+ "Service": "lambda.amazonaws.com",
+ },
+ },
+ ],
+ "Version": "2012-10-17",
+ },
+ "ManagedPolicyArns": [
+ {
+ "Fn::Join": [
+ "",
+ [
+ "arn:",
+ {
+ "Ref": "AWS::Partition",
+ },
+ ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole",
+ ],
+ ],
+ },
+ ],
+ },
+ "Type": "AWS::IAM::Role",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "PolicyDocument": {
+ "Statement": [
+ {
+ "Action": [
+ "xray:PutTraceSegments",
+ "xray:PutTelemetryRecords",
+ ],
+ "Effect": "Allow",
+ "Resource": "*",
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "sqs:ReceiveMessage",
+ "sqs:ChangeMessageVisibility",
+ "sqs:GetQueueUrl",
+ "sqs:DeleteMessage",
+ "sqs:GetQueueAttributes",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "dynamodb:BatchGetItem",
+ "dynamodb:GetRecords",
+ "dynamodb:GetShardIterator",
+ "dynamodb:Query",
+ "dynamodb:GetItem",
+ "dynamodb:Scan",
+ "dynamodb:ConditionCheckItem",
+ "dynamodb:BatchWriteItem",
+ "dynamodb:PutItem",
+ "dynamodb:UpdateItem",
+ "dynamodb:DeleteItem",
+ "dynamodb:DescribeTable",
+ ],
+ "Effect": "Allow",
+ "Resource": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesKnownVersions1480B28B",
+ "Arn",
+ ],
+ },
+ },
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubDenyListBucket1B3C2C2E",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubDenyListBucket1B3C2C2E",
+ "Arn",
+ ],
+ },
+ "/*",
+ ],
+ ],
+ },
+ ],
+ },
+ {
+ "Action": [
+ "s3:GetObject*",
+ "s3:GetBucket*",
+ "s3:List*",
+ ],
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubLicenseListBucket9334047F",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubLicenseListBucket9334047F",
+ "Arn",
+ ],
+ },
+ "/*",
+ ],
+ ],
+ },
+ ],
+ },
+ {
+ "Action": "lambda:InvokeFunction",
+ "Effect": "Allow",
+ "Resource": [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ "Arn",
+ ],
+ },
+ {
+ "Fn::Join": [
+ "",
+ [
+ {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesNpmJsStageAndNotify591C0CFA",
+ "Arn",
+ ],
+ },
+ ":*",
+ ],
+ ],
+ },
+ ],
+ },
+ ],
+ "Version": "2012-10-17",
+ },
+ "PolicyName": "ConstructHubSourcesPackumentProcessorFunctionServiceRoleDefaultPolicy4D1F4AD4",
+ "Roles": [
+ {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunctionServiceRole384F57F3",
+ },
+ ],
+ },
+ "Type": "AWS::IAM::Policy",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourcedevConstructHubSourcesPackumentProcessorDLQBB0F158EFDD9B731": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "BatchSize": 10,
+ "EventSourceArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "FunctionName": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "FunctionResponseTypes": [
+ "ReportBatchItemFailures",
+ ],
+ },
+ "Type": "AWS::Lambda::EventSourceMapping",
+ },
+ "ConstructHubSourcesPackumentProcessorFunctionSqsEventSourcedevConstructHubSourcesPackumentProcessorQueueB1811D0E3C6BE98A": {
+ "DependsOn": [
+ "ConstructHubLicenseListAwsCliLayer59592811",
+ "ConstructHubLicenseListCustomResource323F0FD4",
+ ],
+ "Properties": {
+ "BatchSize": 10,
+ "EventSourceArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578",
+ "Arn",
+ ],
+ },
+ "FunctionName": {
+ "Ref": "ConstructHubSourcesPackumentProcessorFunction0E5F8177",
+ },
+ "FunctionResponseTypes": [
+ "ReportBatchItemFailures",
+ ],
+ "ScalingConfig": {
+ "MaximumConcurrency": 2,
+ },
+ },
+ "Type": "AWS::Lambda::EventSourceMapping",
+ },
+ "ConstructHubSourcesPackumentProcessorQueue34BDE578": {
+ "DeletionPolicy": "Delete",
+ "Properties": {
+ "KmsMasterKeyId": "alias/aws/sqs",
+ "MessageRetentionPeriod": 1209600,
+ "RedrivePolicy": {
+ "deadLetterTargetArn": {
+ "Fn::GetAtt": [
+ "ConstructHubSourcesPackumentProcessorDLQF401BD77",
+ "Arn",
+ ],
+ },
+ "maxReceiveCount": 10,
+ },
+ "VisibilityTimeout": 1800,
+ },
+ "Type": "AWS::SQS::Queue",
+ "UpdateReplacePolicy": "Delete",
+ },
"ConstructHubSourcesReStagePackageVersion5E1211D1": {
"DependsOn": [
"ConstructHubSourcesReStagePackageVersionServiceRoleDefaultPolicyB4F9CE40",
@@ -12842,7 +13300,7 @@ Direct link to Lambda function: /lambda/home#/functions/",
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}",
},
- "S3Key": "2bc29d2913a9accb30972e2ebbb8cf405efc9e8b73ea4605c359d99907fa0dbd.zip",
+ "S3Key": "3667bd11e9ba1e2476b569411bb892b311eda3d4b8dbe5880034486059b18bed.zip",
},
"Description": "Manually re-stage a package version",
"Environment": {
diff --git a/src/__tests__/package-sources/npmjs.test.ts b/src/__tests__/package-sources/npmjs.test.ts
index a5b6b5edd..c6d948b01 100644
--- a/src/__tests__/package-sources/npmjs.test.ts
+++ b/src/__tests__/package-sources/npmjs.test.ts
@@ -110,6 +110,100 @@ test('the known versions migration can be turned off', () => {
).toHaveLength(0);
});
+test('retries laggy packuments through a queue that moves give-ups to a dead-letter queue', () => {
+ const template = synth(new NpmJs());
+
+ const dlqs = template.findResources('AWS::SQS::Queue', {
+ Properties: { MessageRetentionPeriod: 14 * 24 * 60 * 60 },
+ });
+ template.hasResourceProperties('AWS::SQS::Queue', {
+ MessageRetentionPeriod: 14 * 24 * 60 * 60,
+ VisibilityTimeout: 30 * 60,
+ RedrivePolicy: {
+ deadLetterTargetArn: {
+ 'Fn::GetAtt': [Match.stringLikeRegexp('PackumentProcessorDLQ'), 'Arn'],
+ },
+ maxReceiveCount: 10,
+ },
+ });
+ expect(
+ Object.keys(dlqs).some((id) => id.includes('PackumentProcessorDLQ'))
+ ).toBe(true);
+
+ const processors = template.findResources('AWS::Lambda::Function', {
+ Properties: {
+ Description: Match.stringLikeRegexp(
+ 'Processes packages from the npm registry'
+ ),
+ },
+ });
+ expect(Object.keys(processors)).toHaveLength(1);
+ template.hasResourceProperties('AWS::Lambda::EventSourceMapping', {
+ FunctionName: { Ref: Object.keys(processors)[0] },
+ BatchSize: 10,
+ FunctionResponseTypes: ['ReportBatchItemFailures'],
+ ScalingConfig: { MaximumConcurrency: 2 },
+ });
+
+ // The same function reads the dead-letter queue, to log give-ups.
+ const mappings = Object.values(
+ template.findResources('AWS::Lambda::EventSourceMapping', {
+ Properties: { FunctionName: { Ref: Object.keys(processors)[0] } },
+ })
+ ).map((mapping: any) => JSON.stringify(mapping.Properties.EventSourceArn));
+ expect(mappings).toHaveLength(2);
+ expect(mappings.some((arn) => arn.includes('PackumentProcessorDLQ'))).toBe(
+ true
+ );
+
+ // There is no alarm on the dead-letter queue (it is only graphed), and no
+ // alarm on the give-up metric.
+ const alarmNames = Object.values(
+ template.findResources('AWS::CloudWatch::Alarm')
+ ).map((alarm: any) => String(alarm.Properties.AlarmName));
+ expect(
+ alarmNames.some(
+ (name) =>
+ name.endsWith('DLQNotEmpty') && name.includes('PackumentProcessor')
+ )
+ ).toBe(false);
+ expect(
+ alarmNames.some((name) => name.endsWith('LaggyPackumentGiveUps'))
+ ).toBe(false);
+
+ // A burst of missing packuments within 3 hours raises an alarm.
+ template.hasResourceProperties('AWS::CloudWatch::Alarm', {
+ AlarmName: 'Test/ConstructHub/Sources/NpmJs/Follower/MissingPackumentsHigh',
+ MetricName: 'MissingPackuments',
+ Period: 3 * 60 * 60,
+ Statistic: 'Sum',
+ Threshold: 1_500,
+ });
+ // So does a high laggy packument lag.
+ template.hasResourceProperties('AWS::CloudWatch::Alarm', {
+ AlarmName:
+ 'Test/ConstructHub/Sources/NpmJs/PackumentProcessor/LaggyPackumentLagHigh',
+ MetricName: 'LaggyPackumentLag',
+ Period: 3 * 60 * 60,
+ ExtendedStatistic: 'p90',
+ Threshold: 6 * 60 * 60 * 1_000,
+ });
+
+ // Processor failures raise a low severity alarm too.
+ template.hasResourceProperties('AWS::CloudWatch::Alarm', {
+ AlarmName: 'Test/ConstructHub/Sources/NpmJs/PackumentProcessor/Failures',
+ MetricName: 'Errors',
+ });
+
+ // The follower sends laggy packuments to the queue.
+ template.hasResourceProperties('AWS::Lambda::Function', {
+ Description: Match.stringLikeRegexp('Periodically query npmjs.com'),
+ Environment: {
+ Variables: Match.objectLike({ PACKUMENT_QUEUE_URL: Match.anyValue() }),
+ },
+ });
+});
+
test('the billing mode of the known versions table is configurable', () => {
const template = synth(
new NpmJs({
diff --git a/src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts b/src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts
index cd4698f2a..d3d250522 100644
--- a/src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts
+++ b/src/__tests__/package-sources/npmjs/follower-state.lambda-shared.test.ts
@@ -14,7 +14,6 @@ describe('text format', () => {
state.addCheckpoint(90, T0);
state.addCheckpoint(300, T0 + HOUR);
state.addSeqs([100, 250]);
- state.recordLaggyPackument('@scope/laggy-package', 5, 123, T0);
state.completeDeepScan(T0 + HOUR);
expect(state.toText()).toBe(
@@ -23,7 +22,6 @@ describe('text format', () => {
`w ${T0 + HOUR}`,
`c ${T0} 90`,
`c ${T0 + HOUR} 300`,
- `l ${T0} 5 123 @scope/laggy-package`,
's 100',
's 250',
'', // trailing newline
@@ -35,7 +33,6 @@ describe('text format', () => {
const state = new FollowerState();
state.addCheckpoint(90, T0);
state.addSeqs([100, 105, 250]);
- state.recordLaggyPackument('laggy-package', 7, undefined, T0);
state.completeDeepScan(T0);
const restored = FollowerState.fromText(state.toText());
@@ -47,16 +44,28 @@ describe('text format', () => {
expect(restored.receiptCount).toBe(3);
expect(restored.checkpointCount).toBe(1);
expect(restored.lastDeepScanAt).toBe(T0);
- expect(restored.laggyPackuments()).toEqual([
- { name: 'laggy-package', expectedRev: 7, seq: undefined, firstSeen: T0 },
- ]);
+ });
+
+ test('ignores the laggy packuments recorded by earlier versions', () => {
+ const text = [
+ '#chfollower/1',
+ `w ${T0}`,
+ `c ${T0} 90`,
+ `l ${T0} 5 123 @scope/laggy-package`,
+ 's 100',
+ '',
+ ].join('\n');
+
+ const restored = FollowerState.fromText(text);
+
+ expect(restored.has(100)).toBe(true);
+ expect(restored.toText()).not.toContain('laggy-package');
});
test('a fresh state round-trips', () => {
const restored = FollowerState.fromText(new FollowerState().toText());
expect(restored.receiptCount).toBe(0);
expect(restored.checkpointCount).toBe(0);
- expect(restored.laggyPackumentCount).toBe(0);
expect(restored.lastDeepScanAt).toBeUndefined();
});
@@ -64,7 +73,6 @@ describe('text format', () => {
const state = new FollowerState();
state.addCheckpoint(90, T0);
state.addSeqs([100, 250, 251]);
- state.recordLaggyPackument('laggy-package', 7, 200, T0);
const once = state.toText();
const twice = FollowerState.fromText(once).toText();
@@ -171,18 +179,16 @@ describe('checkpoints and receipts', () => {
expect(state.has(500)).toBe(true);
});
- test('resetPosition clears receipts and checkpoints but keeps laggy packuments', () => {
+ test('resetPosition clears receipts and checkpoints', () => {
const state = new FollowerState();
state.addCheckpoint(100, T0);
state.addSeqs([150]);
- state.recordLaggyPackument('laggy-package', 5, 120, T0);
state.resetPosition();
expect(state.checkpointCount).toBe(0);
expect(state.receiptCount).toBe(0);
expect(state.newestCheckpointSeq()).toBeUndefined();
- expect(state.laggyPackumentCount).toBe(1);
});
});
@@ -198,30 +204,3 @@ describe('deep scans', () => {
expect(state.deepScanDue(T0 + DEEP_SCAN_INTERVAL_MS)).toBe(true);
});
});
-
-describe('laggy packuments', () => {
- test('keeps the earliest firstSeen and the highest expected revision', () => {
- const state = new FollowerState();
- state.recordLaggyPackument('laggy-package', 5, 100, T0);
- state.recordLaggyPackument('laggy-package', 7, 200, T0 + HOUR);
- state.recordLaggyPackument('laggy-package', 6, 300, T0 + 2 * HOUR);
-
- expect(state.laggyPackuments()).toEqual([
- { name: 'laggy-package', expectedRev: 7, seq: 100, firstSeen: T0 },
- ]);
- });
-
- test('lists expectations oldest first and removes them', () => {
- const state = new FollowerState();
- state.recordLaggyPackument('second', 2, undefined, T0 + HOUR);
- state.recordLaggyPackument('first', 1, undefined, T0);
-
- expect(state.laggyPackuments().map((l) => l.name)).toEqual([
- 'first',
- 'second',
- ]);
-
- state.removeLaggyPackument('first');
- expect(state.laggyPackumentCount).toBe(1);
- });
-});
diff --git a/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts
index e93482311..7892b6068 100644
--- a/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts
+++ b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts
@@ -10,6 +10,11 @@ import {
PutObjectCommand,
S3Client,
} from '@aws-sdk/client-s3';
+import {
+ GetQueueAttributesCommand,
+ SendMessageBatchCommand,
+ SQSClient,
+} from '@aws-sdk/client-sqs';
import type { metricScope, MetricsLogger } from 'aws-embedded-metrics';
import { Unit } from 'aws-embedded-metrics';
import type { Context, ScheduledEvent } from 'aws-lambda';
@@ -23,9 +28,11 @@ import {
import { EnvironmentVariables as LicenseListEnv } from '../../../backend/license-list/constants';
import {
ENV_KNOWN_VERSIONS_TABLE_NAME,
+ ENV_PACKUMENT_QUEUE_URL,
FOLLOWER_STATE_FILE_NAME,
MARKER_FILE_NAME,
MetricName,
+ PACKUMENT_INITIAL_DELAY_SECONDS,
} from '../../../package-sources/npmjs/constants.lambda-shared';
import { FollowerState } from '../../../package-sources/npmjs/follower-state.lambda-shared';
import { handler } from '../../../package-sources/npmjs/npm-js-follower.lambda';
@@ -59,6 +66,8 @@ const REGISTRY = 'https://registry.npmjs.org';
const MOCK_BUCKET = 'mock-staging-bucket';
const MOCK_STAGING_FUNCTION = 'mock-staging-function';
const MOCK_KNOWN_VERSIONS_TABLE = 'mock-known-versions-table';
+const MOCK_PACKUMENT_QUEUE_URL =
+ 'https://sqs.us-east-1.amazonaws.com/123456789012/mock-packument-queue';
const MOCK_DENY_LIST_BUCKET = 'deny-list-bucket-name';
const MOCK_DENY_LIST_OBJECT = 'my-deny-list.json';
const MOCK_LICENSE_LIST_BUCKET = 'license-list-bucket-name';
@@ -69,6 +78,7 @@ const HOUR = 60 * 60 * 1_000;
const mockS3 = mockClient(S3Client);
const mockLambda = mockClient(LambdaClient);
const mockDynamoDB = mockClient(DynamoDBClient);
+const mockSQS = mockClient(SQSClient);
const context: Context = {
awsRequestId: 'mock-request-id',
@@ -83,11 +93,13 @@ beforeEach(() => {
mockS3.reset();
mockLambda.reset();
mockDynamoDB.reset();
+ mockSQS.reset();
mockPutMetric.mockClear();
process.env.BUCKET_NAME = MOCK_BUCKET;
process.env.FUNCTION_NAME = MOCK_STAGING_FUNCTION;
process.env[ENV_KNOWN_VERSIONS_TABLE_NAME] = MOCK_KNOWN_VERSIONS_TABLE;
+ process.env[ENV_PACKUMENT_QUEUE_URL] = MOCK_PACKUMENT_QUEUE_URL;
process.env[ENV_DENY_LIST_BUCKET_NAME] = MOCK_DENY_LIST_BUCKET;
process.env[ENV_DENY_LIST_OBJECT_KEY] = MOCK_DENY_LIST_OBJECT;
process.env[LicenseListEnv.BUCKET_NAME] = MOCK_LICENSE_LIST_BUCKET;
@@ -116,6 +128,16 @@ beforeEach(() => {
mockDynamoDB.on(QueryCommand).resolves({ Items: [] });
mockDynamoDB.on(BatchWriteItemCommand).resolves({ UnprocessedItems: {} });
+ // Empty packument queue
+ mockSQS.on(SendMessageBatchCommand).resolves({ Successful: [], Failed: [] });
+ mockSQS.on(GetQueueAttributesCommand).resolves({
+ Attributes: {
+ ApproximateNumberOfMessages: '0',
+ ApproximateNumberOfMessagesNotVisible: '0',
+ ApproximateNumberOfMessagesDelayed: '0',
+ },
+ });
+
mockS3.on(PutObjectCommand).resolves({});
mockLambda.on(InvokeCommand).resolves({ StatusCode: 202 });
});
@@ -124,6 +146,7 @@ afterEach(() => {
delete process.env.BUCKET_NAME;
delete process.env.FUNCTION_NAME;
delete process.env[ENV_KNOWN_VERSIONS_TABLE_NAME];
+ delete process.env[ENV_PACKUMENT_QUEUE_URL];
delete process.env[ENV_DENY_LIST_BUCKET_NAME];
delete process.env[ENV_DENY_LIST_OBJECT_KEY];
delete process.env[LicenseListEnv.BUCKET_NAME];
@@ -328,7 +351,17 @@ test('discovers and stages late-inserted changes', async () => {
expect(saved.newestCheckpointSeq()).toBe(1_000);
});
-test('laggy packument: processes served versions and records the expectation', async () => {
+/** The packument requests sent to the packument queue. */
+function queuedRequests(): any[] {
+ return mockSQS.commandCalls(SendMessageBatchCommand).flatMap((call) =>
+ call.args[0].input.Entries!.map((entry) => ({
+ ...JSON.parse(entry.MessageBody!),
+ delaySeconds: entry.DelaySeconds,
+ }))
+ );
+}
+
+test('laggy packument: processes served versions and sends it to the packument queue', async () => {
givenHead(2_010);
givenNoStateFile();
givenLegacyMarker(2_000);
@@ -351,101 +384,164 @@ test('laggy packument: processes served versions and records the expectation', a
nock(REGISTRY)
.get('/laggy-package')
.reply(200, packageDoc('laggy-package', '3-ffffff', '1.0.0'));
+ mockSQS.on(GetQueueAttributesCommand).resolves({
+ Attributes: {
+ ApproximateNumberOfMessages: '0',
+ ApproximateNumberOfMessagesNotVisible: '0',
+ ApproximateNumberOfMessagesDelayed: '1',
+ },
+ });
+ const before = Date.now();
await handler(event, context);
// The versions the registry did serve are processed anyway.
expect(stagedPackages()).toEqual([
{ name: 'laggy-package', version: '1.0.0' },
]);
- // ... and the expectation of rev 5 is recorded for later re-checks. The
- // gauge is emitted at the end of the run and includes it.
+ // ... and a request to check for rev 5 is queued, delayed.
+ const [request] = queuedRequests();
+ expect(request).toMatchObject({
+ name: 'laggy-package',
+ expectedRev: 5,
+ seq: 2_005,
+ reason: 'laggy',
+ delaySeconds: PACKUMENT_INITIAL_DELAY_SECONDS,
+ });
+ expect(request.firstSeen).toBeGreaterThanOrEqual(before);
+ expect(mockSQS).toHaveReceivedCommandWith(SendMessageBatchCommand, {
+ QueueUrl: MOCK_PACKUMENT_QUEUE_URL,
+ });
+ // The gauge reports the size of the packument queue.
expect(mockPutMetric).toHaveBeenCalledWith(
MetricName.LAGGY_PACKUMENTS,
1,
Unit.Count
);
- const state = savedState();
- expect(state.laggyPackuments()).toEqual([
- expect.objectContaining({
- name: 'laggy-package',
- expectedRev: 5,
- seq: 2_005,
- }),
- ]);
+ // The receipt is recorded.
+ expect(savedState().has(2_005)).toBe(true);
});
-test('laggy packument: recovered once the registry catches up', async () => {
- const now = Date.now();
- givenHead(3_000);
+test('laggy packument: a failure to queue it leaves the entry without a receipt', async () => {
+ givenHead(2_010);
+ givenNoStateFile();
+ givenLegacyMarker(2_000);
+ nock(REPLICA)
+ .get('/registry/_changes')
+ .query({ limit: '10000', since: '2000' })
+ .reply(200, {
+ results: [
+ {
+ seq: 2_005,
+ id: 'laggy-package',
+ changes: [{ rev: '5-eeeeee' }],
+ deleted: false,
+ },
+ ],
+ last_seq: 2_005,
+ });
+ nock(REGISTRY)
+ .get('/laggy-package')
+ .reply(200, packageDoc('laggy-package', '3-ffffff', '1.0.0'));
+ mockSQS.on(SendMessageBatchCommand).rejects(new Error('network'));
- const state = new FollowerState();
- state.addCheckpoint(3_000, now - 5 * 60_000);
- state.completeDeepScan(now - HOUR);
- state.recordLaggyPackument('recovered-package', 4, 1_500, now - HOUR);
- givenStateFile(state);
+ await expect(handler(event, context)).rejects.toThrow(/network/);
- // The registry has now caught up (rev 4 >= expected 4), serving a version
- // we don't know yet.
- nock(REGISTRY)
- .get('/recovered-package')
- .reply(200, packageDoc('recovered-package', '4-gggggg', '2.0.0'));
- // The scan itself is empty.
+ // The next run processes the entry again.
+ expect(savedState().has(2_005)).toBe(false);
+});
+
+test('missing packument: a change entry the registry has no packument for yet is sent to the packument queue', async () => {
+ givenHead(2_010);
+ givenNoStateFile();
+ givenLegacyMarker(2_000);
nock(REPLICA)
.get('/registry/_changes')
- .query({ limit: '10000', since: '3000' })
- .reply(200, { results: [], last_seq: 3_000 });
+ .query({ limit: '10000', since: '2000' })
+ .reply(200, {
+ results: [
+ {
+ seq: 2_005,
+ id: 'brand-new-package',
+ changes: [{ rev: '1-aaaaaa' }],
+ deleted: false,
+ },
+ ],
+ last_seq: 2_005,
+ });
+ nock(REGISTRY).get('/brand-new-package').reply(404, 'not found');
await handler(event, context);
- expect(stagedPackages()).toEqual([
- { name: 'recovered-package', version: '2.0.0' },
+ expect(stagedPackages()).toEqual([]);
+ expect(queuedRequests()).toEqual([
+ expect.objectContaining({
+ name: 'brand-new-package',
+ expectedRev: 1,
+ seq: 2_005,
+ reason: 'missing',
+ }),
]);
expect(mockPutMetric).toHaveBeenCalledWith(
- MetricName.LAGGY_PACKUMENTS_RECOVERED,
+ MetricName.MISSING_PACKUMENTS,
1,
Unit.Count
);
- expect(mockPutMetric).toHaveBeenCalledWith(
- MetricName.LAGGY_PACKUMENTS,
- 0,
- Unit.Count
- );
- expect(savedState().laggyPackumentCount).toBe(0);
+ expect(savedState().has(2_005)).toBe(true);
});
-test('laggy packument: gives up after the maximum age, still processing what is served', async () => {
- const now = Date.now();
- givenHead(3_000);
+test('missing packument: an entry that announces no revision expects any packument', async () => {
+ givenHead(2_010);
+ givenNoStateFile();
+ givenLegacyMarker(2_000);
+ nock(REPLICA)
+ .get('/registry/_changes')
+ .query({ limit: '10000', since: '2000' })
+ .reply(200, {
+ results: [
+ { seq: 2_005, id: 'no-rev-package', changes: [], deleted: false },
+ ],
+ last_seq: 2_005,
+ });
+ nock(REGISTRY).get('/no-rev-package').reply(404, 'not found');
- const state = new FollowerState();
- state.addCheckpoint(3_000, now - 5 * 60_000);
- state.completeDeepScan(now - HOUR);
- // First seen 25 hours ago: beyond the give-up age.
- state.recordLaggyPackument('stuck-package', 9, 1_600, now - 25 * HOUR);
- givenStateFile(state);
+ await handler(event, context);
- // The registry still serves rev 7 < expected 9 - but with a version we
- // don't know, which is processed regardless.
- nock(REGISTRY)
- .get('/stuck-package')
- .reply(200, packageDoc('stuck-package', '7-hhhhhh', '3.0.0'));
+ expect(queuedRequests()).toEqual([
+ expect.objectContaining({ name: 'no-rev-package', expectedRev: 0 }),
+ ]);
+});
+
+test('ignores change entries for deleted packages, as if they were not in the feed', async () => {
+ givenHead(2_010);
+ givenNoStateFile();
+ givenLegacyMarker(2_000);
nock(REPLICA)
.get('/registry/_changes')
- .query({ limit: '10000', since: '3000' })
- .reply(200, { results: [], last_seq: 3_000 });
+ .query({ limit: '10000', since: '2000' })
+ .reply(200, {
+ results: [
+ {
+ seq: 2_005,
+ id: 'deleted-package',
+ changes: [{ rev: '3-aaaaaa' }],
+ deleted: true,
+ },
+ ],
+ last_seq: 2_005,
+ });
await handler(event, context);
- expect(stagedPackages()).toEqual([
- { name: 'stuck-package', version: '3.0.0' },
- ]);
+ // No packument fetch (nock would fail on an unexpected request), nothing
+ // queued, nothing counted, and no receipt.
+ expect(queuedRequests()).toEqual([]);
expect(mockPutMetric).toHaveBeenCalledWith(
- MetricName.LAGGY_PACKUMENT_GIVE_UPS,
- 1,
+ MetricName.CHANGE_COUNT,
+ 0,
Unit.Count
);
- expect(savedState().laggyPackumentCount).toBe(0);
+ expect(savedState().has(2_005)).toBe(false);
});
test('seeds at the beginning of the feed on first deployment (automatic backfill)', async () => {
diff --git a/src/__tests__/package-sources/npmjs/packument-processor.lambda.test.ts b/src/__tests__/package-sources/npmjs/packument-processor.lambda.test.ts
new file mode 100644
index 000000000..4ec778c11
--- /dev/null
+++ b/src/__tests__/package-sources/npmjs/packument-processor.lambda.test.ts
@@ -0,0 +1,498 @@
+import {
+ BatchWriteItemCommand,
+ DynamoDBClient,
+ QueryCommand,
+} from '@aws-sdk/client-dynamodb';
+import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda';
+import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { ChangeMessageVisibilityCommand, SQSClient } from '@aws-sdk/client-sqs';
+import type { metricScope, MetricsLogger } from 'aws-embedded-metrics';
+import { Unit } from 'aws-embedded-metrics';
+import type { Context, SQSEvent, SQSRecord } from 'aws-lambda';
+import { mockClient } from 'aws-sdk-client-mock';
+import 'aws-sdk-client-mock-jest';
+import nock from 'nock';
+import {
+ ENV_DENY_LIST_BUCKET_NAME,
+ ENV_DENY_LIST_OBJECT_KEY,
+} from '../../../backend/deny-list/constants';
+import { EnvironmentVariables as LicenseListEnv } from '../../../backend/license-list/constants';
+import {
+ ENV_KNOWN_VERSIONS_TABLE_NAME,
+ ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN,
+ ENV_PACKUMENT_QUEUE_URL,
+ MetricName,
+ PACKUMENT_MAX_RETRY_DELAY_SECONDS,
+ PACKUMENT_MIN_RETRY_DELAY_SECONDS,
+} from '../../../package-sources/npmjs/constants.lambda-shared';
+import type { PackumentRequest } from '../../../package-sources/npmjs/packument-processing.lambda-shared';
+import { handler } from '../../../package-sources/npmjs/packument-processor.lambda';
+import { stringToStream } from '../../streams';
+
+jest.mock('aws-embedded-metrics');
+
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const mockMetricScope = require('aws-embedded-metrics')
+ .metricScope as jest.MockedFunction;
+const mockPutMetric = jest.fn() as jest.MockedFunction<
+ MetricsLogger['putMetric']
+>;
+const mockMetrics: MetricsLogger = {
+ putMetric: mockPutMetric,
+ setDimensions: jest.fn(),
+ setProperty: jest.fn(),
+} as any;
+mockMetricScope.mockImplementation((cb) => {
+ const impl = cb(mockMetrics);
+ return async (...args) => impl(...args);
+});
+
+const REGISTRY = 'https://registry.npmjs.org';
+const QUEUE_URL =
+ 'https://sqs.us-east-1.amazonaws.com/123456789012/mock-packument-queue';
+const QUEUE_ARN = 'arn:aws:sqs:us-east-1:123456789012:mock-packument-queue';
+const DLQ_ARN = 'arn:aws:sqs:us-east-1:123456789012:mock-packument-dlq';
+const MINUTE = 60 * 1_000;
+const HOUR = 60 * MINUTE;
+const TABLE = 'mock-known-versions-table';
+const STAGING_FUNCTION = 'mock-staging-function';
+const DENY_LIST_BUCKET = 'deny-list-bucket-name';
+const DENY_LIST_OBJECT = 'deny-list.json';
+const LICENSE_LIST_BUCKET = 'license-list-bucket-name';
+const LICENSE_LIST_OBJECT = 'license-list.json';
+
+const mockS3 = mockClient(S3Client);
+const mockLambda = mockClient(LambdaClient);
+const mockDynamoDB = mockClient(DynamoDBClient);
+const mockSQS = mockClient(SQSClient);
+
+const context: Context = {} as any;
+
+beforeEach(() => {
+ mockS3.reset();
+ mockLambda.reset();
+ mockDynamoDB.reset();
+ mockSQS.reset();
+ mockPutMetric.mockClear();
+
+ process.env.FUNCTION_NAME = STAGING_FUNCTION;
+ process.env[ENV_PACKUMENT_QUEUE_URL] = QUEUE_URL;
+ process.env[ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN] = DLQ_ARN;
+ // No jitter, unless a test says otherwise.
+ jest.spyOn(Math, 'random').mockReturnValue(0.5);
+ process.env[ENV_KNOWN_VERSIONS_TABLE_NAME] = TABLE;
+ process.env[ENV_DENY_LIST_BUCKET_NAME] = DENY_LIST_BUCKET;
+ process.env[ENV_DENY_LIST_OBJECT_KEY] = DENY_LIST_OBJECT;
+ process.env[LicenseListEnv.BUCKET_NAME] = LICENSE_LIST_BUCKET;
+ process.env[LicenseListEnv.OBJECT_KEY] = LICENSE_LIST_OBJECT;
+
+ mockS3
+ .on(GetObjectCommand, { Bucket: DENY_LIST_BUCKET, Key: DENY_LIST_OBJECT })
+ .resolves({ Body: stringToStream(JSON.stringify({})) });
+ mockS3
+ .on(GetObjectCommand, {
+ Bucket: LICENSE_LIST_BUCKET,
+ Key: LICENSE_LIST_OBJECT,
+ })
+ .resolves({ Body: stringToStream(JSON.stringify(['Apache-2.0'])) });
+ mockDynamoDB.on(QueryCommand).resolves({ Items: [] });
+ mockDynamoDB.on(BatchWriteItemCommand).resolves({ UnprocessedItems: {} });
+ mockLambda.on(InvokeCommand).resolves({ StatusCode: 202 });
+ mockSQS.on(ChangeMessageVisibilityCommand).resolves({});
+});
+
+afterEach(() => {
+ for (const name of [
+ 'FUNCTION_NAME',
+ ENV_PACKUMENT_QUEUE_URL,
+ ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN,
+ ENV_KNOWN_VERSIONS_TABLE_NAME,
+ ENV_DENY_LIST_BUCKET_NAME,
+ ENV_DENY_LIST_OBJECT_KEY,
+ LicenseListEnv.BUCKET_NAME,
+ LicenseListEnv.OBJECT_KEY,
+ ]) {
+ delete process.env[name];
+ }
+ nock.cleanAll();
+ jest.restoreAllMocks();
+});
+
+function record(
+ messageId: string,
+ request: Partial | string,
+ { source = QUEUE_ARN, age = 0 }: { source?: string; age?: number } = {}
+): SQSRecord {
+ return {
+ messageId,
+ receiptHandle: `receipt-${messageId}`,
+ eventSourceARN: source,
+ body:
+ typeof request === 'string'
+ ? request
+ : JSON.stringify({
+ seq: 1_000,
+ firstSeen: Date.now() - age,
+ ...request,
+ }),
+ attributes: { ApproximateReceiveCount: '1' },
+ } as any;
+}
+
+function packageDoc(name: string, rev: string, ...versions: string[]) {
+ const time: Record = {
+ created: '2023-01-01T00:00:00.000Z',
+ modified: '2023-06-01T00:00:00.000Z',
+ };
+ const versionInfos: Record = {};
+ for (const [i, version] of versions.entries()) {
+ time[version] = `2023-06-01T00:0${i}:00.000Z`;
+ versionInfos[version] = {
+ dependencies: { constructs: '^10.0.0' },
+ dist: {
+ shasum: 'mock-shasum',
+ tarball: `${REGISTRY}/${name}/-/${name}-${version}.tgz`,
+ },
+ jsii: {},
+ keywords: ['cdk'],
+ license: 'Apache-2.0',
+ name,
+ version,
+ };
+ }
+ return {
+ _id: name,
+ _rev: rev,
+ 'dist-tags': { latest: versions[versions.length - 1] },
+ name,
+ time,
+ versions: versionInfos,
+ };
+}
+
+function stagedPackages(): Array<{ name: string; version: string }> {
+ return mockLambda.commandCalls(InvokeCommand).map((call) => {
+ const payload = JSON.parse(
+ Buffer.from(call.args[0].input.Payload! as Uint8Array).toString('utf-8')
+ );
+ return { name: payload.name, version: payload.version };
+ });
+}
+
+function event(...records: SQSRecord[]): SQSEvent {
+ return { Records: records };
+}
+
+test('completes a request once the registry serves the expected revision', async () => {
+ nock(REGISTRY)
+ .get('/caught-up')
+ .reply(200, packageDoc('caught-up', '5-aaaaaa', '1.0.0', '1.1.0'));
+
+ const result = await handler(
+ event(record('m1', { name: 'caught-up', expectedRev: 5 })),
+ context
+ );
+
+ expect(result.batchItemFailures).toEqual([]);
+ expect(stagedPackages()).toEqual([
+ { name: 'caught-up', version: '1.0.0' },
+ { name: 'caught-up', version: '1.1.0' },
+ ]);
+ expect(mockDynamoDB).toHaveReceivedCommand(BatchWriteItemCommand);
+ expect(mockSQS).not.toHaveReceivedCommand(ChangeMessageVisibilityCommand);
+ expect(mockPutMetric).toHaveBeenCalledWith(
+ MetricName.LAGGY_PACKUMENTS_RECOVERED,
+ 1,
+ Unit.Count
+ );
+});
+
+test('processes what is served, and waits as long as the request is old before the next attempt', async () => {
+ nock(REGISTRY)
+ .get('/still-laggy')
+ .reply(200, packageDoc('still-laggy', '3-aaaaaa', '1.0.0'));
+
+ const result = await handler(
+ event(
+ record(
+ 'm1',
+ { name: 'still-laggy', expectedRev: 5 },
+ { age: 40 * MINUTE }
+ )
+ ),
+ context
+ );
+
+ expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm1' }]);
+ expect(stagedPackages()).toEqual([{ name: 'still-laggy', version: '1.0.0' }]);
+ expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, {
+ QueueUrl: QUEUE_URL,
+ ReceiptHandle: 'receipt-m1',
+ VisibilityTimeout: 40 * 60,
+ });
+});
+
+test('keeps the wait within the minimum and maximum delay', async () => {
+ nock(REGISTRY)
+ .persist()
+ .get('/still-laggy')
+ .reply(200, packageDoc('still-laggy', '3-aaaaaa', '1.0.0'));
+
+ await handler(
+ event(
+ record('young', { name: 'still-laggy', expectedRev: 5 }, { age: 0 }),
+ record('old', { name: 'still-laggy', expectedRev: 5 }, { age: 20 * HOUR })
+ ),
+ context
+ );
+
+ expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, {
+ ReceiptHandle: 'receipt-young',
+ VisibilityTimeout: PACKUMENT_MIN_RETRY_DELAY_SECONDS,
+ });
+ expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, {
+ ReceiptHandle: 'receipt-old',
+ VisibilityTimeout: PACKUMENT_MAX_RETRY_DELAY_SECONDS,
+ });
+});
+
+test('adds jitter to the wait', async () => {
+ nock(REGISTRY)
+ .persist()
+ .get('/still-laggy')
+ .reply(200, packageDoc('still-laggy', '3-aaaaaa', '1.0.0'));
+ jest.spyOn(Math, 'random').mockReturnValueOnce(0).mockReturnValueOnce(1);
+
+ await handler(
+ event(
+ record('m1', { name: 'still-laggy', expectedRev: 5 }, { age: HOUR }),
+ record('m2', { name: 'still-laggy', expectedRev: 5 }, { age: HOUR })
+ ),
+ context
+ );
+
+ const timeouts = mockSQS
+ .commandCalls(ChangeMessageVisibilityCommand)
+ .map((call) => call.args[0].input.VisibilityTimeout)
+ .sort();
+ expect(timeouts).toEqual([0.8 * 3_600, 1.2 * 3_600]);
+});
+
+test('retries a request while the registry has no packument for the package yet', async () => {
+ nock(REGISTRY).get('/not-yet').reply(404, 'not found');
+
+ const result = await handler(
+ event(record('m1', { name: 'not-yet', expectedRev: 1, reason: 'missing' })),
+ context
+ );
+
+ expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm1' }]);
+ expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, {
+ VisibilityTimeout: PACKUMENT_MIN_RETRY_DELAY_SECONDS,
+ });
+ expect(stagedPackages()).toEqual([]);
+});
+
+test('retries a request that failed with an error, without failing the others', async () => {
+ nock(REGISTRY).get('/broken').reply(403, 'forbidden');
+ nock(REGISTRY)
+ .get('/fine')
+ .reply(200, packageDoc('fine', '5-aaaaaa', '1.0.0'));
+
+ const result = await handler(
+ event(
+ record('m1', { name: 'broken', expectedRev: 5 }),
+ record('m2', { name: 'fine', expectedRev: 5 }),
+ record('m3', 'not json')
+ ),
+ context
+ );
+
+ expect(result.batchItemFailures).toEqual([
+ { itemIdentifier: 'm1' },
+ { itemIdentifier: 'm3' },
+ ]);
+ expect(stagedPackages()).toEqual([{ name: 'fine', version: '1.0.0' }]);
+});
+
+test('does not stage versions that are already known', async () => {
+ nock(REGISTRY)
+ .get('/known')
+ .reply(200, packageDoc('known', '5-aaaaaa', '1.0.0', '1.1.0'));
+ mockDynamoDB
+ .on(QueryCommand)
+ .resolves({ Items: [{ version: { S: '1.0.0' } }] });
+
+ await handler(
+ event(record('m1', { name: 'known', expectedRev: 5 })),
+ context
+ );
+
+ expect(stagedPackages()).toEqual([{ name: 'known', version: '1.1.0' }]);
+});
+
+test('keeps the usual waits when the whole batch fails to set up', async () => {
+ mockS3
+ .on(GetObjectCommand, { Bucket: DENY_LIST_BUCKET, Key: DENY_LIST_OBJECT })
+ .rejects(new Error('s3 is down'));
+
+ await expect(
+ handler(
+ event(
+ record('m1', { name: 'one', expectedRev: 5 }, { age: 20 * MINUTE }),
+ record('m2', { name: 'two', expectedRev: 5 }, { age: 2 * HOUR })
+ ),
+ context
+ )
+ ).rejects.toThrow(/s3 is down/);
+
+ expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, {
+ ReceiptHandle: 'receipt-m1',
+ VisibilityTimeout: 20 * 60,
+ });
+ expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, {
+ ReceiptHandle: 'receipt-m2',
+ VisibilityTimeout: 2 * 3_600,
+ });
+});
+
+test('checks caught-up packages again when their versions could not be recorded', async () => {
+ nock(REGISTRY)
+ .get('/caught-up')
+ .reply(200, packageDoc('caught-up', '5-aaaaaa', '1.0.0'));
+ mockDynamoDB.on(BatchWriteItemCommand).rejects(new Error('dynamodb is down'));
+
+ const result = await handler(
+ event(record('m1', { name: 'caught-up', expectedRev: 5 })),
+ context
+ );
+
+ expect(result.batchItemFailures).toEqual([{ itemIdentifier: 'm1' }]);
+ expect(mockSQS).toHaveReceivedCommandWith(ChangeMessageVisibilityCommand, {
+ VisibilityTimeout: PACKUMENT_MIN_RETRY_DELAY_SECONDS,
+ });
+});
+
+test('reports how long laggy packuments took to catch up', async () => {
+ nock(REGISTRY)
+ .get('/caught-up')
+ .reply(200, packageDoc('caught-up', '5-aaaaaa', '1.0.0'));
+
+ await handler(
+ event(
+ record('m1', { name: 'caught-up', expectedRev: 5 }, { age: 2 * HOUR })
+ ),
+ context
+ );
+
+ const lag = mockPutMetric.mock.calls.find(
+ ([name]) => name === MetricName.LAGGY_PACKUMENT_LAG
+ );
+ expect(lag?.[1]).toBeGreaterThanOrEqual(2 * HOUR);
+ expect(lag?.[2]).toBe(Unit.Milliseconds);
+});
+
+test('counts missing packuments that arrived separately from laggy ones, without a lag', async () => {
+ nock(REGISTRY)
+ .get('/arrived')
+ .reply(200, packageDoc('arrived', '1-aaaaaa', '1.0.0'));
+
+ await handler(
+ event(record('m1', { name: 'arrived', expectedRev: 1, reason: 'missing' })),
+ context
+ );
+
+ expect(mockPutMetric).toHaveBeenCalledWith(
+ MetricName.MISSING_PACKUMENTS_RECOVERED,
+ 1,
+ Unit.Count
+ );
+ expect(mockPutMetric).toHaveBeenCalledWith(
+ MetricName.LAGGY_PACKUMENTS_RECOVERED,
+ 0,
+ Unit.Count
+ );
+ expect(mockPutMetric).not.toHaveBeenCalledWith(
+ MetricName.LAGGY_PACKUMENT_LAG,
+ expect.anything(),
+ expect.anything()
+ );
+});
+
+describe('dead-letter queue', () => {
+ test('logs, counts and deletes requests that were given up on, without checking them', async () => {
+ const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+
+ const result = await handler(
+ event(
+ record(
+ 'm1',
+ { name: 'stuck', expectedRev: 5 },
+ { source: DLQ_ARN, age: 29 * HOUR }
+ ),
+ record(
+ 'm2',
+ { name: 'never-arrived', expectedRev: 1, reason: 'missing' },
+ { source: DLQ_ARN }
+ )
+ ),
+ context
+ );
+
+ // Deleted (no fetch: nock would reject one).
+ expect(result.batchItemFailures).toEqual([]);
+ expect(mockSQS).not.toHaveReceivedCommand(ChangeMessageVisibilityCommand);
+ expect(stagedPackages()).toEqual([]);
+ expect(mockPutMetric).toHaveBeenCalledWith(
+ MetricName.LAGGY_PACKUMENT_GIVE_UPS,
+ 1,
+ Unit.Count
+ );
+ expect(mockPutMetric).toHaveBeenCalledWith(
+ MetricName.MISSING_PACKUMENT_GIVE_UPS,
+ 1,
+ Unit.Count
+ );
+ const lag = mockPutMetric.mock.calls.find(
+ ([name]) => name === MetricName.LAGGY_PACKUMENT_LAG
+ );
+ expect(lag?.[1]).toBeGreaterThanOrEqual(29 * HOUR);
+ expect(warn).toHaveBeenCalledWith(
+ expect.stringContaining('Giving up on laggy packument for stuck')
+ );
+ expect(warn).toHaveBeenCalledWith(
+ expect.stringContaining(
+ 'Giving up on missing packument for never-arrived'
+ )
+ );
+ });
+
+ test('deletes an invalid request it was given up on', async () => {
+ jest.spyOn(console, 'warn').mockImplementation(() => {});
+
+ const result = await handler(
+ event(record('m1', 'not json', { source: DLQ_ARN })),
+ context
+ );
+
+ expect(result.batchItemFailures).toEqual([]);
+ });
+
+ test('handles a batch from either queue on its own', async () => {
+ nock(REGISTRY)
+ .get('/caught-up')
+ .reply(200, packageDoc('caught-up', '5-aaaaaa', '1.0.0'));
+ jest.spyOn(console, 'warn').mockImplementation(() => {});
+
+ const result = await handler(
+ event(
+ record('m1', { name: 'caught-up', expectedRev: 5 }),
+ record('m2', { name: 'stuck', expectedRev: 5 }, { source: DLQ_ARN })
+ ),
+ context
+ );
+
+ expect(result.batchItemFailures).toEqual([]);
+ expect(stagedPackages()).toEqual([{ name: 'caught-up', version: '1.0.0' }]);
+ });
+});
diff --git a/src/package-sources/npmjs.ts b/src/package-sources/npmjs.ts
index 5ae6a3b33..4cec8ec7c 100644
--- a/src/package-sources/npmjs.ts
+++ b/src/package-sources/npmjs.ts
@@ -21,7 +21,7 @@ import {
} from 'aws-cdk-lib/aws-dynamodb';
import { Rule, Schedule } from 'aws-cdk-lib/aws-events';
import { LambdaFunction } from 'aws-cdk-lib/aws-events-targets';
-import { Tracing } from 'aws-cdk-lib/aws-lambda';
+import { IFunction, Tracing } from 'aws-cdk-lib/aws-lambda';
import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources';
import { BlockPublicAccess, IBucket } from 'aws-cdk-lib/aws-s3';
import { Queue, QueueEncryption } from 'aws-cdk-lib/aws-sqs';
@@ -57,6 +57,7 @@ import {
} from './npmjs/constants.lambda-shared';
import { KnownVersionsTableMigration } from './npmjs/known-versions-table-migration';
import { NpmJsFollower } from './npmjs/npm-js-follower';
+import { PackumentQueueProcessor } from './npmjs/packument-queue-processor';
import { StageAndNotify } from './npmjs/stage-and-notify';
import { S3StorageFactory } from '../s3/storage';
import { ReStagePackageVersion } from './npmjs/re-stage-package-version';
@@ -82,6 +83,19 @@ const FOLLOWER_RUN_RATE = Duration.minutes(5);
*/
const NO_CHANGES_ALARM_DURATION = Duration.hours(24);
+/**
+ * Alarm if more change entries than this had no packument in the registry
+ * within 3 hours. In October 2026, a 3 hour window usually saw 170 of them,
+ * and at most about 500. Well above that means a registry outage.
+ */
+const MISSING_PACKUMENTS_ALARM_THRESHOLD = 1_500;
+
+/**
+ * Alarm if 90% of the laggy packuments that caught up (or were given up on)
+ * within 3 hours took longer than this.
+ */
+const LAGGY_PACKUMENT_LAG_ALARM_THRESHOLD = Duration.hours(6);
+
export interface NpmJsProps {
/**
* The bucket to use for staging npm packages.
@@ -283,6 +297,16 @@ export class NpmJs implements IPackageSource {
licenseList.grantRead(follower);
stager.grantInvoke(follower);
+ // Requests to process a package from the registry again, because the
+ // registry served an older revision than the changes feed announced, or
+ // no packument at all yet.
+ const packuments = new PackumentQueueProcessor(
+ scope,
+ 'PackumentProcessor',
+ { knownVersions, stager, denyList, licenseList }
+ );
+ packuments.grantSendRequests(follower);
+
const restager = new ReStagePackageVersion(scope, 'ReStagePackageVersion', {
description: `Manually re-stage a package version`,
environment: {
@@ -300,7 +324,14 @@ export class NpmJs implements IPackageSource {
targets: [new LambdaFunction(follower)],
});
- this.registerAlarms(scope, follower, stager, monitoring, rule);
+ this.registerAlarms(
+ scope,
+ follower,
+ stager,
+ packuments.function,
+ monitoring,
+ rule
+ );
stager.deadLetterQueue &&
overviewDashboard.addDLQMetricToDashboard(
@@ -343,6 +374,11 @@ export class NpmJs implements IPackageSource {
},
{ name: 'Stager', url: lambdaFunctionUrl(stager) },
{ name: 'Stager DLQ', url: sqsQueueUrl(stager.deadLetterQueue!) },
+ { name: 'Packument Queue', url: sqsQueueUrl(packuments.queue) },
+ {
+ name: 'Packument Processor',
+ url: lambdaFunctionUrl(packuments.function),
+ },
{
name: 'Pipeline Trace (Log Analytics)',
url: logAnalyticsUrl(
@@ -453,6 +489,22 @@ export class NpmJs implements IPackageSource {
}),
0
),
+ fillMetric(
+ this.metricMissingPackuments({ label: 'Missing Packuments' }),
+ 0
+ ),
+ fillMetric(
+ this.metricMissingPackumentsRecovered({
+ label: 'Missing Packuments Recovered',
+ }),
+ 0
+ ),
+ fillMetric(
+ this.metricMissingPackumentGiveUps({
+ label: 'Missing Packument Give-Ups',
+ }),
+ 0
+ ),
fillMetric(
this.metricMetadataFetchFailures({
label: 'Metadata Fetch Failures',
@@ -474,6 +526,38 @@ export class NpmJs implements IPackageSource {
rightYAxis: { label: 'Milliseconds', min: 0, showUnits: false },
period: Duration.minutes(5),
}),
+ new GraphWidget({
+ height: 6,
+ width: 12,
+ title: 'Packument Queue',
+ // SQS reports these every 5 minutes, so no gaps need filling.
+ left: [
+ packuments.queue.metricApproximateNumberOfMessagesVisible({
+ label: 'Waiting',
+ }),
+ packuments.queue.metricApproximateNumberOfMessagesNotVisible({
+ label: 'Waiting for next attempt',
+ }),
+ packuments.queue.metricApproximateNumberOfMessagesDelayed({
+ label: 'Delayed',
+ }),
+ ],
+ leftYAxis: { min: 0 },
+ right: [
+ this.metricLaggyPackumentLag({
+ label: 'Laggy Packument Lag (p90)',
+ }),
+ ],
+ rightAnnotations: [
+ {
+ color: '#ff0000',
+ label: 'Alarm',
+ value: LAGGY_PACKUMENT_LAG_ALARM_THRESHOLD.toMilliseconds(),
+ },
+ ],
+ rightYAxis: { label: 'Milliseconds', min: 0, showUnits: false },
+ period: Duration.minutes(5),
+ }),
],
[
new GraphWidget({
@@ -605,10 +689,9 @@ export class NpmJs implements IPackageSource {
}
/**
- * The number of packages for which the registry packument is still behind
- * the revision announced by the `_changes` feed. The versions served so far
- * have been processed; the follower keeps re-checking for the announced
- * revision.
+ * The number of requests in the packument queue, for laggy and missing
+ * packuments. A package that lagged in several change entries has several
+ * requests.
*/
public metricLaggyPackuments(opts?: MetricOptions): Metric {
return new Metric({
@@ -635,7 +718,7 @@ export class NpmJs implements IPackageSource {
}
/**
- * The number of laggy packuments the follower gave up on: the registry
+ * The number of laggy packuments given up on: the registry
* never served the revision announced by the `_changes` feed within the
* maximum retry age. A version announced by the feed may be missing until
* the affected package publishes again.
@@ -650,6 +733,63 @@ export class NpmJs implements IPackageSource {
});
}
+ /**
+ * How long laggy packuments took to catch up (or until they were given up
+ * on), measured from when the follower first saw them.
+ */
+ public metricLaggyPackumentLag(opts?: MetricOptions): Metric {
+ return new Metric({
+ period: Duration.minutes(5),
+ statistic: Stats.p(90),
+ ...opts,
+ metricName: MetricName.LAGGY_PACKUMENT_LAG,
+ namespace: METRICS_NAMESPACE,
+ });
+ }
+
+ /**
+ * The number of change entries for which the registry had no packument at
+ * all (HTTP 404). Usually a new package that has not replicated to the
+ * registry yet; the packument processor keeps re-checking for it.
+ */
+ public metricMissingPackuments(opts?: MetricOptions): Metric {
+ return new Metric({
+ period: Duration.minutes(5),
+ statistic: Stats.SUM,
+ ...opts,
+ metricName: MetricName.MISSING_PACKUMENTS,
+ namespace: METRICS_NAMESPACE,
+ });
+ }
+
+ /**
+ * The number of missing packuments the registry has since served.
+ */
+ public metricMissingPackumentsRecovered(opts?: MetricOptions): Metric {
+ return new Metric({
+ period: Duration.minutes(5),
+ statistic: Stats.SUM,
+ ...opts,
+ metricName: MetricName.MISSING_PACKUMENTS_RECOVERED,
+ namespace: METRICS_NAMESPACE,
+ });
+ }
+
+ /**
+ * The number of missing packuments given up on: the registry never served a
+ * packument for the package within the maximum retry age. Usually an old
+ * change entry for a package that no longer exists.
+ */
+ public metricMissingPackumentGiveUps(opts?: MetricOptions): Metric {
+ return new Metric({
+ period: Duration.minutes(5),
+ statistic: Stats.SUM,
+ ...opts,
+ metricName: MetricName.MISSING_PACKUMENT_GIVE_UPS,
+ namespace: METRICS_NAMESPACE,
+ });
+ }
+
public metricNpmJsChangeAge(opts?: MetricOptions): Metric {
return new Metric({
period: Duration.minutes(1),
@@ -731,6 +871,7 @@ export class NpmJs implements IPackageSource {
scope: Construct,
follower: NpmJsFollower,
stager: StageAndNotify,
+ packumentProcessor: IFunction,
monitoring: IMonitoring,
schedule: Rule
) {
@@ -848,27 +989,72 @@ export class NpmJs implements IPackageSource {
lateChangeLagAlarm
);
- const laggyPackumentGiveUpsAlarm = this.metricLaggyPackumentGiveUps({
- period: Duration.hours(1),
- }).createAlarm(scope, 'NpmJs/Follower/LaggyPackumentGiveUps', {
- alarmName: `${scope.node.path}/NpmJs/Follower/LaggyPackumentGiveUps`,
+ const missingPackumentsAlarm = this.metricMissingPackuments({
+ period: Duration.hours(3),
+ }).createAlarm(scope, 'NpmJs/Follower/MissingPackumentsHigh', {
+ alarmName: `${scope.node.path}/NpmJs/Follower/MissingPackumentsHigh`,
alarmDescription: [
- 'The npm registry never served the packument revision announced by the CouchDB changes feed',
- 'for one or more packages, even after repeated re-checks. All versions the registry did serve',
- 'have been processed, but a version announced by the feed may be missing until the affected',
- 'package publishes again. The follower logs identify the affected packages, which can be',
- 'manually ingested using the ReStagePackageVersion function.',
+ 'The npm registry had no packument for an unusually high number of packages announced by',
+ 'the CouchDB changes feed. This usually means the registry has an outage or is far behind',
+ 'the feed. The packument processor keeps re-checking these packages.',
'',
`Runbook: ${RUNBOOK_URL}`,
].join('\n'),
- comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+ comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD,
evaluationPeriods: 1,
- threshold: 1,
+ threshold: MISSING_PACKUMENTS_ALARM_THRESHOLD,
+ treatMissingData: TreatMissingData.NOT_BREACHING,
+ });
+ monitoring.addLowSeverityAlarm(
+ 'NpmJs/Follower Missing Packuments High',
+ missingPackumentsAlarm
+ );
+
+ const laggyPackumentLagAlarm = this.metricLaggyPackumentLag({
+ period: Duration.hours(3),
+ statistic: Stats.p(90),
+ }).createAlarm(scope, 'NpmJs/PackumentProcessor/LaggyPackumentLagHigh', {
+ alarmName: `${scope.node.path}/NpmJs/PackumentProcessor/LaggyPackumentLagHigh`,
+ alarmDescription: [
+ 'Laggy packuments take unusually long to catch up: the npm registry serves older revisions',
+ 'than the CouchDB changes feed announced, for hours. This usually means the registry has an',
+ 'outage or is far behind the feed. The packument processor keeps re-checking these packages.',
+ '',
+ `Runbook: ${RUNBOOK_URL}`,
+ ].join('\n'),
+ comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD,
+ evaluationPeriods: 1,
+ threshold: LAGGY_PACKUMENT_LAG_ALARM_THRESHOLD.toMilliseconds(),
treatMissingData: TreatMissingData.NOT_BREACHING,
});
monitoring.addLowSeverityAlarm(
- 'NpmJs/Follower gave up on laggy packuments',
- laggyPackumentGiveUpsAlarm
+ 'NpmJs/PackumentProcessor Laggy Packument Lag High',
+ laggyPackumentLagAlarm
+ );
+
+ const packumentProcessorFailuresAlarm = packumentProcessor
+ .metricErrors({ period: Duration.minutes(15) })
+ .createAlarm(scope, 'NpmJs/PackumentProcessor/Failures', {
+ alarmName: `${scope.node.path}/NpmJs/PackumentProcessor/Failures`,
+ alarmDescription: [
+ 'The packument processor function is failing (errors or time-outs). Requests in the',
+ 'packument queue use up their attempts while it fails, and may be given up on early.',
+ '',
+ `Link to the lambda function: ${lambdaFunctionUrl(
+ packumentProcessor
+ )}`,
+ '',
+ `Runbook: ${RUNBOOK_URL}`,
+ ].join('\n'),
+ comparisonOperator:
+ ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+ evaluationPeriods: 2,
+ threshold: 1,
+ treatMissingData: TreatMissingData.NOT_BREACHING,
+ });
+ monitoring.addLowSeverityAlarm(
+ 'NpmJs/PackumentProcessor Failures',
+ packumentProcessorFailuresAlarm
);
// Finally - the "not running" alarm depends on the schedule (it won't run until the schedule
diff --git a/src/package-sources/npmjs/constants.lambda-shared.ts b/src/package-sources/npmjs/constants.lambda-shared.ts
index c32054769..7d8972c83 100644
--- a/src/package-sources/npmjs/constants.lambda-shared.ts
+++ b/src/package-sources/npmjs/constants.lambda-shared.ts
@@ -54,13 +54,39 @@ export const enum MetricName {
LAGGY_PACKUMENTS_RECOVERED = 'LaggyPackumentsRecovered',
/**
- * Number of laggy packuments the follower gave up on: the registry never
+ * Number of laggy packuments given up on: the registry never
* served the revision announced by the `_changes` feed within the maximum
* retry age. Any versions the registry did serve have been processed, but a
* version announced by the feed may be missing until the package publishes
* again.
*/
LAGGY_PACKUMENT_GIVE_UPS = 'LaggyPackumentGiveUps',
+
+ /**
+ * How long a laggy packument took to catch up (or until it was given up
+ * on), measured from when the follower first saw it.
+ */
+ LAGGY_PACKUMENT_LAG = 'LaggyPackumentLag',
+
+ /**
+ * Number of change entries for which the registry had no packument at all
+ * (HTTP 404) when the follower processed them. Usually a new package that
+ * has not replicated to the registry yet; they are re-checked like laggy
+ * packuments.
+ */
+ MISSING_PACKUMENTS = 'MissingPackuments',
+
+ /**
+ * Number of missing packuments the registry has since served.
+ */
+ MISSING_PACKUMENTS_RECOVERED = 'MissingPackumentsRecovered',
+
+ /**
+ * Number of missing packuments given up on: the registry never served a
+ * packument for the package within the maximum retry age. Usually an old
+ * change entry for a deleted package.
+ */
+ MISSING_PACKUMENT_GIVE_UPS = 'MissingPackumentGiveUps',
}
export const enum S3KeyPrefix {
@@ -133,8 +159,46 @@ export const DEEP_SCAN_INTERVAL_MS = 24 * 60 * 60 * 1_000; // 24 hours
export const STATE_RETENTION_MS = 24 * 60 * 60 * 1_000; // 24 hours
/**
- * How long the follower keeps re-checking a laggy packument (a package whose
- * registry packument is behind the revision announced by the `_changes` feed)
- * before giving up on the announced revision.
+ * The npm registry replica serving the `_changes` feed.
+ */
+export const NPM_REPLICA_REGISTRY_URL = 'https://replicate.npmjs.com/';
+
+/**
+ * The environment variable holding the URL of the packument queue: requests
+ * to process a package from the registry, retried with back-off until the
+ * registry serves the revision announced by the `_changes` feed.
+ */
+export const ENV_PACKUMENT_QUEUE_URL = 'PACKUMENT_QUEUE_URL';
+
+/**
+ * The environment variable holding the ARN of the packument dead-letter queue.
+ * The packument processor reads both queues, and tells them apart by this.
+ */
+export const ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN =
+ 'PACKUMENT_DEAD_LETTER_QUEUE_ARN';
+
+/**
+ * How long a new packument request waits before it is first processed.
+ */
+export const PACKUMENT_INITIAL_DELAY_SECONDS = 5 * 60; // 5 minutes
+
+/**
+ * The shortest and longest wait before the next attempt. In between, a request
+ * waits as long as it is already old, so waits double with each attempt. SQS
+ * caps a visibility timeout at 12 hours.
+ */
+export const PACKUMENT_MIN_RETRY_DELAY_SECONDS = 5 * 60; // 5 minutes
+export const PACKUMENT_MAX_RETRY_DELAY_SECONDS = 6 * 60 * 60; // 6 hours
+
+/**
+ * Waits are randomly shortened or lengthened by up to this fraction, so that
+ * requests made together do not keep coming back together.
+ */
+export const PACKUMENT_RETRY_JITTER = 0.2;
+
+/**
+ * The number of attempts before a request moves to the dead-letter queue (the
+ * queue's `maxReceiveCount`). With the waits above, that is after about 29
+ * hours.
*/
-export const LAGGY_PACKUMENT_GIVE_UP_MS = 24 * 60 * 60 * 1_000; // 24 hours
+export const PACKUMENT_MAX_ATTEMPTS = 10;
diff --git a/src/package-sources/npmjs/couch-changes.lambda-shared.ts b/src/package-sources/npmjs/couch-changes.lambda-shared.ts
index 243ba742b..235e5d658 100644
--- a/src/package-sources/npmjs/couch-changes.lambda-shared.ts
+++ b/src/package-sources/npmjs/couch-changes.lambda-shared.ts
@@ -109,26 +109,32 @@ export class CouchChanges extends EventEmitter {
/**
* Fetches the registry metadata for each of the provided change entries and
- * attaches it to the entry. Entries for deleted or unreachable packages are
- * dropped.
+ * attaches it to the entry. Entries for deleted packages are dropped.
*
* The registry is a separate system from the changes feed and can serve a
* packument revision older than the revision the feed announced (a "laggy
* packument"). Such entries are returned with `laggy: true`: the caller
* should process the versions that were served, and separately track the
* expectation that the announced revision will eventually appear.
+ *
+ * The registry can also have no packument at all yet (HTTP 404), usually
+ * because a new package has not replicated to it yet. Such entries are
+ * returned in `missing`, and should be tracked the same way.
*/
public async attachAllMetadata(
changes: readonly DatabaseChange[]
): Promise {
const ok = new Array();
+ const missing = new Array();
const failedSeqs = new Array();
await Promise.all(
changes.map(async (change) => {
try {
const outcome = await this.attachMetadata(change);
- if (outcome !== undefined) {
- ok.push(outcome);
+ if (outcome?.attached) {
+ ok.push(outcome.attached);
+ } else if (outcome?.missing) {
+ missing.push(outcome.missing);
}
} catch (error) {
// One unreachable packument must not fail the whole batch. The
@@ -144,12 +150,16 @@ export class CouchChanges extends EventEmitter {
}
})
);
- return { ok, failedSeqs };
+ return { ok, missing, failedSeqs };
}
private async attachMetadata(
change: DatabaseChange
- ): Promise {
+ ): Promise<
+ | { attached: AttachedChange; missing?: undefined }
+ | { attached?: undefined; missing: MissingPackument }
+ | undefined
+ > {
// Filter out deleted packages or null ids
if (change.deleted || !change.id) {
console.log(`Skipping ${change.id}: deleted or null id`);
@@ -168,9 +178,11 @@ export class CouchChanges extends EventEmitter {
} catch (e: any) {
if (e.message?.includes('HTTP 404')) {
console.log(
- `Skipping ${change.id} because of HTTP 404 (Not Found) error`
+ `${change.id}: the registry has no packument for it yet (HTTP 404)`
);
- return undefined;
+ return {
+ missing: { change, announcedRev: getMaxSequentialRevision(change) },
+ };
}
throw e;
}
@@ -191,10 +203,12 @@ export class CouchChanges extends EventEmitter {
change.doc = meta; // add metadata to the change object
return {
- change,
- announcedRev,
- servedRev,
- laggy: servedRev < announcedRev,
+ attached: {
+ change,
+ announcedRev,
+ servedRev,
+ laggy: servedRev < announcedRev,
+ },
};
}
@@ -344,7 +358,9 @@ export function parseSequentialRevision(rev: string): number {
* The highest sequential revision announced by a change entry.
*/
export function getMaxSequentialRevision(change: DatabaseChange): number {
+ // 0 when the entry announces no revision, so any packument satisfies it.
return Math.max(
+ 0,
...change.changes
.map((c) => parseSequentialRevision(c.rev))
.filter((num) => !isNaN(num))
@@ -385,6 +401,11 @@ export interface AttachedMetadata {
*/
readonly ok: AttachedChange[];
+ /**
+ * The change entries for which the registry has no packument (yet).
+ */
+ readonly missing: MissingPackument[];
+
/**
* The sequence numbers of change entries whose metadata fetch failed. No
* receipt must be recorded for these, so that a later scan retries them.
@@ -392,6 +413,22 @@ export interface AttachedMetadata {
readonly failedSeqs: number[];
}
+/**
+ * A change entry for which the registry has no packument (HTTP 404). Usually
+ * a new package that has not replicated to the registry yet; rarely an old
+ * change entry for a package that no longer exists.
+ */
+export interface MissingPackument {
+ /** The change entry. */
+ readonly change: DatabaseChange;
+
+ /**
+ * The highest sequential revision announced by the changes feed for this
+ * entry.
+ */
+ readonly announcedRev: number;
+}
+
/**
* A change entry with its registry metadata attached.
*/
diff --git a/src/package-sources/npmjs/follower-state.lambda-shared.ts b/src/package-sources/npmjs/follower-state.lambda-shared.ts
index fdf9af06d..2ab987a52 100644
--- a/src/package-sources/npmjs/follower-state.lambda-shared.ts
+++ b/src/package-sources/npmjs/follower-state.lambda-shared.ts
@@ -20,35 +20,6 @@ export interface Checkpoint {
readonly seq: number;
}
-/**
- * A package whose registry packument is behind the revision announced by the
- * `_changes` feed. The versions the registry did serve have been processed;
- * this expectation tracks the revision we are still waiting for.
- */
-export interface LaggyPackument {
- /**
- * The package name.
- */
- readonly name: string;
-
- /**
- * The revision (sequential prefix) announced by the changes feed.
- */
- readonly expectedRev: number;
-
- /**
- * The sequence number of the change entry that announced the revision, if
- * known.
- */
- readonly seq?: number;
-
- /**
- * Epoch milliseconds at which the expectation was first recorded. Drives
- * the give-up policy.
- */
- readonly firstSeen: number;
-}
-
/**
* The persisted state of the *NpmJs Follower*, in a newline-delimited
* plain-text format so it can be scanned cheaply (e.g. `grep '^s 131711133$'`)
@@ -59,7 +30,6 @@ export interface LaggyPackument {
* #chfollower/1 <- header (format magic + version)
* w <- deep scan bookkeeping ("-" when unset)
* c <- checkpoint (repeated, ascending by time)
- * l <- laggy packument (repeated; seq is "-" when unknown)
* s <- received sequence number (repeated, ascending)
* ```
*
@@ -73,7 +43,7 @@ const STATE_FORMAT_HEADER = '#chfollower/1';
/**
* Tracks which `_changes` feed entries (by sequence number) the follower has
* already received, rolling time/seq checkpoints mapping times to feed
- * positions, and the set of laggy packuments awaiting a registry catch-up.
+ * positions.
*
* The npm `_changes` feed does not guarantee that entries only ever become
* visible above previously returned `last_seq` values. The receipt set
@@ -107,12 +77,7 @@ export class FollowerState {
});
break;
case 'l':
- state.laggy.set(parts.slice(4).join(' '), {
- name: parts.slice(4).join(' '),
- firstSeen: Number(parts[1]),
- expectedRev: Number(parts[2]),
- seq: parts[3] === '-' ? undefined : Number(parts[3]),
- });
+ // Laggy packument, recorded by earlier versions: no longer used.
break;
case 'w':
state.lastDeepScanAt =
@@ -129,7 +94,6 @@ export class FollowerState {
private checkpoints: Checkpoint[] = [];
private readonly seqs = new Set();
- private readonly laggy = new Map();
// The threshold below which receipts have already been pruned; avoids
// re-iterating the (potentially large) receipt set on every checkpoint.
private prunedBelow = 0;
@@ -228,8 +192,7 @@ export class FollowerState {
/**
* Discards all position state (receipts and checkpoints). Used when the
* feed's sequence space is no longer compatible with the recorded state
- * (e.g. the feed head regressed below our newest checkpoint). Laggy
- * packument expectations are package-keyed and remain valid.
+ * (e.g. the feed head regressed below our newest checkpoint).
*/
public resetPosition(): void {
this.checkpoints = [];
@@ -257,53 +220,6 @@ export class FollowerState {
}
//#endregion
- //#region Laggy packuments
- /**
- * Records (or extends) the expectation that the registry packument for the
- * provided package should reach the provided revision. The earliest
- * `firstSeen` is retained (so the give-up policy measures from the first
- * occurrence), and the highest announced revision wins.
- */
- public recordLaggyPackument(
- name: string,
- expectedRev: number,
- seq: number | undefined,
- now: number = Date.now()
- ): void {
- const existing = this.laggy.get(name);
- this.laggy.set(name, {
- name,
- expectedRev: Math.max(expectedRev, existing?.expectedRev ?? 0),
- seq: existing?.seq ?? seq,
- firstSeen: existing?.firstSeen ?? now,
- });
- }
-
- /**
- * Removes the expectation for the provided package (recovered, gone, or
- * given up).
- */
- public removeLaggyPackument(name: string): void {
- this.laggy.delete(name);
- }
-
- /**
- * All laggy packument expectations, oldest first.
- */
- public laggyPackuments(): LaggyPackument[] {
- return Array.from(this.laggy.values()).sort(
- (a, b) => a.firstSeen - b.firstSeen
- );
- }
-
- /**
- * The number of laggy packument expectations currently tracked.
- */
- public get laggyPackumentCount(): number {
- return this.laggy.size;
- }
- //#endregion
-
public toText(): string {
const lines = new Array();
lines.push(STATE_FORMAT_HEADER);
@@ -311,13 +227,6 @@ export class FollowerState {
for (const { at, seq } of this.checkpoints) {
lines.push(`c ${at} ${seq}`);
}
- for (const laggy of this.laggyPackuments()) {
- lines.push(
- `l ${laggy.firstSeen} ${laggy.expectedRev} ${laggy.seq ?? '-'} ${
- laggy.name
- }`
- );
- }
for (const seq of Array.from(this.seqs).sort((a, b) => a - b)) {
lines.push(`s ${seq}`);
}
diff --git a/src/package-sources/npmjs/npm-js-follower.lambda.ts b/src/package-sources/npmjs/npm-js-follower.lambda.ts
index 7fc296ab1..a3d025078 100644
--- a/src/package-sources/npmjs/npm-js-follower.lambda.ts
+++ b/src/package-sources/npmjs/npm-js-follower.lambda.ts
@@ -1,5 +1,4 @@
import * as console from 'console';
-import { InvokeCommand } from '@aws-sdk/client-lambda';
import {
metricScope,
Configuration,
@@ -17,17 +16,26 @@ import {
FOLLOWER_STATE_FILE_NAME,
SCAN_WINDOW_MS,
STATE_RETENTION_MS,
- LAGGY_PACKUMENT_GIVE_UP_MS,
+ ENV_PACKUMENT_QUEUE_URL,
+ NPM_REPLICA_REGISTRY_URL,
} from './constants.lambda-shared';
import {
CouchChanges,
DatabaseChange,
DatabaseChanges,
- parseSequentialRevision,
} from './couch-changes.lambda-shared';
import { FollowerState } from './follower-state.lambda-shared';
import { KnownVersions } from './known-versions.lambda-shared';
-import { PackageVersion } from './stage-and-notify.lambda';
+import {
+ PackumentProcessingContext,
+ PackumentRequest,
+ PackumentRequestReason,
+ processPackuments,
+} from './packument-processing.lambda-shared';
+import {
+ enqueuePackumentRequests,
+ queuedPackumentRequests,
+} from './packument-queue.lambda-shared';
import {
loadContentFromS3,
MarkerFileSchema,
@@ -35,20 +43,8 @@ import {
} from './staging-bucket.lambda-shared';
import { DenyListClient } from '../../backend/deny-list/client.lambda-shared';
import { LicenseListClient } from '../../backend/license-list/client.lambda-shared';
-import { LAMBDA_CLIENT } from '../../backend/shared/aws.lambda-shared';
import { compressContent } from '../../backend/shared/compress-content.lambda-shared';
import { requireEnv } from '../../backend/shared/env.lambda-shared';
-// eslint-disable-next-line @typescript-eslint/no-require-imports
-const normalizeNPMMetadata = require('normalize-registry-metadata');
-
-const CONSTRUCT_KEYWORDS: ReadonlySet = new Set([
- 'cdk',
- 'aws-cdk',
- 'awscdk',
- 'cdk8s',
- 'cdktf',
-]);
-const NPM_REPLICA_REGISTRY_URL = 'https://replicate.npmjs.com/';
/**
* The release date of `aws-cdk@0.8.0`. Anything earlier than this basically is
@@ -73,17 +69,6 @@ const SCAN_BATCH_SIZE = 10_000;
*/
const SCAN_TIME_BUDGET_MS = 60_000;
-/**
- * Laggy packuments are only re-checked while at least this much time remains
- * in the Lambda execution, so that re-checks cannot starve the scan.
- */
-const LAGGY_RETRY_TIME_BUDGET_MS = 240_000;
-
-/**
- * The maximum number of laggy packuments re-checked per run.
- */
-const MAX_LAGGY_RETRIES_PER_RUN = 50;
-
// Configure embedded metrics format
Configuration.namespace = METRICS_NAMESPACE;
@@ -113,8 +98,9 @@ captureHTTPsGlobal(require('http'));
* version it contains is processed (deny list, license, and known-versions
* checks apply). If the packument is behind the revision announced by the
* feed (a "laggy packument"), the served versions are still processed, and
- * the follower records the expectation and keeps re-checking on subsequent
- * runs until the announced revision appears or the expectation ages out.
+ * a request to check the package again is sent to the packument queue,
+ * where it is retried until the announced revision appears or it is given
+ * up on.
*
* npm registry API docs: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md
*
@@ -124,13 +110,16 @@ export async function handler(event: ScheduledEvent, context: Context) {
console.log(`Event: ${JSON.stringify(event, null, 2)}`);
const stagingBucket = requireEnv('BUCKET_NAME');
- const stagingFunction = requireEnv('FUNCTION_NAME');
+ const queueUrl = requireEnv(ENV_PACKUMENT_QUEUE_URL);
const knownVersions = new KnownVersions(
requireEnv(ENV_KNOWN_VERSIONS_TABLE_NAME)
);
-
- const denyList = await DenyListClient.newClient();
- const licenseList = await LicenseListClient.newClient();
+ const processing: PackumentProcessingContext = {
+ stagingFunction: requireEnv('FUNCTION_NAME'),
+ denyList: await DenyListClient.newClient(),
+ licenseList: await LicenseListClient.newClient(),
+ knownVersions,
+ };
const npm = new CouchChanges(NPM_REPLICA_REGISTRY_URL, 'registry/_changes');
@@ -150,44 +139,20 @@ export async function handler(event: ScheduledEvent, context: Context) {
}
try {
- // Re-check laggy packuments (packages whose registry packument was behind
- // the revision announced by the feed when we first saw them).
- await retryLaggyPackuments(
- context,
- npm,
- state,
- stagingFunction,
- denyList,
- licenseList,
- knownVersions
- );
-
// Scan the trailing window of the feed. Progress is persisted after
// every processed chunk (see PersistState), so even a run that is hard
// killed by the Lambda timeout keeps its completed work.
const persist: PersistState = () =>
persistProgress(context, stagingBucket, state, knownVersions);
- await runScan(
- context,
- npm,
- state,
- head,
- stagingFunction,
- denyList,
- licenseList,
- knownVersions,
- persist
- );
+ await runScan(context, npm, state, head, processing, queueUrl, persist);
- // The laggy packument gauge is emitted at the end of the run, so it
- // reflects the expectations recorded by this run's scan (emitting it during
- // the retry phase would always read the trough: after recoveries, before
- // new expectations are recorded).
+ // Requests waiting in the packument queue, including those this run sent.
+ const laggyPackuments = await queuedPackumentRequests(queueUrl);
await metricScope((metrics) => async () => {
metrics.setDimensions({});
metrics.putMetric(
MetricName.LAGGY_PACKUMENTS,
- state.laggyPackumentCount,
+ laggyPackuments,
Unit.Count
);
})();
@@ -247,10 +212,8 @@ async function runScan(
npm: CouchChanges,
state: FollowerState,
head: number,
- stagingFunction: string,
- denyList: DenyListClient,
- licenseList: LicenseListClient,
- knownVersions: KnownVersions,
+ processing: PackumentProcessingContext,
+ queueUrl: string,
persist: PersistState
): Promise {
const now = Date.now();
@@ -302,10 +265,8 @@ async function runScan(
state,
lateBoundary,
head,
- stagingFunction,
- denyList,
- licenseList,
- knownVersions,
+ processing,
+ queueUrl,
persist
);
staged += outcome.staged;
@@ -375,18 +336,19 @@ async function processBatch(
state: FollowerState,
lateBoundary: number,
head: number,
- stagingFunction: string,
- denyList: DenyListClient,
- licenseList: LicenseListClient,
- knownVersions: KnownVersions,
+ processing: PackumentProcessingContext,
+ queueUrl: string,
persist: PersistState
): Promise<{ staged: number; fullyProcessed: boolean }> {
const startTime = Date.now();
let staged = 0;
let fullyProcessed = true;
try {
+ // Entries for deleted packages are ignored, as if they were not in the
+ // feed: there is nothing to process, and no receipt is recorded.
const fresh = batch.results.filter(
- (change) => change.seq != null && !state.has(Number(change.seq))
+ (change) =>
+ !change.deleted && change.seq != null && !state.has(Number(change.seq))
);
console.log(
`Received ${batch.totalCount} change entr(ies), ${fresh.length} without a receipt`
@@ -424,7 +386,11 @@ async function processBatch(
);
}
- const { ok: attached, failedSeqs } = await npm.attachAllMetadata(chunk);
+ const {
+ ok: attached,
+ missing,
+ failedSeqs,
+ } = await npm.attachAllMetadata(chunk);
metrics.putMetric(
MetricName.METADATA_FETCH_FAILURES,
failedSeqs.length,
@@ -435,7 +401,7 @@ async function processBatch(
let lastModified: Date | undefined;
// Emit npm.js replication lag
for (const { change } of attached) {
- const doc = change.doc as Change['doc'] | undefined;
+ const doc = change.doc as { time?: { modified?: string } } | undefined;
if (doc?.time?.modified) {
const modified = new Date(doc.time.modified);
metrics.putMetric(
@@ -455,46 +421,60 @@ async function processBatch(
`Skipping chunk as the latest modification is ${lastModified}, which is pre-Constructs`
);
} else {
+ chunkStaged = await processPackuments(
+ attached.map((a) => a.change),
+ metrics,
+ processing
+ );
+ staged += chunkStaged;
+
// Laggy packuments: the registry served an older revision than the
- // feed announced. Process the versions we did get, and record the
- // expectation so the announced revision is re-checked on later runs.
- for (const laggy of attached.filter((a) => a.laggy)) {
+ // feed announced (the versions we did get were processed above), or
+ // no packument at all yet. The packument queue checks the package
+ // again until the announced revision appears. This happens before the
+ // receipts are recorded, so a failure here makes the next run process
+ // the chunk again.
+ const laggy = attached.filter((a) => a.laggy);
+ for (const { change, servedRev, announcedRev } of laggy) {
console.log(
- `${laggy.change.id}: registry packument rev ${laggy.servedRev} is behind announced rev ${laggy.announcedRev}, recording laggy packument`
+ `${change.id}: registry packument rev ${servedRev} is behind announced rev ${announcedRev}, sending it to the packument queue`
);
- state.recordLaggyPackument(
- laggy.change.id,
- laggy.announcedRev,
- isNaN(Number(laggy.change.seq))
- ? undefined
- : Number(laggy.change.seq)
+ }
+ for (const { change } of missing) {
+ console.log(
+ `${change.id}: the registry has no packument for it yet, sending it to the packument queue`
);
}
-
- const versionInfos = await getRelevantVersionInfos(
- attached.map((a) => a.change) as unknown as readonly Change[],
- metrics,
- denyList,
- licenseList,
- knownVersions
- );
- console.log(
- `Identified ${versionInfos.length} relevant package version update(s)`
- );
metrics.putMetric(
- MetricName.RELEVANT_PACKAGE_VERSIONS,
- versionInfos.length,
+ MetricName.MISSING_PACKUMENTS,
+ missing.length,
Unit.Count
);
- await stageVersions(versionInfos, stagingFunction, knownVersions);
- chunkStaged = versionInfos.length;
- staged += chunkStaged;
+ const now = Date.now();
+ const request = (
+ change: DatabaseChange,
+ expectedRev: number,
+ reason: PackumentRequestReason
+ ): PackumentRequest => ({
+ name: change.id,
+ expectedRev,
+ seq: isNaN(Number(change.seq)) ? undefined : Number(change.seq),
+ firstSeen: now,
+ reason,
+ });
+ await enqueuePackumentRequests(queueUrl, [
+ ...laggy.map(({ change, announcedRev }) =>
+ request(change, announcedRev, 'laggy')
+ ),
+ ...missing.map(({ change, announcedRev }) =>
+ request(change, announcedRev, 'missing')
+ ),
+ ]);
}
- // Record receipts for every entry of the chunk (including deleted or
- // unpublished packages: they were received and deliberately skipped) -
- // except entries whose metadata fetch failed, so a later scan retries
- // them. Then persist, so a timeout cannot undo this chunk.
+ // Record a receipt for every entry of the chunk, except those whose
+ // metadata fetch failed (a later scan retries them). Then persist, so a
+ // timeout cannot undo this chunk.
const failed = new Set(failedSeqs);
state.addSeqs(
chunk
@@ -520,130 +500,6 @@ async function processBatch(
}
//#endregion
-//#region Laggy packuments
-/**
- * Re-checks laggy packuments: packages whose registry packument was behind
- * the revision announced by the changes feed. Each re-check processes
- * whatever the registry serves now (new versions are staged), and the
- * expectation is cleared when the announced revision appears - or aged out
- * after `LAGGY_PACKUMENT_GIVE_UP_MS`, at which point a version announced by
- * the feed may be missing until the package publishes again.
- *
- * @returns the number of package versions sent for staging.
- */
-async function retryLaggyPackuments(
- context: Context,
- npm: CouchChanges,
- state: FollowerState,
- stagingFunction: string,
- denyList: DenyListClient,
- licenseList: LicenseListClient,
- knownVersions: KnownVersions
-): Promise {
- let staged = 0;
- await metricScope((metrics) => async () => {
- metrics.setDimensions({});
- let recovered = 0;
- let gaveUp = 0;
- const now = Date.now();
- for (const laggy of state
- .laggyPackuments()
- .slice(0, MAX_LAGGY_RETRIES_PER_RUN)) {
- if (context.getRemainingTimeInMillis() < LAGGY_RETRY_TIME_BUDGET_MS) {
- break;
- }
- const doc = await npm.getPackageDoc(laggy.name);
- if (doc == null || doc._rev == null) {
- console.log(
- `Laggy packument for ${laggy.name} is no longer available in the registry, dropping the expectation`
- );
- state.removeLaggyPackument(laggy.name);
- continue;
- }
-
- // Process whatever the registry serves now; knownVersions dedupes.
- const change: DatabaseChange = {
- changes: [{ rev: doc._rev as string }],
- deleted: false,
- id: laggy.name,
- seq: laggy.seq,
- doc,
- };
- const versionInfos = await getRelevantVersionInfos(
- [change] as unknown as readonly Change[],
- metrics,
- denyList,
- licenseList,
- knownVersions
- );
- await stageVersions(versionInfos, stagingFunction, knownVersions);
- staged += versionInfos.length;
-
- if (parseSequentialRevision(doc._rev as string) >= laggy.expectedRev) {
- console.log(
- `Laggy packument for ${laggy.name} caught up (rev ${doc._rev} >= ${laggy.expectedRev})`
- );
- state.removeLaggyPackument(laggy.name);
- recovered += 1;
- } else if (now - laggy.firstSeen > LAGGY_PACKUMENT_GIVE_UP_MS) {
- console.warn(
- `Giving up on laggy packument for ${
- laggy.name
- }: the registry never served rev ${
- laggy.expectedRev
- } (first seen ${new Date(
- laggy.firstSeen
- ).toISOString()}). A version announced by the changes feed may be missing until the package publishes again.`
- );
- state.removeLaggyPackument(laggy.name);
- gaveUp += 1;
- }
- }
- metrics.putMetric(
- MetricName.LAGGY_PACKUMENTS_RECOVERED,
- recovered,
- Unit.Count
- );
- metrics.putMetric(MetricName.LAGGY_PACKUMENT_GIVE_UPS, gaveUp, Unit.Count);
- })();
- return staged;
-}
-//#endregion
-
-/**
- * Sends the provided package version updates to the staging function
- * ("fire-and-forget"), and records them as known versions.
- */
-async function stageVersions(
- versionInfos: readonly UpdatedVersion[],
- stagingFunction: string,
- knownVersions: KnownVersions
-): Promise {
- await Promise.all(
- versionInfos.map(async ({ packageName, infos, modified, seq }) => {
- const invokeArgs: PackageVersion = {
- integrity: infos.dist.shasum,
- modified: modified.toISOString(),
- name: infos.name,
- seq: seq?.toString(),
- tarballUrl: infos.dist.tarball,
- version: infos.version,
- };
- // "Fire-and-forget" invocation here.
- console.log(`Sending ${invokeArgs.tarballUrl} for staging`);
- await LAMBDA_CLIENT.send(
- new InvokeCommand({
- FunctionName: stagingFunction,
- InvocationType: 'Event',
- Payload: Buffer.from(JSON.stringify(invokeArgs)),
- })
- );
- // Record that this is now a "known" version (no need to re-discover)
- knownVersions.add(packageName, infos.version);
- })
- );
-}
-
//#region State
/**
* Loads the follower state from S3. When the state file does not exist (or
@@ -665,7 +521,7 @@ async function loadFollowerState(
try {
const state = FollowerState.fromText(content);
console.log(
- `Loaded follower state: ${state.receiptCount} receipt(s), ${state.checkpointCount} checkpoint(s), ${state.laggyPackumentCount} laggy packument(s)`
+ `Loaded follower state: ${state.receiptCount} receipt(s), ${state.checkpointCount} checkpoint(s)`
);
return state;
} catch (error) {
@@ -728,7 +584,7 @@ async function saveFollowerState(
Buffer.from(state.toText(), 'utf-8')
);
console.log(
- `Updating follower state (${state.receiptCount} receipt(s), ${state.checkpointCount} checkpoint(s), ${state.laggyPackumentCount} laggy packument(s))`
+ `Updating follower state (${state.receiptCount} receipt(s), ${state.checkpointCount} checkpoint(s))`
);
await putObject(context, stagingBucket, FOLLOWER_STATE_FILE_NAME, buffer, {
ContentType: 'text/plain',
@@ -738,283 +594,3 @@ async function saveFollowerState(
}
//#endregion
-
-/**
- * Obtains the `VersionInfo` corresponding to the modified version(s) in the
- * provided `Change` objects, ensures they are relevant (construct libraries),
- * and returns those only.
- *
- * Known versions are only looked up for packages that have at least one
- * construct library version: only those versions are ever recorded, so every
- * other package has no known versions by definition. Changes are processed
- * concurrently; the result keeps their order.
- *
- * @param changes the changes to be processed.
- * @param metrics the metrics logger to use.
- * @param denyList deny list client
- *
- * @returns a list of `VersionInfo` objects
- */
-async function getRelevantVersionInfos(
- changes: readonly Change[],
- metrics: MetricsLogger,
- denyList: DenyListClient,
- licenseList: LicenseListClient,
- knownVersions: KnownVersions
-): Promise {
- const perChange = await Promise.all(
- changes.map((change) => relevantVersionInfosOf(change))
- );
- return perChange.flat();
-
- async function relevantVersionInfosOf(
- change: Change
- ): Promise {
- // Filter out all elements that don't have a "name" in the document, as
- // these are schemas, which are not relevant to our business here.
- if (change.doc.name === undefined) {
- console.error(
- `[${change.seq}] Changed document contains no 'name': ${change.id}`
- );
- metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count);
- return [];
- }
-
- // The normalize function change the object in place, if the doc object is invalid it will return undefined
- if (normalizeNPMMetadata(change.doc) === undefined) {
- console.error(
- `[${change.seq}] Changed document invalid, npm normalize returned undefined: ${change.id}`
- );
- metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count);
- return [];
- }
-
- // Sometimes, there are no versions in the document. We skip those.
- if (change.doc.versions == null) {
- console.error(
- `[${change.seq}] Changed document contains no 'versions': ${change.id}`
- );
- metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count);
- return [];
- }
-
- // Sometimes, there is no 'time' entry in the document. We skip those.
- if (change.doc.time == null) {
- console.error(
- `[${change.seq}] Changed document contains no 'time': ${change.id}`
- );
- metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count);
- return [];
- }
-
- const name = change.doc.name;
- const isRelevant = Object.values(change.doc.versions).some(
- (infos) => infos != null && isConstructLibrary(infos)
- );
-
- // Get the last modification date from the change
- const packageVersionUpdates = Object.entries(change.doc.time)
- // Ignore the "created" and "modified" keys here
- .filter(([key]) => key !== 'created' && key !== 'modified')
- // Parse all the dates to ensure they are comparable
- .map(([version, isoDate]) => [version, new Date(isoDate)] as const);
- metrics.putMetric(
- MetricName.PACKAGE_VERSION_COUNT,
- packageVersionUpdates.length,
- Unit.Count
- );
-
- const result = new Array();
- const unpublishedVersions: string[] = [];
- for (const [version, modified] of packageVersionUpdates) {
- if (isRelevant && (await knownVersions.has(name, version))) {
- continue;
- }
- const infos = change.doc.versions[version];
- if (infos == null) {
- // Could be the version in question was un-published.
- unpublishedVersions.push(`${name}@${version}`);
- } else if (isConstructLibrary(infos)) {
- // skip if this package is denied
- const denied = denyList.lookup(infos.name, infos.version);
- if (denied) {
- console.log(
- `[${change.seq}] Package denied: ${JSON.stringify(denied)}`
- );
- knownVersions.add(name, version);
- metrics.putMetric(MetricName.DENY_LISTED_COUNT, 1, Unit.Count);
- continue;
- }
-
- metrics.putMetric(
- MetricName.PACKAGE_VERSION_AGE,
- Date.now() - modified.getTime(),
- Unit.Milliseconds
- );
- const isEligible =
- licenseList.lookup(infos.license ?? 'UNLICENSED') != null;
- metrics.putMetric(
- MetricName.INELIGIBLE_LICENSE,
- isEligible ? 0 : 1,
- Unit.Count
- );
- if (isEligible) {
- result.push({
- packageName: name,
- infos,
- modified,
- seq: change.seq,
- });
- } else {
- console.log(
- `[${
- change.seq
- }] Package "${name}@${version}" does not use allow-listed license: ${
- infos.license ?? 'UNLICENSED'
- }`
- );
- knownVersions.add(name, version);
- }
- }
- // Else this is not a construct library, so we'll just ignore it...
- }
-
- if (unpublishedVersions.length > 0) {
- console.log(
- `[${
- change.seq
- }] Could not find info for the following versions. Were they un-published?\n${unpublishedVersions.join(
- ',\n'
- )}`
- );
- }
- return result;
- }
-
- /**
- * This determines whether a package is "interesting" to ConstructHub or not. This is related but
- * not necessarily identical to the logic in the ingestion process that annotates package metadata
- * with a construct framework name + version (those could ultimately be re-factored to share more
- * of the logic/heuristics, though).
- *
- * Concretely, it checks for a list of known "official" packages for various construct frameworks,
- * and packages that have a dependency on such a package. It also has a keywords allow-list as a
- * fall-back (the current dependency-based logic does not consider transitive dependencies and
- * might hence miss certain rare use-cases, which keywords would rescue).
- */
- function isConstructLibrary(infos: VersionInfo): boolean {
- if (infos.jsii == null) {
- return false;
- }
- // The "constructs" package is a sign of a constructs library
- return (
- isConstructFrameworkPackage(infos.name) ||
- // Recursively apply on dependencies
- Object.keys(infos.dependencies ?? {}).some(isConstructFrameworkPackage) ||
- Object.keys(infos.devDependencies ?? {}).some(
- isConstructFrameworkPackage
- ) ||
- Object.keys(infos.peerDependencies ?? {}).some(
- isConstructFrameworkPackage
- ) ||
- // Keyword-based fallback
- infos.keywords?.some((kw) => CONSTRUCT_KEYWORDS.has(kw))
- );
- }
-
- /**
- * Package is one of the known construct framework's first party packages:
- * - @aws-cdk/*
- * - @cdktf/*
- * - cdk8s or cdk8s-plus
- */
- function isConstructFrameworkPackage(name: string): boolean {
- // IMPORTANT NOTE: Prefix matching should only be used for @scope/ names.
-
- // The low-level constructs package
- return (
- name === 'constructs' ||
- // AWS CDK Packages
- name === 'aws-cdk-lib' ||
- name === 'monocdk' ||
- name.startsWith('@aws-cdk/') ||
- // CDK8s packages
- name === 'cdk8s' ||
- /^cdk8s-plus(?:-(?:17|20|21|22))?$/.test(name) ||
- // CDKTf packages
- name === 'cdktf' ||
- name.startsWith('@cdktf/')
- );
- }
-}
-
-/**
- * The scheme of a package version in the update. Includes the package.json keys, as well as some additional npm metadata
- * @see https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version
- */
-export interface VersionInfo {
- readonly dependencies?: { readonly [name: string]: string };
- readonly devDependencies?: { readonly [name: string]: string };
- readonly peerDependencies?: { readonly [name: string]: string };
- readonly jsii: unknown;
- readonly license?: string;
- readonly name: string;
- readonly [key: string]: unknown;
- readonly keywords: string[];
- readonly dist: {
- readonly shasum: string;
- readonly tarball: string;
- };
- readonly version: string;
-}
-
-interface UpdatedVersion {
- /**
- * The name of the package document the version was found in (the key under
- * which known versions are recorded).
- */
- readonly packageName: string;
-
- /**
- * The `VersionInfo` for the modified package version.
- */
- readonly infos: VersionInfo;
-
- /**
- * The time at which the `VersionInfo` was last modified.
- */
- readonly modified: Date;
-
- /**
- * The CouchDB transaction number for the update.
- */
- readonly seq?: string | number;
-}
-
-interface Document {
- /**
- * a List of all Version objects for the package
- */
- readonly versions: { [key: string]: VersionInfo | undefined };
-
- /**
- * The package's name.
- */
- readonly name: string;
-
- /**
- * Timestamps associated with this document. The values are ISO-8601 encoded
- * timestamps.
- */
- readonly time: {
- readonly created: string;
- readonly modified: string;
- readonly [version: string]: string;
- };
-
- readonly [key: string]: unknown;
-}
-
-interface Change extends DatabaseChange {
- readonly doc: Document;
-}
diff --git a/src/package-sources/npmjs/packument-processing.lambda-shared.ts b/src/package-sources/npmjs/packument-processing.lambda-shared.ts
new file mode 100644
index 000000000..5fc54f0de
--- /dev/null
+++ b/src/package-sources/npmjs/packument-processing.lambda-shared.ts
@@ -0,0 +1,449 @@
+import { InvokeCommand } from '@aws-sdk/client-lambda';
+import { MetricsLogger, Unit } from 'aws-embedded-metrics';
+import { MetricName } from './constants.lambda-shared';
+import {
+ CouchChanges,
+ DatabaseChange,
+ parseSequentialRevision,
+} from './couch-changes.lambda-shared';
+import { KnownVersions } from './known-versions.lambda-shared';
+import type { PackageVersion } from './stage-and-notify.lambda';
+import { DenyListClient } from '../../backend/deny-list/client.lambda-shared';
+import { LicenseListClient } from '../../backend/license-list/client.lambda-shared';
+import { LAMBDA_CLIENT } from '../../backend/shared/aws.lambda-shared';
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const normalizeNPMMetadata = require('normalize-registry-metadata');
+
+const CONSTRUCT_KEYWORDS: ReadonlySet = new Set([
+ 'cdk',
+ 'aws-cdk',
+ 'awscdk',
+ 'cdk8s',
+ 'cdktf',
+]);
+
+/**
+ * What processing a packument needs: where to send relevant versions, which
+ * versions to skip, and where to record the processed ones.
+ */
+export interface PackumentProcessingContext {
+ /** The name of the staging function. */
+ readonly stagingFunction: string;
+ readonly denyList: DenyListClient;
+ readonly licenseList: LicenseListClient;
+ readonly knownVersions: KnownVersions;
+}
+
+/**
+ * Processes packuments: identifies the relevant package versions they contain
+ * (construct libraries, not denied, with an eligible license, not processed
+ * before), sends them for staging, and records them as known versions. The
+ * caller persists the known versions (`KnownVersions.flush`).
+ *
+ * @param changes change entries with their packument attached as `doc`.
+ *
+ * @returns the number of package versions sent for staging.
+ */
+export async function processPackuments(
+ changes: readonly DatabaseChange[],
+ metrics: MetricsLogger,
+ context: PackumentProcessingContext
+): Promise {
+ const versionInfos = await getRelevantVersionInfos(
+ changes as unknown as readonly Change[],
+ metrics,
+ context.denyList,
+ context.licenseList,
+ context.knownVersions
+ );
+ console.log(
+ `Identified ${versionInfos.length} relevant package version update(s)`
+ );
+ metrics.putMetric(
+ MetricName.RELEVANT_PACKAGE_VERSIONS,
+ versionInfos.length,
+ Unit.Count
+ );
+ await stageVersions(
+ versionInfos,
+ context.stagingFunction,
+ context.knownVersions
+ );
+ return versionInfos.length;
+}
+
+/**
+ * A request to process a package from the registry, and to keep checking it
+ * until the registry serves at least the announced revision.
+ */
+export interface PackumentRequest {
+ /** The package name. */
+ readonly name: string;
+ /** The sequential revision announced by the changes feed. */
+ readonly expectedRev: number;
+ /** The changes feed position that announced it, if known. */
+ readonly seq?: number;
+ /** When the request was first made (epoch milliseconds). */
+ readonly firstSeen: number;
+ /**
+ * Why the package is checked again: the registry served an older revision
+ * than announced (`laggy`), or no packument at all (`missing`).
+ *
+ * @default 'laggy'
+ */
+ readonly reason?: PackumentRequestReason;
+}
+
+/** Why a package is checked again. */
+export type PackumentRequestReason = 'laggy' | 'missing';
+
+/** The outcome of `checkPackument`. */
+export type PackumentCheckResult =
+ /** The registry serves the expected revision (or a newer one). */
+ | 'caught-up'
+ /** The registry still serves an older revision. */
+ | 'laggy'
+ /** The registry has no packument for the package (HTTP 404). */
+ | 'missing';
+
+/**
+ * Fetches the packument a request is about from the registry, processes the
+ * versions it contains (whatever revision it is at), and reports whether the
+ * registry has caught up with the expected revision.
+ */
+export async function checkPackument(
+ npm: CouchChanges,
+ request: PackumentRequest,
+ metrics: MetricsLogger,
+ context: PackumentProcessingContext
+): Promise {
+ const doc = await npm.getPackageDoc(request.name);
+ if (doc == null || doc._rev == null) {
+ return 'missing';
+ }
+ const change: DatabaseChange = {
+ changes: [{ rev: doc._rev as string }],
+ deleted: false,
+ id: request.name,
+ seq: request.seq,
+ doc,
+ };
+ await processPackuments([change], metrics, context);
+ return parseSequentialRevision(doc._rev as string) >= request.expectedRev
+ ? 'caught-up'
+ : 'laggy';
+}
+
+/**
+ * Sends the provided package version updates to the staging function
+ * ("fire-and-forget"), and records them as known versions.
+ */
+export async function stageVersions(
+ versionInfos: readonly UpdatedVersion[],
+ stagingFunction: string,
+ knownVersions: KnownVersions
+): Promise {
+ await Promise.all(
+ versionInfos.map(async ({ packageName, infos, modified, seq }) => {
+ const invokeArgs: PackageVersion = {
+ integrity: infos.dist.shasum,
+ modified: modified.toISOString(),
+ name: infos.name,
+ seq: seq?.toString(),
+ tarballUrl: infos.dist.tarball,
+ version: infos.version,
+ };
+ // "Fire-and-forget" invocation here.
+ console.log(`Sending ${invokeArgs.tarballUrl} for staging`);
+ await LAMBDA_CLIENT.send(
+ new InvokeCommand({
+ FunctionName: stagingFunction,
+ InvocationType: 'Event',
+ Payload: Buffer.from(JSON.stringify(invokeArgs)),
+ })
+ );
+ // Record that this is now a "known" version (no need to re-discover)
+ knownVersions.add(packageName, infos.version);
+ })
+ );
+}
+
+/**
+ * Obtains the `VersionInfo` corresponding to the modified version(s) in the
+ * provided `Change` objects, ensures they are relevant (construct libraries),
+ * and returns those only.
+ *
+ * Known versions are only looked up for packages that have at least one
+ * construct library version: only those versions are ever recorded, so every
+ * other package has no known versions by definition. Changes are processed
+ * concurrently; the result keeps their order.
+ *
+ * @param changes the changes to be processed.
+ * @param metrics the metrics logger to use.
+ * @param denyList deny list client
+ *
+ * @returns a list of `VersionInfo` objects
+ */
+export async function getRelevantVersionInfos(
+ changes: readonly Change[],
+ metrics: MetricsLogger,
+ denyList: DenyListClient,
+ licenseList: LicenseListClient,
+ knownVersions: KnownVersions
+): Promise {
+ const perChange = await Promise.all(
+ changes.map((change) => relevantVersionInfosOf(change))
+ );
+ return perChange.flat();
+
+ async function relevantVersionInfosOf(
+ change: Change
+ ): Promise {
+ // Filter out all elements that don't have a "name" in the document, as
+ // these are schemas, which are not relevant to our business here.
+ if (change.doc.name === undefined) {
+ console.error(
+ `[${change.seq}] Changed document contains no 'name': ${change.id}`
+ );
+ metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count);
+ return [];
+ }
+
+ // The normalize function change the object in place, if the doc object is invalid it will return undefined
+ if (normalizeNPMMetadata(change.doc) === undefined) {
+ console.error(
+ `[${change.seq}] Changed document invalid, npm normalize returned undefined: ${change.id}`
+ );
+ metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count);
+ return [];
+ }
+
+ // Sometimes, there are no versions in the document. We skip those.
+ if (change.doc.versions == null) {
+ console.error(
+ `[${change.seq}] Changed document contains no 'versions': ${change.id}`
+ );
+ metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count);
+ return [];
+ }
+
+ // Sometimes, there is no 'time' entry in the document. We skip those.
+ if (change.doc.time == null) {
+ console.error(
+ `[${change.seq}] Changed document contains no 'time': ${change.id}`
+ );
+ metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count);
+ return [];
+ }
+
+ const name = change.doc.name;
+ const isRelevant = Object.values(change.doc.versions).some(
+ (infos) => infos != null && isConstructLibrary(infos)
+ );
+
+ // Get the last modification date from the change
+ const packageVersionUpdates = Object.entries(change.doc.time)
+ // Ignore the "created" and "modified" keys here
+ .filter(([key]) => key !== 'created' && key !== 'modified')
+ // Parse all the dates to ensure they are comparable
+ .map(([version, isoDate]) => [version, new Date(isoDate)] as const);
+ metrics.putMetric(
+ MetricName.PACKAGE_VERSION_COUNT,
+ packageVersionUpdates.length,
+ Unit.Count
+ );
+
+ const result = new Array();
+ const unpublishedVersions: string[] = [];
+ for (const [version, modified] of packageVersionUpdates) {
+ if (isRelevant && (await knownVersions.has(name, version))) {
+ continue;
+ }
+ const infos = change.doc.versions[version];
+ if (infos == null) {
+ // Could be the version in question was un-published.
+ unpublishedVersions.push(`${name}@${version}`);
+ } else if (isConstructLibrary(infos)) {
+ // skip if this package is denied
+ const denied = denyList.lookup(infos.name, infos.version);
+ if (denied) {
+ console.log(
+ `[${change.seq}] Package denied: ${JSON.stringify(denied)}`
+ );
+ knownVersions.add(name, version);
+ metrics.putMetric(MetricName.DENY_LISTED_COUNT, 1, Unit.Count);
+ continue;
+ }
+
+ metrics.putMetric(
+ MetricName.PACKAGE_VERSION_AGE,
+ Date.now() - modified.getTime(),
+ Unit.Milliseconds
+ );
+ const isEligible =
+ licenseList.lookup(infos.license ?? 'UNLICENSED') != null;
+ metrics.putMetric(
+ MetricName.INELIGIBLE_LICENSE,
+ isEligible ? 0 : 1,
+ Unit.Count
+ );
+ if (isEligible) {
+ result.push({
+ packageName: name,
+ infos,
+ modified,
+ seq: change.seq,
+ });
+ } else {
+ console.log(
+ `[${
+ change.seq
+ }] Package "${name}@${version}" does not use allow-listed license: ${
+ infos.license ?? 'UNLICENSED'
+ }`
+ );
+ knownVersions.add(name, version);
+ }
+ }
+ // Else this is not a construct library, so we'll just ignore it...
+ }
+
+ if (unpublishedVersions.length > 0) {
+ console.log(
+ `[${
+ change.seq
+ }] Could not find info for the following versions. Were they un-published?\n${unpublishedVersions.join(
+ ',\n'
+ )}`
+ );
+ }
+ return result;
+ }
+
+ /**
+ * This determines whether a package is "interesting" to ConstructHub or not. This is related but
+ * not necessarily identical to the logic in the ingestion process that annotates package metadata
+ * with a construct framework name + version (those could ultimately be re-factored to share more
+ * of the logic/heuristics, though).
+ *
+ * Concretely, it checks for a list of known "official" packages for various construct frameworks,
+ * and packages that have a dependency on such a package. It also has a keywords allow-list as a
+ * fall-back (the current dependency-based logic does not consider transitive dependencies and
+ * might hence miss certain rare use-cases, which keywords would rescue).
+ */
+ function isConstructLibrary(infos: VersionInfo): boolean {
+ if (infos.jsii == null) {
+ return false;
+ }
+ // The "constructs" package is a sign of a constructs library
+ return (
+ isConstructFrameworkPackage(infos.name) ||
+ // Recursively apply on dependencies
+ Object.keys(infos.dependencies ?? {}).some(isConstructFrameworkPackage) ||
+ Object.keys(infos.devDependencies ?? {}).some(
+ isConstructFrameworkPackage
+ ) ||
+ Object.keys(infos.peerDependencies ?? {}).some(
+ isConstructFrameworkPackage
+ ) ||
+ // Keyword-based fallback
+ infos.keywords?.some((kw) => CONSTRUCT_KEYWORDS.has(kw))
+ );
+ }
+
+ /**
+ * Package is one of the known construct framework's first party packages:
+ * - @aws-cdk/*
+ * - @cdktf/*
+ * - cdk8s or cdk8s-plus
+ */
+ function isConstructFrameworkPackage(name: string): boolean {
+ // IMPORTANT NOTE: Prefix matching should only be used for @scope/ names.
+
+ // The low-level constructs package
+ return (
+ name === 'constructs' ||
+ // AWS CDK Packages
+ name === 'aws-cdk-lib' ||
+ name === 'monocdk' ||
+ name.startsWith('@aws-cdk/') ||
+ // CDK8s packages
+ name === 'cdk8s' ||
+ /^cdk8s-plus(?:-(?:17|20|21|22))?$/.test(name) ||
+ // CDKTf packages
+ name === 'cdktf' ||
+ name.startsWith('@cdktf/')
+ );
+ }
+}
+
+/**
+ * The scheme of a package version in the update. Includes the package.json keys, as well as some additional npm metadata
+ * @see https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version
+ */
+export interface VersionInfo {
+ readonly dependencies?: { readonly [name: string]: string };
+ readonly devDependencies?: { readonly [name: string]: string };
+ readonly peerDependencies?: { readonly [name: string]: string };
+ readonly jsii: unknown;
+ readonly license?: string;
+ readonly name: string;
+ readonly [key: string]: unknown;
+ readonly keywords: string[];
+ readonly dist: {
+ readonly shasum: string;
+ readonly tarball: string;
+ };
+ readonly version: string;
+}
+
+export interface UpdatedVersion {
+ /**
+ * The name of the package document the version was found in (the key under
+ * which known versions are recorded).
+ */
+ readonly packageName: string;
+
+ /**
+ * The `VersionInfo` for the modified package version.
+ */
+ readonly infos: VersionInfo;
+
+ /**
+ * The time at which the `VersionInfo` was last modified.
+ */
+ readonly modified: Date;
+
+ /**
+ * The CouchDB transaction number for the update.
+ */
+ readonly seq?: string | number;
+}
+
+interface Document {
+ /**
+ * a List of all Version objects for the package
+ */
+ readonly versions: { [key: string]: VersionInfo | undefined };
+
+ /**
+ * The package's name.
+ */
+ readonly name: string;
+
+ /**
+ * Timestamps associated with this document. The values are ISO-8601 encoded
+ * timestamps.
+ */
+ readonly time: {
+ readonly created: string;
+ readonly modified: string;
+ readonly [version: string]: string;
+ };
+
+ readonly [key: string]: unknown;
+}
+
+interface Change extends DatabaseChange {
+ readonly doc: Document;
+}
diff --git a/src/package-sources/npmjs/packument-processor.lambda.ts b/src/package-sources/npmjs/packument-processor.lambda.ts
new file mode 100644
index 000000000..df9026539
--- /dev/null
+++ b/src/package-sources/npmjs/packument-processor.lambda.ts
@@ -0,0 +1,274 @@
+import {
+ metricScope,
+ Configuration,
+ MetricsLogger,
+ Unit,
+} from 'aws-embedded-metrics';
+import type {
+ Context,
+ SQSBatchResponse,
+ SQSEvent,
+ SQSRecord,
+} from 'aws-lambda';
+import { captureHTTPsGlobal } from 'aws-xray-sdk-core';
+import {
+ ENV_KNOWN_VERSIONS_TABLE_NAME,
+ ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN,
+ ENV_PACKUMENT_QUEUE_URL,
+ METRICS_NAMESPACE,
+ MetricName,
+ NPM_REPLICA_REGISTRY_URL,
+} from './constants.lambda-shared';
+import { CouchChanges } from './couch-changes.lambda-shared';
+import { KnownVersions } from './known-versions.lambda-shared';
+import {
+ checkPackument,
+ PackumentProcessingContext,
+ PackumentRequest,
+ PackumentRequestReason,
+} from './packument-processing.lambda-shared';
+import {
+ parsePackumentRequest,
+ retryDelaySeconds,
+ setVisibilityTimeout,
+} from './packument-queue.lambda-shared';
+import { DenyListClient } from '../../backend/deny-list/client.lambda-shared';
+import { LicenseListClient } from '../../backend/license-list/client.lambda-shared';
+import { requireEnv } from '../../backend/shared/env.lambda-shared';
+
+Configuration.namespace = METRICS_NAMESPACE;
+
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+captureHTTPsGlobal(require('https'));
+
+/**
+ * Processes packument requests: fetches each package from the registry and
+ * processes the versions it serves. A request is done once the registry serves
+ * the announced revision. Otherwise it is checked again later, with growing
+ * waits. After about 29 hours the queue moves it to the dead-letter queue.
+ *
+ * This function also reads the dead-letter queue. A request that arrives from
+ * there was given up on: it is logged, counted, and deleted.
+ */
+export async function handler(
+ event: SQSEvent,
+ _context: Context
+): Promise {
+ const deadLetterQueueArn = requireEnv(ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN);
+ const givenUp = event.Records.filter(
+ (record) => record.eventSourceARN === deadLetterQueueArn
+ );
+ const pending = event.Records.filter(
+ (record) => record.eventSourceARN !== deadLetterQueueArn
+ );
+ if (givenUp.length > 0) {
+ await reportGiveUps(givenUp);
+ }
+ if (pending.length === 0) {
+ return { batchItemFailures: [] };
+ }
+
+ const queueUrl = requireEnv(ENV_PACKUMENT_QUEUE_URL);
+ let processing: PackumentProcessingContext;
+ try {
+ processing = {
+ stagingFunction: requireEnv('FUNCTION_NAME'),
+ denyList: await DenyListClient.newClient(),
+ licenseList: await LicenseListClient.newClient(),
+ knownVersions: new KnownVersions(
+ requireEnv(ENV_KNOWN_VERSIONS_TABLE_NAME)
+ ),
+ };
+ } catch (error) {
+ // Keep the usual waits; otherwise a broken processor would use up all
+ // attempts at the queue's visibility timeout.
+ console.error(`Could not set up processing, retrying the batch: ${error}`);
+ await Promise.all(
+ pending.map((record) => scheduleNextAttempt(record, queueUrl))
+ );
+ throw error;
+ }
+ const npm = new CouchChanges(NPM_REPLICA_REGISTRY_URL, 'registry/_changes');
+
+ let outcomes = new Array();
+ await metricScope((metrics) => async () => {
+ metrics.setDimensions({});
+ outcomes = await Promise.all(
+ pending.map((record) => processRecord(record, npm, metrics, processing))
+ );
+ const recovered = (reason: PackumentRequestReason) =>
+ outcomes.filter((o) => o.done && o.reason === reason).length;
+ metrics.putMetric(
+ MetricName.LAGGY_PACKUMENTS_RECOVERED,
+ recovered('laggy'),
+ Unit.Count
+ );
+ metrics.putMetric(
+ MetricName.MISSING_PACKUMENTS_RECOVERED,
+ recovered('missing'),
+ Unit.Count
+ );
+ for (const { lag } of outcomes) {
+ if (lag != null) {
+ metrics.putMetric(
+ MetricName.LAGGY_PACKUMENT_LAG,
+ lag,
+ Unit.Milliseconds
+ );
+ }
+ }
+ })();
+
+ try {
+ await processing.knownVersions.flush();
+ } catch (error) {
+ // The versions were staged but not recorded. Check these packages again
+ // so they get recorded.
+ console.error(`Could not record the known versions: ${error}`);
+ for (const outcome of outcomes) {
+ outcome.done = false;
+ }
+ }
+
+ const failures = new Array();
+ await Promise.all(
+ pending.map(async (record, i) => {
+ if (!outcomes[i].done) {
+ failures.push(record.messageId);
+ await scheduleNextAttempt(record, queueUrl);
+ }
+ })
+ );
+ return {
+ batchItemFailures: failures.map((itemIdentifier) => ({ itemIdentifier })),
+ };
+}
+
+interface RecordOutcome {
+ readonly reason?: PackumentRequestReason;
+ /** For a laggy packument that caught up: how long it took. */
+ readonly lag?: number;
+ /** Whether the message can be deleted from the queue. */
+ done: boolean;
+}
+
+/**
+ * Processes one queue message.
+ */
+async function processRecord(
+ record: SQSRecord,
+ npm: CouchChanges,
+ metrics: MetricsLogger,
+ processing: PackumentProcessingContext
+): Promise {
+ let name = record.messageId;
+ let reason: PackumentRequestReason | undefined;
+ try {
+ const request = parsePackumentRequest(record.body);
+ name = request.name;
+ reason = request.reason ?? 'laggy';
+ const result = await checkPackument(npm, request, metrics, processing);
+ if (result !== 'caught-up') {
+ return { reason, done: false };
+ }
+ console.log(
+ `Packument for ${name} caught up with rev ${request.expectedRev} (${reason})`
+ );
+ return { reason, lag: lagOf(request), done: true };
+ } catch (error) {
+ console.error(
+ `Failed to process the packument request for ${name}: ${error}`
+ );
+ return { reason, done: false };
+ }
+}
+
+/**
+ * Logs and counts requests that were given up on. They are deleted from the
+ * dead-letter queue afterwards: nothing would redrive them.
+ */
+async function reportGiveUps(records: readonly SQSRecord[]): Promise {
+ await metricScope((metrics) => async () => {
+ metrics.setDimensions({});
+ const counts: Record = {
+ laggy: 0,
+ missing: 0,
+ };
+ for (const record of records) {
+ let request: PackumentRequest;
+ try {
+ request = parsePackumentRequest(record.body);
+ } catch {
+ console.warn(
+ `Giving up on an invalid packument request: ${record.body}`
+ );
+ continue;
+ }
+ const reason = request.reason ?? 'laggy';
+ counts[reason] += 1;
+ const problem =
+ reason === 'missing'
+ ? 'never served a packument for it'
+ : `never served rev ${request.expectedRev}`;
+ console.warn(
+ `Giving up on ${reason} packument for ${
+ request.name
+ }: the registry ${problem} (first seen ${new Date(
+ request.firstSeen
+ ).toISOString()}). A version announced by the changes feed may be missing until the package publishes again.`
+ );
+ const lag = lagOf(request);
+ if (lag != null) {
+ metrics.putMetric(
+ MetricName.LAGGY_PACKUMENT_LAG,
+ lag,
+ Unit.Milliseconds
+ );
+ }
+ }
+ metrics.putMetric(
+ MetricName.LAGGY_PACKUMENT_GIVE_UPS,
+ counts.laggy,
+ Unit.Count
+ );
+ metrics.putMetric(
+ MetricName.MISSING_PACKUMENT_GIVE_UPS,
+ counts.missing,
+ Unit.Count
+ );
+ })();
+}
+
+/** How long a request has been waiting. */
+function lagOf(request: PackumentRequest): number | undefined {
+ return (request.reason ?? 'laggy') === 'laggy'
+ ? Date.now() - request.firstSeen
+ : undefined;
+}
+
+/**
+ * Makes a message visible again for its next attempt.
+ */
+async function scheduleNextAttempt(
+ record: SQSRecord,
+ queueUrl: string
+): Promise {
+ let firstSeen: number | undefined;
+ try {
+ firstSeen = parsePackumentRequest(record.body).firstSeen;
+ } catch {
+ // An invalid request gets the shortest wait, and ends up given up on.
+ }
+ try {
+ await setVisibilityTimeout(
+ queueUrl,
+ record.receiptHandle,
+ retryDelaySeconds(firstSeen)
+ );
+ } catch (error) {
+ // The message returns after the queue's visibility timeout instead.
+ console.error(
+ `Could not schedule the next attempt for ${record.messageId}: ${error}`
+ );
+ }
+}
diff --git a/src/package-sources/npmjs/packument-processor.ts b/src/package-sources/npmjs/packument-processor.ts
new file mode 100644
index 000000000..81125362a
--- /dev/null
+++ b/src/package-sources/npmjs/packument-processor.ts
@@ -0,0 +1,20 @@
+// ~~ Generated by projen. To modify, edit .projenrc.ts and run "yarn projen".
+import * as path from 'path';
+import * as lambda from 'aws-cdk-lib/aws-lambda';
+import { Construct } from 'constructs';
+
+export interface PackumentProcessorProps extends lambda.FunctionOptions {
+}
+
+export class PackumentProcessor extends lambda.Function {
+ constructor(scope: Construct, id: string, props?: PackumentProcessorProps) {
+ super(scope, id, {
+ description: 'package-sources/npmjs/packument-processor.lambda.ts',
+ ...props,
+ architecture: lambda.Architecture.ARM_64,
+ runtime: lambda.Runtime.NODEJS_22_X,
+ handler: 'index.handler',
+ code: lambda.Code.fromAsset(path.join(__dirname, '/packument-processor.lambda.bundle')),
+ });
+ }
+}
\ No newline at end of file
diff --git a/src/package-sources/npmjs/packument-queue-processor.ts b/src/package-sources/npmjs/packument-queue-processor.ts
new file mode 100644
index 000000000..def1da211
--- /dev/null
+++ b/src/package-sources/npmjs/packument-queue-processor.ts
@@ -0,0 +1,111 @@
+import { Duration } from 'aws-cdk-lib';
+import { ITableV2 } from 'aws-cdk-lib/aws-dynamodb';
+import { IFunction, Function, Tracing } from 'aws-cdk-lib/aws-lambda';
+import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources';
+import { IQueue, Queue, QueueEncryption } from 'aws-cdk-lib/aws-sqs';
+import { Construct } from 'constructs';
+import {
+ ENV_KNOWN_VERSIONS_TABLE_NAME,
+ ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN,
+ ENV_PACKUMENT_QUEUE_URL,
+ PACKUMENT_MAX_ATTEMPTS,
+} from './constants.lambda-shared';
+import { PackumentProcessor } from './packument-processor';
+import { IDenyList } from '../../backend/deny-list/api';
+import { ILicenseList } from '../../backend/license-list/api';
+
+export interface PackumentQueueProcessorProps {
+ /** The table recording the package versions that were processed. */
+ readonly knownVersions: ITableV2;
+
+ /** The function that stages relevant package versions. */
+ readonly stager: IFunction;
+
+ readonly denyList?: IDenyList;
+ readonly licenseList: ILicenseList;
+}
+
+/**
+ * A queue of requests to process a package from the npm registry, and the
+ * function that works through it. A request is retried with back-off until the
+ * registry serves the revision the changes feed announced. After about 29
+ * hours it moves to the dead-letter queue, where the same function logs it as
+ * given up on.
+ */
+export class PackumentQueueProcessor extends Construct {
+ /** The packument requests. */
+ public readonly queue: IQueue;
+
+ /** The packument requests that were given up on. */
+ public readonly deadLetterQueue: IQueue;
+
+ /** The function processing the requests. */
+ public readonly function: Function;
+
+ public constructor(
+ scope: Construct,
+ id: string,
+ props: PackumentQueueProcessorProps
+ ) {
+ super(scope, id);
+
+ // Given-up requests. The function logs and deletes them right away; they
+ // are only kept while it is broken.
+ this.deadLetterQueue = new Queue(this, 'DLQ', {
+ encryption: QueueEncryption.KMS_MANAGED,
+ retentionPeriod: Duration.days(14),
+ });
+ this.queue = new Queue(this, 'Queue', {
+ encryption: QueueEncryption.KMS_MANAGED,
+ retentionPeriod: Duration.days(14),
+ // Only used when the processor cannot schedule the next attempt itself,
+ // for example on a timeout. AWS recommends 6x the function timeout.
+ visibilityTimeout: Duration.minutes(30),
+ deadLetterQueue: {
+ queue: this.deadLetterQueue,
+ maxReceiveCount: PACKUMENT_MAX_ATTEMPTS,
+ },
+ });
+
+ this.function = new PackumentProcessor(this, 'Function', {
+ description: `[${this.node.path}] Processes packages from the npm registry, retrying until the registry serves the revision announced by the changes feed`,
+ environment: {
+ AWS_EMF_ENVIRONMENT: 'Local',
+ FUNCTION_NAME: props.stager.functionName,
+ [ENV_KNOWN_VERSIONS_TABLE_NAME]: props.knownVersions.tableName,
+ [ENV_PACKUMENT_QUEUE_URL]: this.queue.queueUrl,
+ [ENV_PACKUMENT_DEAD_LETTER_QUEUE_ARN]: this.deadLetterQueue.queueArn,
+ },
+ memorySize: 2_048,
+ timeout: Duration.minutes(5),
+ tracing: Tracing.ACTIVE,
+ });
+ this.function.addEventSource(
+ new SqsEventSource(this.queue, {
+ batchSize: 10,
+ reportBatchItemFailures: true,
+ // Be gentle with the npm registry.
+ maxConcurrency: 2,
+ })
+ );
+ this.function.addEventSource(
+ new SqsEventSource(this.deadLetterQueue, {
+ batchSize: 10,
+ reportBatchItemFailures: true,
+ })
+ );
+ props.knownVersions.grantReadWriteData(this.function);
+ props.denyList?.grantRead(this.function);
+ props.licenseList.grantRead(this.function);
+ props.stager.grantInvoke(this.function);
+ }
+
+ /**
+ * Allows a function to send requests to the queue, and tells it where the
+ * queue is.
+ */
+ public grantSendRequests(fn: Function): void {
+ fn.addEnvironment(ENV_PACKUMENT_QUEUE_URL, this.queue.queueUrl);
+ this.queue.grantSendMessages(fn);
+ }
+}
diff --git a/src/package-sources/npmjs/packument-queue.lambda-shared.ts b/src/package-sources/npmjs/packument-queue.lambda-shared.ts
new file mode 100644
index 000000000..bd2af94c4
--- /dev/null
+++ b/src/package-sources/npmjs/packument-queue.lambda-shared.ts
@@ -0,0 +1,149 @@
+import {
+ ChangeMessageVisibilityCommand,
+ GetQueueAttributesCommand,
+ SendMessageBatchCommand,
+ SendMessageBatchRequestEntry,
+} from '@aws-sdk/client-sqs';
+import {
+ PACKUMENT_INITIAL_DELAY_SECONDS,
+ PACKUMENT_MAX_RETRY_DELAY_SECONDS,
+ PACKUMENT_MIN_RETRY_DELAY_SECONDS,
+ PACKUMENT_RETRY_JITTER,
+} from './constants.lambda-shared';
+import type { PackumentRequest } from './packument-processing.lambda-shared';
+import { SQS_CLIENT } from '../../backend/shared/aws.lambda-shared';
+
+/** The maximum number of messages in a single `SendMessageBatch` request. */
+const SEND_BATCH_SIZE = 10;
+
+/** How often the failed entries of a batch send are retried. */
+const MAX_SEND_ATTEMPTS = 3;
+
+/**
+ * Sends packument requests to the packument queue. The first attempt happens
+ * after `PACKUMENT_INITIAL_DELAY_SECONDS`.
+ *
+ * @throws if any request could not be sent.
+ */
+export async function enqueuePackumentRequests(
+ queueUrl: string,
+ requests: readonly PackumentRequest[]
+): Promise {
+ const batches = new Array();
+ for (let i = 0; i < requests.length; i += SEND_BATCH_SIZE) {
+ batches.push(requests.slice(i, i + SEND_BATCH_SIZE));
+ }
+ await Promise.all(batches.map((batch) => sendBatch(queueUrl, batch)));
+}
+
+async function sendBatch(
+ queueUrl: string,
+ batch: readonly PackumentRequest[]
+): Promise {
+ let entries: SendMessageBatchRequestEntry[] = batch.map((request, i) => ({
+ Id: i.toString(),
+ MessageBody: JSON.stringify(request),
+ DelaySeconds: PACKUMENT_INITIAL_DELAY_SECONDS,
+ }));
+ for (let attempt = 1; ; attempt++) {
+ const response = await SQS_CLIENT.send(
+ new SendMessageBatchCommand({ QueueUrl: queueUrl, Entries: entries })
+ );
+ const failed = new Set((response.Failed ?? []).map((f) => f.Id));
+ if (failed.size === 0) {
+ return;
+ }
+ if (attempt >= MAX_SEND_ATTEMPTS) {
+ const reasons = (response.Failed ?? [])
+ .map((f) => `${f.Code}: ${f.Message}`)
+ .join('; ');
+ throw new Error(
+ `Could not send ${failed.size} packument request(s) to ${queueUrl}: ${reasons}`
+ );
+ }
+ entries = entries.filter((entry) => failed.has(entry.Id));
+ }
+}
+
+/**
+ * Parses the body of a packument queue message.
+ *
+ * @throws if the body is not a packument request.
+ */
+export function parsePackumentRequest(body: string): PackumentRequest {
+ const request = JSON.parse(body) as Partial;
+ if (
+ typeof request.name !== 'string' ||
+ typeof request.expectedRev !== 'number' ||
+ typeof request.firstSeen !== 'number' ||
+ (request.reason != null &&
+ request.reason !== 'laggy' &&
+ request.reason !== 'missing')
+ ) {
+ throw new Error(`Not a packument request: ${body}`);
+ }
+ return request as PackumentRequest;
+}
+
+/**
+ * How long to wait before the next attempt. A request waits as long as it is
+ * already old, within the minimum and maximum delay, plus or minus some jitter.
+ * This doubles the wait with each attempt, without counting attempts.
+ *
+ * @param firstSeen when the request was made (epoch milliseconds), if known.
+ */
+export function retryDelaySeconds(
+ firstSeen: number | undefined,
+ now: number = Date.now(),
+ random: () => number = Math.random
+): number {
+ const ageSeconds = firstSeen == null ? 0 : (now - firstSeen) / 1_000;
+ const delay = Math.min(
+ Math.max(ageSeconds, PACKUMENT_MIN_RETRY_DELAY_SECONDS),
+ PACKUMENT_MAX_RETRY_DELAY_SECONDS
+ );
+ const jitter = 1 + PACKUMENT_RETRY_JITTER * (2 * random() - 1);
+ return Math.round(delay * jitter);
+}
+
+/**
+ * Makes a message the queue already delivered visible again after the given
+ * number of seconds, instead of the queue's visibility timeout.
+ */
+export async function setVisibilityTimeout(
+ queueUrl: string,
+ receiptHandle: string,
+ seconds: number
+): Promise {
+ await SQS_CLIENT.send(
+ new ChangeMessageVisibilityCommand({
+ QueueUrl: queueUrl,
+ ReceiptHandle: receiptHandle,
+ VisibilityTimeout: seconds,
+ })
+ );
+}
+
+/**
+ * The approximate number of requests in the packument queue: waiting, being
+ * processed, and waiting for their next attempt.
+ */
+export async function queuedPackumentRequests(
+ queueUrl: string
+): Promise {
+ const { Attributes: attributes = {} } = await SQS_CLIENT.send(
+ new GetQueueAttributesCommand({
+ QueueUrl: queueUrl,
+ AttributeNames: [
+ 'ApproximateNumberOfMessages',
+ 'ApproximateNumberOfMessagesNotVisible',
+ 'ApproximateNumberOfMessagesDelayed',
+ ],
+ })
+ );
+ return (
+ Number(attributes.ApproximateNumberOfMessages ?? 0) +
+ Number(attributes.ApproximateNumberOfMessagesNotVisible ?? 0) +
+ Number(attributes.ApproximateNumberOfMessagesDelayed ?? 0)
+ );
+}
diff --git a/src/package-sources/npmjs/re-stage-package-version.lambda.ts b/src/package-sources/npmjs/re-stage-package-version.lambda.ts
index d2c943c41..dd30754c8 100644
--- a/src/package-sources/npmjs/re-stage-package-version.lambda.ts
+++ b/src/package-sources/npmjs/re-stage-package-version.lambda.ts
@@ -1,5 +1,5 @@
import { InvokeCommand } from '@aws-sdk/client-lambda';
-import { VersionInfo } from './npm-js-follower.lambda';
+import type { VersionInfo } from './packument-processing.lambda-shared';
import { PackageVersion } from './stage-and-notify.lambda';
import { LAMBDA_CLIENT } from '../../backend/shared/aws.lambda-shared';
import { requireEnv } from '../../backend/shared/env.lambda-shared';