Run nightly Stellar testnet payment checks

.github/workflows/nightly-testnet-payments.yml

@@ -0,0 +1,129 @@
+name: Nightly testnet payments
+
+on:
+  schedule:
+    - cron: '17 2 * * *'
+  workflow_dispatch:
+    inputs:
+      simulate_failure:
+        description: Exercise the failure-reporting path without running the test suite
+        type: boolean
+        default: false
+        required: false
+
+permissions:
+  contents: read
+  issues: write
+
+concurrency:
+  group: nightly-testnet-payments
+  cancel-in-progress: false
+
+jobs:
+  testnet-payments:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: backend
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: backend/package-lock.json
+
+      - run: npm ci
+      - run: npm run prisma:generate
+
+      # These are public testnet identifiers from backend/.env.example, not
+      # preview deployment secrets. Repository variables can rotate them.
+      - name: Generate fresh testnet identities and fund with Friendbot
+        id: identities
+        continue-on-error: true
+        env:
+          ESCROW_CONTRACT_ID: ${{ vars.ESCROW_CONTRACT_ID || 'CAXESEEAJILFFCN3PH2HHNMWAAHLEB64UTJHZ6IP7C5BBICT3S3VODSM' }}
+          ESCROW_TOKEN_ID: ${{ vars.ESCROW_TOKEN_ID || 'CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC' }}
+        run: node scripts/gen-platform-identities.mjs
+
+      - name: Simulate a test failure
+        id: simulated_failure
+        if: github.event_name == 'workflow_dispatch' && inputs.simulate_failure
+        continue-on-error: true
+        run: |
+          echo 'Deliberate failure to verify the workflow failure reporter.'
+          exit 1
+
+      # The test suite starts and tears down its own isolated PostgreSQL
+      # cluster, so no shared service database or credentials are needed.
+      - name: Run payments against Stellar testnet
+        id: test_suite
+        if: steps.identities.outcome == 'success' && steps.simulated_failure.outcome != 'failure'
+        continue-on-error: true
+        env:
+          ESCROW_CONTRACT_ID: ${{ vars.ESCROW_CONTRACT_ID || 'CAXESEEAJILFFCN3PH2HHNMWAAHLEB64UTJHZ6IP7C5BBICT3S3VODSM' }}
+          ESCROW_TOKEN_ID: ${{ vars.ESCROW_TOKEN_ID || 'CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC' }}
+        run: |
+          set +e
+          npm run test:integration:payments-testnet > "$RUNNER_TEMP/payments-testnet.log" 2>&1
+          result=$?
+          cat "$RUNNER_TEMP/payments-testnet.log"
+          set -e
+          if [[ "$result" -eq 0 ]]; then
+            echo 'failure_class=none' >> "$GITHUB_OUTPUT"
+          elif grep -Eiq 'friendbot|ECONN|ETIMEDOUT|ENOTFOUND|fetch failed|MissingValue|NoSuchContract|contract.{0,40}(missing|not found|does not exist)|(missing|not found|does not exist).{0,40}contract' "$RUNNER_TEMP/payments-testnet.log"; then
+            echo 'failure_class=infrastructure_or_contract_reset' >> "$GITHUB_OUTPUT"
+          else
+            echo 'failure_class=test_failure' >> "$GITHUB_OUTPUT"
+          fi
+
+          exit "$result"
+
+      - name: Update failure summary and tracking issue
+        if: always() && (steps.identities.outcome == 'failure' || steps.simulated_failure.outcome == 'failure' || steps.test_suite.outcome == 'failure')
+        env:
+          GH_TOKEN: ${{ github.token }}
+          IDENTITY_OUTCOME: ${{ steps.identities.outcome }}
+          SIMULATED_OUTCOME: ${{ steps.simulated_failure.outcome }}
+          TEST_OUTCOME: ${{ steps.test_suite.outcome }}
+          TEST_FAILURE_CLASS: ${{ steps.test_suite.outputs.failure_class }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          if [[ "$IDENTITY_OUTCOME" == failure ]]; then
+            failure_class='Infrastructure failure: Friendbot funding or identity setup failed.'
+          elif [[ "$SIMULATED_OUTCOME" == failure ]]; then
+            failure_class='Simulated test failure: the manual failure-reporting check was requested.'
+          elif [[ "$TEST_FAILURE_CLASS" == infrastructure_or_contract_reset ]]; then
+            failure_class='Infrastructure failure: Friendbot/RPC outage or missing/reset testnet contract.'
+          else
+            failure_class='Payment test failure: the test suite failed without a recognized infrastructure signature.'
+          fi
+
+          {
+            echo '### Nightly testnet payments failed'
+            echo
+            echo "**Classification:** $failure_class"
+            echo "**Run:** $RUN_URL"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          body=$(printf 'The scheduled/manual Stellar testnet payment run failed.\n\n**Classification:** %s\n**Run:** %s\n\nThis issue is updated in place on subsequent failures to avoid notification spam.' "$failure_class" "$RUN_URL")
+          issue_number=$(gh issue list --state all --limit 100 --json number,title --jq '.[] | select(.title == "[Nightly Testnet Payments] Failure") | .number' | head -n 1)
+          if [[ -n "$issue_number" ]]; then
+            gh issue reopen "$issue_number" 2>/dev/null || true
+            gh issue edit "$issue_number" --body "$body"
+          else
+            gh issue create --title '[Nightly Testnet Payments] Failure' --body "$body"
+          fi
+
+      - name: Fail the workflow when a required step failed
+        if: always()
+        env:
+          IDENTITY_OUTCOME: ${{ steps.identities.outcome }}
+          SIMULATED_OUTCOME: ${{ steps.simulated_failure.outcome }}
+          TEST_OUTCOME: ${{ steps.test_suite.outcome }}
+        run: |
+          if [[ "$IDENTITY_OUTCOME" == failure || "$SIMULATED_OUTCOME" == failure || "$TEST_OUTCOME" == failure ]]; then
+            exit 1
+          fi

ROADMAP.md

@@ -87,6 +87,7 @@ for status and issue links.
 - [ ] Idempotent escrow funding + on-chain reconcile job (`fundPaymentIntent` is check-then-act — docforum-escrow threat model §5). Tracked as [issue #24](https://github.com/DocForum/docforum-core/issues/24) (High, 200 pts).
 - [ ] ADR: releaser key custody — multisig, KMS, or per-period rotation (threat model T1/T2). Tracked as [issue #25](https://github.com/DocForum/docforum-core/issues/25) (Medium, 150 pts).
 - [ ] Checksum-validate facility wallet keys, not just regex. Tracked as [issue #26](https://github.com/DocForum/docforum-core/issues/26) (Trivial, 100 pts).
+- [x] Nightly/manual testnet payment verification with fresh Friendbot-funded identities and deduplicated failure tracking. Tracked as [issue #91](https://github.com/DocForum/docforum-core/issues/91).
 - [ ] Non-custodial patient payments (v2): patients fund escrows from their own Stellar wallet, custodial kept as fallback. **Proposed** in `docs/adr/0005-non-custodial-patient-payments.md`; implementation issues to be opened once it's Accepted.
 - [ ] Upgrade to `@docforum/escrow-sdk` 0.3.0 and the current testnet contract (`refund_after`, balance-delta check, `getEscrow`) — step 1 of ADR 0005's plan, but worth doing regardless. Tracked as [issue #32](https://github.com/DocForum/docforum-core/issues/32) (Medium, 150 pts); depends on `docforum-escrow` publishing `sdk-v0.3.0`.
 - [ ] Doc reconciliation: PRD §8.1.6 (facility fulfillment integrity) — not marked resolved; this phase proves the payment *mechanism* works, not the fulfillment-integrity question, which still needs real `FulfillmentRecord`/facility-matching (Phase 5) to actually address.
@@ -405,3 +406,8 @@ for status and issue links.
   observed state, so a racing second action gets 409 instead of silently
   overwriting). Rollback coupling proven with a real injected DB failure,
   not a mock. Phase 3/4 wiring explicitly left pending.
+- 2026-10-19 — Implemented issue #91's scheduled and manually dispatched
+  testnet payment workflow. Every run generates new test identities,
+  retries Friendbot with bounded jitter, reports infrastructure/contract
+  reset failures separately from assertion failures, and updates one open
+  tracking issue instead of posting repeated reports.

backend/scripts/gen-platform-identities.mjs

@@ -1,9 +1,6 @@
-// One-off local-dev helper: generates two fresh testnet Stellar keypairs
-// (platform payer + releaser, see docs/adr/0004-custodial-payments-v1.md),
-// funds them via Friendbot, and appends the secrets DIRECTLY to .env —
-// never printed to stdout/logs. Only the public keys (safe to display)
-// are printed. Delete this script's output and rerun for a fresh pair;
-// never commit the resulting .env.
+// Generates fresh testnet payer/releaser identities. Locally, credentials
+// are appended directly to .env; in Actions, they go to GITHUB_ENV and are
+// masked. Secret values are never printed. See ADR 0004.
 import { Keypair } from "@stellar/stellar-sdk";
 import fs from "node:fs";
 import path from "node:path";
@@ -12,9 +9,28 @@ import { fileURLToPath } from "node:url";
 const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
 const ENV_PATH = path.join(ROOT, ".env");
 
+const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
 async function fund(publicKey) {
-  const res = await fetch(`https://friendbot.stellar.org?addr=${encodeURIComponent(publicKey)}`);
-  if (!res.ok) throw new Error(`Friendbot funding failed for ${publicKey}: ${res.status} ${await res.text()}`);
+  const maxAttempts = 5;
+  let lastFailure;
+  let attempts = 0;
+  for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+    attempts = attempt + 1;
+    try {
+      const res = await fetch(`https://friendbot.stellar.org?addr=${encodeURIComponent(publicKey)}`);
+      if (res.ok) return;
+      lastFailure = `HTTP ${res.status}`;
+      if (res.status !== 429 && res.status < 500) break;
+    } catch (error) {
+      lastFailure = error instanceof Error ? error.name : 'network error';
+    }
+    if (attempt < maxAttempts - 1) {
+      const backoff = Math.min(1000 * 2 ** attempt, 16_000);
+      await wait(backoff + Math.floor(Math.random() * 500));
+    }
+  }
+  throw new Error(`Friendbot funding failed for ${publicKey} after ${attempts} attempt(s) (${lastFailure}).`);
 }
 
 const payer = Keypair.random();
@@ -23,16 +39,24 @@ const releaser = Keypair.random();
 await fund(payer.publicKey());
 await fund(releaser.publicKey());
 
-const lines = [
-  "",
-  "# Generated by scripts/gen-platform-identities.mjs — testnet only, never commit.",
+const envLines = [
   `PLATFORM_PAYER_SECRET=${payer.secret()}`,
   `PLATFORM_RELEASER_SECRET=${releaser.secret()}`,
-  "",
 ].join("\n");
 
-fs.appendFileSync(ENV_PATH, lines);
+if (process.env.GITHUB_ENV) {
+  process.stdout.write(`::add-mask::${payer.secret()}\n::add-mask::${releaser.secret()}\n`);
+  fs.appendFileSync(process.env.GITHUB_ENV, `${envLines}\n`);
+  console.log("Funded fresh testnet identities for this workflow run.");
+} else {
+  fs.appendFileSync(
+    ENV_PATH,
+    ["", "# Generated by scripts/gen-platform-identities.mjs — testnet only, never commit.", envLines, ""].join("\n"),
+  );
+  console.log("Funded and wrote secrets to .env (not printed here). Public keys:");
+}
 
-console.log("Funded and wrote secrets to .env (not printed here). Public keys:");
-console.log("  payer:   ", payer.publicKey());
-console.log("  releaser:", releaser.publicKey());
+if (!process.env.GITHUB_ENV) {
+    console.log("  payer:   ", payer.publicKey());
+    console.log("  releaser:", releaser.publicKey());
+}

backend/tests/testnet-integration/payments.test.mts

@@ -41,6 +41,7 @@ const pg = new EmbeddedPostgres({
   port: PORT,
   persistent: false,
 });
+
 const databaseUrl = `postgresql://docforum:docforum@localhost:${PORT}/docforum_payments_test`;
 
 // Phase 8, issue #43: fund/release/refund now take the audit actor. This
@@ -50,9 +51,22 @@ const TEST_ACTOR = { userId: null, role: null, requestId: `testnet-${runId}` };
 
 async function fundedKeypair(): Promise<Keypair> {
   const kp = Keypair.random();
-  const res = await fetch(`https://friendbot.stellar.org?addr=${encodeURIComponent(kp.publicKey())}`);
-  if (!res.ok) throw new Error(`Friendbot funding failed: ${res.status} ${await res.text()}`);
-  return kp;
+  let lastFailure: string | undefined;
+  for (let attempt = 0; attempt < 5; attempt += 1) {
+    try {
+      const res = await fetch(`https://friendbot.stellar.org?addr=${encodeURIComponent(kp.publicKey())}`);
+      if (res.ok) return kp;
+      lastFailure = `HTTP ${res.status}`;
+      if (res.status !== 429 && res.status < 500) break;
+    } catch (error) {
+      lastFailure = error instanceof Error ? error.name : 'network error';
+    }
+    if (attempt < 4) {
+      const backoff = Math.min(1000 * 2 ** attempt, 16_000);
+      await new Promise((resolve) => setTimeout(resolve, backoff + Math.floor(Math.random() * 500)));
+    }
+  }
+  throw new Error(`Friendbot funding failed after 5 attempts (${lastFailure}).`);
 }
 
 describe('payments module against live testnet', () => {
@@ -106,6 +120,7 @@ describe('payments module against live testnet', () => {
   afterAll(async () => {
     await prisma?.$disconnect();
     await pg.stop();
+    fs.rmSync(dataDir, { recursive: true, force: true });
   });
 
   it('funds and releases a payment intent end-to-end on testnet', async () => {