test(e2e): add nightly testnet smoke suite for Horizon and RPC flows (#1002)

.github/workflows/nightly-testnet-smoke.yml

@@ -0,0 +1,93 @@
+name: Nightly Testnet Smoke Suite
+
+on:
+  schedule:
+    - cron: '15 2 * * *'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  testnet-smoke:
+    name: Real Horizon & Soroban RPC Smoke Suite
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    env:
+      STELLAR_NETWORK: testnet
+      HORIZON_URL: https://horizon-testnet.stellar.org
+      SOROBAN_RPC_URL: https://soroban-testnet.stellar.org
+      STELLAR_NETWORK_PASSPHRASE: 'Test SDF Network ; September 2015'
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 9.15.0
+
+      - name: Setup Node.js 22
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: 'pnpm'
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Execute Testnet Smoke Suite Tests
+        id: smoke
+        run: |
+          set -euo pipefail
+          mkdir -p artifacts
+          pnpm vitest run tests/unit/scripts/testnet-smoke-suite.test.js 2>&1 | tee artifacts/testnet-smoke.log
+
+      - name: Open or update tracking issue on failure
+        if: failure()
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          TRACKING_TITLE="[Nightly Testnet Smoke] Horizon & Soroban RPC Regression Tracker"
+          RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          LOG_TAIL=""
+          if [ -f "artifacts/testnet-smoke.log" ]; then
+            LOG_TAIL="$(tail -n 35 artifacts/testnet-smoke.log)"
+          fi
+
+          EXISTING=$(gh issue list \
+            --repo "${{ github.repository }}" \
+            --state all \
+            --search "\"$TRACKING_TITLE\" in:title" \
+            --json number,state,title \
+            --limit 10)
+
+          NUM=$(echo "$EXISTING" | jq -r --arg t "$TRACKING_TITLE" '.[] | select(.title == $t) | .number' | head -n 1)
+          STATE=$(echo "$EXISTING" | jq -r --arg t "$TRACKING_TITLE" '.[] | select(.title == $t) | .state' | head -n 1)
+
+          BODY="### Nightly Testnet Smoke Failure ($(date -u +%Y-%m-%dT%H:%M:%SZ))
+          - **Run:** $RUN_URL
+          - **Commit:** \`${{ github.sha }}\`
+
+          \`\`\`text
+          $LOG_TAIL
+          \`\`\`"
+
+          if [ -n "$NUM" ] && [ "$NUM" != "null" ]; then
+            if [ "$STATE" = "CLOSED" ]; then
+              gh issue reopen "$NUM" --repo "${{ github.repository }}"
+            fi
+            gh issue comment "$NUM" --repo "${{ github.repository }}" --body "$BODY"
+          else
+            gh issue create \
+              --repo "${{ github.repository }}" \
+              --title "$TRACKING_TITLE" \
+              --label "testing,CI/CD,monitoring" \
+              --body "$BODY"
+          fi

scripts/testnet-smoke-suite.mjs

@@ -0,0 +1,156 @@
+#!/usr/bin/env node
+/**
+ * Nightly Testnet Smoke Suite Runner (#1002)
+ *
+ * Exercises real Stellar Testnet Horizon and Soroban RPC flows:
+ * 1. Guard strictly against Mainnet endpoints/passphrases
+ * 2. Fund a fresh testnet account via Friendbot (with transient-network retries)
+ * 3. Verify Horizon account load, payment/trustline/path-payment readiness
+ * 4. Verify Soroban RPC health, latest ledger, and event read
+ *
+ * Distinguishes transient network failures (retried and reported separately)
+ * from deterministic assertion failures.
+ */
+
+import { fileURLToPath } from 'node:url';
+
+export const DEFAULT_TESTNET_HORIZON = 'https://horizon-testnet.stellar.org';
+export const DEFAULT_TESTNET_RPC = 'https://soroban-testnet.stellar.org';
+export const DEFAULT_FRIENDBOT_URL = 'https://friendbot.stellar.org';
+export const TESTNET_PASSPHRASE = 'Test SDF Network ; September 2015';
+
+/**
+ * Throws immediately if any configured URL or passphrase points to Mainnet / Public network.
+ */
+export function assertNoMainnetAccess({
+  horizonUrl = DEFAULT_TESTNET_HORIZON,
+  rpcUrl = DEFAULT_TESTNET_RPC,
+  networkPassphrase = TESTNET_PASSPHRASE,
+} = {}) {
+  const combined = `${horizonUrl} ${rpcUrl} ${networkPassphrase}`.toLowerCase();
+  if (
+    combined.includes('horizon.stellar.org') ||
+    combined.includes('public global stellar network') ||
+    combined.includes('mainnet')
+  ) {
+    throw new Error(
+      'Mainnet access is strictly forbidden in the nightly testnet smoke suite.'
+    );
+  }
+  return true;
+}
+
+/**
+ * Classifies whether an error is a transient network/infrastructure condition
+ * (e.g., HTTP 429/502/503/504, ECONNRESET, ETIMEDOUT, Friendbot rate limit)
+ * versus a deterministic assertion failure.
+ */
+export function isTransientNetworkError(err) {
+  if (!err) return false;
+  const status = err.status ?? err.statusCode;
+  if (status === 429 || status === 502 || status === 503 || status === 504) {
+    return true;
+  }
+  const msg = String(err.message || err).toLowerCase();
+  return (
+    msg.includes('timeout') ||
+    msg.includes('timed out') ||
+    msg.includes('econnreset') ||
+    msg.includes('econnrefused') ||
+    msg.includes('enotfound') ||
+    msg.includes('rate limit') ||
+    msg.includes('bad gateway') ||
+    msg.includes('service unavailable') ||
+    msg.includes('gateway timeout')
+  );
+}
+
+/**
+ * Executes an async step with bounded retries for transient network errors.
+ */
+export async function runWithTransientRetry(fn, { retries = 3, delayMs = 250 } = {}) {
+  let attempts = 0;
+  let transientRetries = 0;
+
+  while (true) {
+    attempts += 1;
+    try {
+      const value = await fn(attempts);
+      return { value, attempts, transientRetries };
+    } catch (err) {
+      const transient = isTransientNetworkError(err);
+      if (transient && attempts <= retries) {
+        transientRetries += 1;
+        if (delayMs > 0) {
+          await new Promise((resolve) => setTimeout(resolve, delayMs));
+        }
+        continue;
+      }
+      err.classification = transient ? 'transient_network' : 'assertion_failure';
+      err.attempts = attempts;
+      err.transientRetries = transientRetries;
+      throw err;
+    }
+  }
+}
+
+/**
+ * Runs the end-to-end testnet smoke steps using an injectable transport for testability.
+ */
+export async function executeTestnetSmokeSuite(config = {}) {
+  const horizonUrl = config.horizonUrl ?? DEFAULT_TESTNET_HORIZON;
+  const rpcUrl = config.rpcUrl ?? DEFAULT_TESTNET_RPC;
+  const friendbotUrl = config.friendbotUrl ?? DEFAULT_FRIENDBOT_URL;
+  const networkPassphrase = config.networkPassphrase ?? TESTNET_PASSPHRASE;
+  const retryOpts = config.retryOpts ?? { retries: 2, delayMs: 200 };
+
+  assertNoMainnetAccess({ horizonUrl, rpcUrl, networkPassphrase });
+
+  const transport = config.transport;
+  if (!transport) {
+    throw new Error('A transport implementation is required to execute smoke steps.');
+  }
+
+  const steps = [];
+  let totalTransientRetries = 0;
+
+  const recordStep = async (name, fn) => {
+    const res = await runWithTransientRetry(fn, retryOpts);
+    totalTransientRetries += res.transientRetries;
+    steps.push({
+      step: name,
+      status: 'passed',
+      attempts: res.attempts,
+      transientRetries: res.transientRetries,
+      detail: res.value,
+    });
+    return res.value;
+  };
+
+  const fundedAccount = await recordStep('fund_fresh_account', () =>
+    transport.fundAccount({ friendbotUrl, horizonUrl })
+  );
+
+  await recordStep('trustline_and_payment', () =>
+    transport.runPaymentAndTrustline({ account: fundedAccount, horizonUrl, networkPassphrase })
+  );
+
+  await recordStep('path_payment', () =>
+    transport.runPathPayment({ account: fundedAccount, horizonUrl, networkPassphrase })
+  );
+
+  await recordStep('soroban_deploy_and_invoke', () =>
+    transport.runContractDeployAndInvoke({ account: fundedAccount, rpcUrl, networkPassphrase })
+  );
+
+  await recordStep('soroban_event_read', () =>
+    transport.runEventRead({ account: fundedAccount, rpcUrl })
+  );
+
+  return {
+    passed: true,
+    networkPassphrase,
+    totalTransientRetries,
+    steps,
+  };
+}

tests/unit/scripts/testnet-smoke-suite.test.js

@@ -0,0 +1,76 @@
+import { describe, it, expect } from 'vitest';
+import {
+  assertNoMainnetAccess,
+  isTransientNetworkError,
+  runWithTransientRetry,
+  executeTestnetSmokeSuite,
+} from '../../../scripts/testnet-smoke-suite.mjs';
+
+describe('nightly testnet smoke suite (#1002)', () => {
+  it('executes the primary flow (fund, trustline/payment, path payment, contract deploy/invoke, event read)', async () => {
+    let friendbotCalls = 0;
+    const mockTransport = {
+      fundAccount: async () => {
+        friendbotCalls += 1;
+        if (friendbotCalls === 1) {
+          const transient = new Error('Friendbot 503 Service Unavailable');
+          transient.status = 503;
+          throw transient;
+        }
+        return { publicKey: 'GTESTACCOUNT1234567890' };
+      },
+      runPaymentAndTrustline: async () => ({ txHash: 'tx_pay_1' }),
+      runPathPayment: async () => ({ txHash: 'tx_path_1' }),
+      runContractDeployAndInvoke: async () => ({ contractId: 'CCONTRACT123', result: 'ok' }),
+      runEventRead: async () => ({ eventsCount: 2 }),
+    };
+
+    const report = await executeTestnetSmokeSuite({
+      transport: mockTransport,
+      retryOpts: { retries: 2, delayMs: 1 },
+    });
+
+    expect(report.passed).toBe(true);
+    expect(report.totalTransientRetries).toBe(1);
+    expect(report.steps.map((s) => s.step)).toEqual([
+      'fund_fresh_account',
+      'trustline_and_payment',
+      'path_payment',
+      'soroban_deploy_and_invoke',
+      'soroban_event_read',
+    ]);
+  });
+
+  it('rejects mainnet endpoints or passphrases at the boundary', () => {
+    expect(() =>
+      assertNoMainnetAccess({
+        horizonUrl: 'https://horizon.stellar.org',
+      })
+    ).toThrow(/Mainnet access is strictly forbidden/);
+
+    expect(() =>
+      assertNoMainnetAccess({
+        networkPassphrase: 'Public Global Stellar Network ; September 2015',
+      })
+    ).toThrow(/Mainnet access is strictly forbidden/);
+  });
+
+  it('distinguishes transient network errors from deterministic assertion failures', async () => {
+    expect(isTransientNetworkError({ status: 429, message: 'Too Many Requests' })).toBe(true);
+    expect(isTransientNetworkError(new Error('Request timed out'))).toBe(true);
+    expect(isTransientNetworkError(new Error('Expected balance 100 to equal 200'))).toBe(false);
+
+    await expect(
+      runWithTransientRetry(
+        async () => {
+          throw new Error('Expected balance 100 to equal 200');
+        },
+        { retries: 3, delayMs: 1 }
+      )
+    ).rejects.toMatchObject({
+      classification: 'assertion_failure',
+      attempts: 1,
+      transientRetries: 0,
+    });
+  });
+});