Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@ DISCORD_WEBHOOK_URL=
DISCORD_NOTIFICATION_MAX_RETRIES=3
DISCORD_NOTIFICATION_RETRY_BASE_DELAY_MS=1000

# Platform balance monitoring (#730)
# Stellar account address (G...) to monitor for low balance warnings
STELLAR_PLATFORM_ADDRESS=
# Balance thresholds (XLM) for triggering alerts. Default: 1 XLM (warning), 0.5 XLM (critical)
STELLAR_LOW_BALANCE_THRESHOLD=1
STELLAR_CRITICAL_BALANCE_THRESHOLD=0.5

# Runtime environment: development | production | test. Controls CORS wildcard
# rules, Sentry activation, migration-on-startup, and more.
NODE_ENV=development
Expand Down
40 changes: 35 additions & 5 deletions backend/src/admin/admin.controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,17 @@ import {
GigAnalyticsDto,
PaginatedEscrowAnalyticsDto,
} from './admin.dto';
import { DiscordService } from '../webhook/discord.service';

@ApiTags('Admin')
@ApiBearerAuth('JWT-auth')
@UseGuards(JwtAuthGuard, AdminGuard)
@Controller('admin/analytics')
export class AdminController {
constructor(private readonly adminService: AdminService) {}
constructor(
private readonly adminService: AdminService,
private readonly discordService: DiscordService,
) {}

@Get('overview')
@ApiOperation({
Expand All @@ -39,10 +43,7 @@ export class AdminController {
@ApiResponse({ status: 200, type: EscrowAnalyticsDto })
@ApiResponse({ status: 401, description: 'Missing or invalid JWT' })
@ApiResponse({ status: 403, description: 'Caller is authenticated but not an admin address' })
getEscrowAnalytics(
@Query('limit') limit = 50,
@Query('cursor') cursor?: string,
) {
getEscrowAnalytics(@Query('limit') limit = 50, @Query('cursor') cursor?: string) {
return this.adminService.getEscrowAnalytics(limit, cursor);
}

Expand All @@ -63,4 +64,33 @@ export class AdminController {
getDisputeAnalytics() {
return this.adminService.getDisputeAnalytics();
}

@Get('discord/failed-notifications')
@ApiOperation({
summary: 'List permanently failed Discord dispute notifications',
description:
'Returns the dead-letter map of dispute notifications that failed permanently after ' +
'exhausting retries. Entries are automatically evicted after 7 days or when the map ' +
'exceeds 1000 entries. Admin-only endpoint for monitoring notification delivery failures.',
})
@ApiResponse({
status: 200,
description: 'Map of dispute IDs to failure records (error message and timestamp)',
schema: {
type: 'object',
additionalProperties: {
type: 'object',
properties: {
error: { type: 'string', description: 'Error message' },
timestamp: { type: 'string', format: 'date-time', description: 'ISO 8601 timestamp' },
},
},
},
})
@ApiResponse({ status: 401, description: 'Missing or invalid JWT' })
@ApiResponse({ status: 403, description: 'Caller is authenticated but not an admin address' })
getFailedDiscordNotifications() {
const failures = this.discordService.getFailedNotifications();
return Object.fromEntries(failures);
}
}
71 changes: 53 additions & 18 deletions backend/src/config/env.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -155,12 +155,22 @@ const EnvSchema = z
DB_POOL_MAX: optionalPositiveInt(),
DB_POOL_IDLE_TIMEOUT_MS: optionalPositiveInt(),
DB_POOL_CONNECTION_TIMEOUT_MS: optionalPositiveInt(),
DB_RETRY_ATTEMPTS: z.coerce.number().int().positive().default(5).describe(
'Maximum number of connection retry attempts during pool initialization. Default 5.',
),
DB_RETRY_BACKOFF_MS: z.coerce.number().int().positive().default(1000).describe(
'Base delay in milliseconds for exponential backoff between connection retries. Default 1000ms.',
),
DB_RETRY_ATTEMPTS: z.coerce
.number()
.int()
.positive()
.default(5)
.describe(
'Maximum number of connection retry attempts during pool initialization. Default 5.',
),
DB_RETRY_BACKOFF_MS: z.coerce
.number()
.int()
.positive()
.default(1000)
.describe(
'Base delay in milliseconds for exponential backoff between connection retries. Default 1000ms.',
),

SWAGGER_ENABLED: optionalBool().describe(
'Serve Swagger UI and the OpenAPI JSON; defaults to true outside production, false in production',
Expand Down Expand Up @@ -213,6 +223,31 @@ const EnvSchema = z
.default(1000)
.describe('Base delay (ms) for exponential backoff in Discord retry logic. Default 1s.'),

// Platform Balance Monitoring (#730)
STELLAR_PLATFORM_ADDRESS: z
.preprocess(
blankToUndefined,
z
.string()
.regex(/^G[A-Z2-7]{55}$/)
.optional(),
)
.describe('Platform Stellar account address to monitor balance (G... format)'),
STELLAR_LOW_BALANCE_THRESHOLD: z.coerce
.number()
.positive()
.default(1)
.describe(
'Balance threshold (XLM) below which a low-balance warning is triggered. Default 1 XLM.',
),
STELLAR_CRITICAL_BALANCE_THRESHOLD: z.coerce
.number()
.positive()
.default(0.5)
.describe(
'Balance threshold (XLM) below which a critical alert is triggered. Default 0.5 XLM.',
),

// Rate Limiting Configuration
RATE_LIMIT_ABUSE_WINDOW_SECONDS: z.coerce.number().int().positive().default(300),
RATE_LIMIT_ABUSE_THRESHOLD: z.coerce.number().int().positive().default(5),
Expand Down Expand Up @@ -279,18 +314,18 @@ const EnvSchema = z
IPFS_PROVIDER_TIMEOUT_MS: z.coerce.number().int().positive().default(30000),

// S3-compatible Storage Configuration (avatar uploads)
S3_BUCKET: z.preprocess(blankToUndefined, z.string().optional()).describe(
'S3 bucket name for avatar uploads. Required in production.',
),
S3_REGION: z.preprocess(blankToUndefined, z.string().optional()).describe(
'AWS region (or equivalent) for the S3 bucket.',
),
S3_ACCESS_KEY_ID: z.preprocess(blankToUndefined, z.string().optional()).describe(
'AWS / S3-compatible access key ID.',
),
S3_SECRET_ACCESS_KEY: z.preprocess(blankToUndefined, z.string().optional()).describe(
'AWS / S3-compatible secret access key.',
),
S3_BUCKET: z
.preprocess(blankToUndefined, z.string().optional())
.describe('S3 bucket name for avatar uploads. Required in production.'),
S3_REGION: z
.preprocess(blankToUndefined, z.string().optional())
.describe('AWS region (or equivalent) for the S3 bucket.'),
S3_ACCESS_KEY_ID: z
.preprocess(blankToUndefined, z.string().optional())
.describe('AWS / S3-compatible access key ID.'),
S3_SECRET_ACCESS_KEY: z
.preprocess(blankToUndefined, z.string().optional())
.describe('AWS / S3-compatible secret access key.'),
S3_ENDPOINT: z
.preprocess(blankToUndefined, z.string().url().optional())
.describe(
Expand Down
13 changes: 11 additions & 2 deletions backend/src/event-ingestion/event-ingestion.controller.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
import { Controller, Post, Get, Body, Param, HttpCode, HttpStatus, UseGuards } from '@nestjs/common';
import {
Controller,
Post,
Get,
Body,
Param,
HttpCode,
HttpStatus,
UseGuards,
} from '@nestjs/common';
import { ApiTags, ApiOperation, ApiResponse, ApiBearerAuth } from '@nestjs/swagger';
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { JwtAuthGuard } from '../auth/auth.guard';
import { AdminGuard } from '../admin/admin.guard';
import { EventIngestionService } from './event-ingestion.service';
import { StartPollingDto, IngestLedgerDto, HandleReorgDto } from './dto/event-ingestion.dto';
Expand Down
17 changes: 12 additions & 5 deletions backend/src/stellar/platform-balance.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { SanitizedLogger } from '../common/logging/sanitized-logger';
import { Cron, CronExpression } from '@nestjs/schedule';
import { StellarService } from './stellar.service';
import { WebhookService } from '../webhook/webhook.service';
import { config } from '../config/env.config';

@Injectable()
export class PlatformBalanceService {
Expand All @@ -17,7 +18,7 @@ export class PlatformBalanceService {

@Cron(CronExpression.EVERY_10_MINUTES)
async checkPlatformBalance(): Promise<void> {
const platformAddress = process.env.STELLAR_PLATFORM_ADDRESS;
const platformAddress = config.STELLAR_PLATFORM_ADDRESS;
if (!platformAddress) {
this.logger.debug('STELLAR_PLATFORM_ADDRESS not configured, skipping balance check');
return;
Expand All @@ -27,10 +28,16 @@ export class PlatformBalanceService {
const balanceXlm = await this.stellarService.getBalance(platformAddress);
const balance = parseFloat(balanceXlm);

const lowBalanceThreshold = parseFloat(process.env.STELLAR_LOW_BALANCE_THRESHOLD || '1');
const criticalBalanceThreshold = parseFloat(
process.env.STELLAR_CRITICAL_BALANCE_THRESHOLD || '0.5',
);
const lowBalanceThreshold = config.STELLAR_LOW_BALANCE_THRESHOLD;
const criticalBalanceThreshold = config.STELLAR_CRITICAL_BALANCE_THRESHOLD;

// Sanity check: if thresholds are invalid (NaN), log error and skip alerting
if (isNaN(balance) || isNaN(lowBalanceThreshold) || isNaN(criticalBalanceThreshold)) {
this.logger.error(
`Invalid balance or threshold values: balance=${balance}, low=${lowBalanceThreshold}, critical=${criticalBalanceThreshold}`,
);
return;
}

if (balance <= criticalBalanceThreshold) {
this.triggerAlert(
Expand Down
1 change: 1 addition & 0 deletions backend/src/user-profile/user-profile.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import { UserType, UserStatus } from './user-profile.entity';
import { CreateUserProfileDto, UpdateUserProfileDto, RateUserDto } from './user-profile.dto';
import { randomUUID } from 'crypto';
import { config } from '../config/env.config';
import { parseAmount, normalizeAmount } from '../common/amount';

export interface UserProfile {
id: string;
Expand Down
66 changes: 61 additions & 5 deletions backend/src/webhook/discord.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,12 @@ interface DiscordWebhookPayload {
export class DiscordService {
private readonly logger = new SanitizedLogger(DiscordService.name);
private readonly webhookUrl: string;
/** Permanent failures that should not be retried. */
/** Permanent failures that should not be retried. Maximum 1000 entries with TTL-based eviction. */
private readonly failedNotifications = new Map<string, { error: string; timestamp: string }>();
/** Maximum number of failed notifications to retain. */
private readonly MAX_FAILED_NOTIFICATIONS = 1000;
/** TTL for failed notifications in milliseconds (7 days). */
private readonly FAILED_NOTIFICATION_TTL_MS = 7 * 24 * 60 * 60 * 1000;

constructor(@Optional() private readonly metrics?: MetricsService) {
this.webhookUrl = config.DISCORD_WEBHOOK_URL || '';
Expand Down Expand Up @@ -103,11 +107,9 @@ export class DiscordService {
this.metrics?.increment('discord_notifications_sent_total');
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
this.failedNotifications.set(disputeId, {
error: message,
timestamp: new Date().toISOString(),
});
this.recordFailedNotification(disputeId, message);
this.metrics?.increment('discord_notifications_failed_total', { reason: 'permanent' });
this.metrics?.increment('discord_notifications_dead_lettered_total');
throw error; // Re-throw so caller knows delivery ultimately failed
}
}
Expand Down Expand Up @@ -172,9 +174,63 @@ export class DiscordService {

/** Returns the current dead-letter list of permanently failed notifications. */
getFailedNotifications(): Map<string, { error: string; timestamp: string }> {
this.evictExpiredNotifications();
return this.failedNotifications;
}

/**
* Record a failed notification with size cap and TTL enforcement.
* Evicts oldest entries when MAX_FAILED_NOTIFICATIONS is exceeded.
*/
private recordFailedNotification(disputeId: string, error: string): void {
// Add new failure
this.failedNotifications.set(disputeId, {
error,
timestamp: new Date().toISOString(),
});

// Enforce size cap by evicting oldest entries
if (this.failedNotifications.size > this.MAX_FAILED_NOTIFICATIONS) {
const entries = Array.from(this.failedNotifications.entries()).sort(
(a, b) => new Date(a[1].timestamp).getTime() - new Date(b[1].timestamp).getTime(),
);

const toDelete = entries.slice(
0,
this.failedNotifications.size - this.MAX_FAILED_NOTIFICATIONS,
);
toDelete.forEach(([key]) => this.failedNotifications.delete(key));

this.logger.debug(
`Evicted ${toDelete.length} oldest failed notifications to maintain size cap of ${this.MAX_FAILED_NOTIFICATIONS}`,
);
}
}

/**
* Remove expired failed notifications based on TTL.
* Called automatically when getFailedNotifications() is invoked.
*/
private evictExpiredNotifications(): void {
const now = Date.now();
const expired: string[] = [];

for (const [disputeId, record] of this.failedNotifications.entries()) {
const age = now - new Date(record.timestamp).getTime();
if (age > this.FAILED_NOTIFICATION_TTL_MS) {
expired.push(disputeId);
}
}

expired.forEach(id => this.failedNotifications.delete(id));

if (expired.length > 0) {
this.logger.debug(
`Evicted ${expired.length} expired failed notifications (TTL: ${this.FAILED_NOTIFICATION_TTL_MS / (24 * 60 * 60 * 1000)} days)`,
);
}
}

private truncateAddress(address: string): string {
if (address.length <= 12) return address;
return `${address.slice(0, 6)}...${address.slice(-4)}`;
Expand Down