From 3e74335ed43ff0a723273b990b277808fb1fa9ee Mon Sep 17 00:00:00 2001 From: Aleksandr Minkin Date: Tue, 7 Apr 2026 21:03:06 +0300 Subject: [PATCH] Use official Fedresurs API for stable collection --- src/config.ts | 10 ++ src/sources/fedresurs/fedresurs-client.ts | 134 ++++++++++++++++++ src/sources/fedresurs/fedresurs-mapper.ts | 21 ++- .../fedresurs/fedresurs-parser.spec.ts | 38 +++++ src/sources/fedresurs/fedresurs-parser.ts | 82 ++++++++++- src/sources/fedresurs/fedresurs-source.ts | 48 ++++++- src/sources/fedresurs/types.ts | 43 ++++++ src/sources/index.spec.ts | 8 ++ src/sources/index.ts | 8 ++ 9 files changed, 382 insertions(+), 10 deletions(-) diff --git a/src/config.ts b/src/config.ts index 2795605..8e1adef 100644 --- a/src/config.ts +++ b/src/config.ts @@ -85,6 +85,16 @@ const envSchema = z.object({ .string() .default("NPPWEB procurement monitor/1.0 (+https://example.local/nppweb)"), FEDRESURS_BASE_URL: z.string().url().default("https://bankrot.fedresurs.ru"), + FEDRESURS_API_URL: z.string().url().optional(), + FEDRESURS_API_LOGIN: z + .string() + .optional() + .transform((value) => value?.trim() || undefined), + FEDRESURS_API_PASSWORD: z + .string() + .optional() + .transform((value) => value?.trim() || undefined), + FEDRESURS_API_LOOKBACK_DAYS: z.coerce.number().int().positive().max(31).default(31), FEDRESURS_SEARCH_URL: z.string().url().default("https://bankrot.fedresurs.ru/Messages.aspx"), FEDRESURS_MAX_ITEMS: z.coerce.number().int().positive().max(20).default(5), FEDRESURS_USER_AGENT: z diff --git a/src/sources/fedresurs/fedresurs-client.ts b/src/sources/fedresurs/fedresurs-client.ts index c55e42c..9e2e8f1 100644 --- a/src/sources/fedresurs/fedresurs-client.ts +++ b/src/sources/fedresurs/fedresurs-client.ts @@ -2,10 +2,14 @@ import type { Logger } from "pino"; import { describeOutboundHttpError, fetch } from "../../http-client"; import { withRetries } from "../../utils/retry"; import { + parseFedresursApiMessage, parseFedresursDetailPage, parseFedresursSearchResults } from "./fedresurs-parser"; import type { + FedresursApiAuthResponse, + FedresursApiMessage, + FedresursApiSearchResponse, FedresursClientConfig, FedresursParsedMessage, FedresursSearchResultLink @@ -14,6 +18,45 @@ import type { export class FedresursClient { constructor(private readonly config: FedresursClientConfig) {} + hasOfficialApiConfig(): boolean { + return Boolean(this.config.apiUrl && this.config.apiLogin && this.config.apiPassword); + } + + async listRecentMessagesFromApi( + logger: Logger, + requestTimeoutMs: number + ): Promise> { + if (!this.hasOfficialApiConfig()) { + return []; + } + + const jwt = await this.authenticate(logger, requestTimeoutMs); + const now = new Date(); + const lookbackDays = Math.max(1, Math.min(this.config.apiLookbackDays ?? 31, 31)); + const from = new Date(now.getTime() - lookbackDays * 24 * 60 * 60 * 1000); + const apiResponse = await this.fetchJson( + this.buildApiUrl("v1/messages", { + DatePublishBegin: formatFedresursApiDate(from), + DatePublishEnd: formatFedresursApiDate(now), + IncludeContent: "true", + IncludeBankruptInfo: "true", + IsAnnulled: "false", + IsLocked: "false", + Limit: String(this.config.maxItems), + Offset: "0" + }), + logger, + requestTimeoutMs, + "messages api", + jwt + ); + + return (apiResponse.pageData ?? []).slice(0, this.config.maxItems).map((item) => ({ + rawJson: JSON.stringify(item, null, 2), + message: parseFedresursApiMessage(item, { baseUrl: this.config.baseUrl }) + })); + } + async listMessageLinks( logger: Logger, requestTimeoutMs: number @@ -38,6 +81,30 @@ export class FedresursClient { }; } + private async authenticate(logger: Logger, requestTimeoutMs: number): Promise { + const response = await this.fetchJson( + this.buildApiUrl("v1/auth"), + logger, + requestTimeoutMs, + "auth api", + undefined, + { + method: "POST", + body: JSON.stringify({ + login: this.config.apiLogin, + password: this.config.apiPassword + }) + } + ); + + const jwt = response.jwt?.trim(); + if (!jwt) { + throw new Error("Fedresurs auth api did not return JWT token"); + } + + return jwt; + } + private async fetchText( url: string, logger: Logger, @@ -83,4 +150,71 @@ export class FedresursClient { } ); } + + private async fetchJson( + url: string, + logger: Logger, + requestTimeoutMs: number, + resourceName: string, + jwt?: string, + init?: { method?: string; body?: string } + ): Promise { + return withRetries( + async () => { + let response; + try { + response = await fetch(url, { + method: init?.method ?? "GET", + body: init?.body, + signal: AbortSignal.timeout(requestTimeoutMs), + headers: { + accept: "application/json", + "content-type": init?.body ? "application/json" : undefined, + authorization: jwt ? `Bearer ${jwt}` : undefined, + "user-agent": this.config.userAgent + } + }); + } catch (error) { + throw describeOutboundHttpError(error, url); + } + + if (!response.ok) { + throw new Error(`Fedresurs ${resourceName} returned HTTP ${response.status}`); + } + + return (await response.json()) as T; + }, + 2, + 1000, + { + onRetry: ({ attempt, delayMs, error }) => { + logger.warn( + { + url, + resourceName, + attempt, + delayMs, + err: error + }, + "fedresurs request failed; retry scheduled" + ); + } + } + ); + } + + private buildApiUrl(path: string, query?: Record): string { + const baseUrl = this.config.apiUrl ?? ""; + const url = new URL(path.replace(/^\/+/, ""), `${baseUrl.replace(/\/+$/, "")}/`); + + for (const [key, value] of Object.entries(query ?? {})) { + url.searchParams.set(key, value); + } + + return url.toString(); + } +} + +function formatFedresursApiDate(value: Date): string { + return value.toISOString().replace(/\.\d{3}Z$/, ""); } diff --git a/src/sources/fedresurs/fedresurs-mapper.ts b/src/sources/fedresurs/fedresurs-mapper.ts index 5755dca..fed80df 100644 --- a/src/sources/fedresurs/fedresurs-mapper.ts +++ b/src/sources/fedresurs/fedresurs-mapper.ts @@ -3,9 +3,18 @@ import type { FedresursParsedMessage } from "./types"; export function mapFedresursMessageToCollectedRecord(input: { message: FedresursParsedMessage; - html: string; + rawDocument: string; + artifactKind?: "RAW_HTML" | "RAW_JSON"; + artifactContentType?: string; + artifactFileExtension?: "html" | "json"; }): CollectedRawRecord { - const { message, html } = input; + const { + message, + rawDocument, + artifactKind = "RAW_HTML", + artifactContentType = "text/html; charset=utf-8", + artifactFileExtension = "html" + } = input; return { url: message.externalUrl, @@ -34,10 +43,10 @@ export function mapFedresursMessageToCollectedRecord(input: { }, artifacts: [ { - kind: "RAW_HTML", - fileName: `fedresurs-${message.externalId}.html`, - contentType: "text/html; charset=utf-8", - body: html, + kind: artifactKind, + fileName: `fedresurs-${message.externalId}.${artifactFileExtension}`, + contentType: artifactContentType, + body: rawDocument, metadata: { externalId: message.externalId, externalUrl: message.externalUrl, diff --git a/src/sources/fedresurs/fedresurs-parser.spec.ts b/src/sources/fedresurs/fedresurs-parser.spec.ts index 319f889..0c54a99 100644 --- a/src/sources/fedresurs/fedresurs-parser.spec.ts +++ b/src/sources/fedresurs/fedresurs-parser.spec.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import { + parseFedresursApiMessage, parseFedresursDetailPage, parseFedresursSearchResults } from "./fedresurs-parser"; @@ -85,3 +86,40 @@ describe("parseFedresursDetailPage", () => { expect(message.checksum).toHaveLength(64); }); }); + +describe("parseFedresursApiMessage", () => { + it("maps official api payload into parser fields", () => { + const message = parseFedresursApiMessage( + { + guid: "2133d803-3d1f-7639-fed4-43e183df196c", + number: "12345678", + datePublish: "2026-04-07T12:30:45.000", + content: + "

В отношении должника введена процедура наблюдения.

Назначен временный управляющий.

", + type: "Observation", + bankruptInfo: { + guid: "05b663e5-2d69-4d20-9a53-f85d89fcb212", + type: "Company", + data: { + name: 'ООО "Стройресурс"', + inn: "7701234567", + ogrn: "1027700123456" + } + } + }, + { baseUrl: "https://bankrot.fedresurs.ru" } + ); + + expect(message.externalId).toBe("12345678"); + expect(message.externalUrl).toBe( + "https://bankrot.fedresurs.ru/sfactmessages/2133d803-3d1f-7639-fed4-43e183df196c" + ); + expect(message.messageType).toBe("Observation"); + expect(message.subjectName).toBe('ООО "Стройресурс"'); + expect(message.subjectInn).toBe("7701234567"); + expect(message.subjectOgrn).toBe("1027700123456"); + expect(message.publishedAt).toBe("2026-04-07T12:30:45+03:00"); + expect(message.description).toContain("процедура наблюдения"); + expect(message.checksum).toHaveLength(64); + }); +}); diff --git a/src/sources/fedresurs/fedresurs-parser.ts b/src/sources/fedresurs/fedresurs-parser.ts index cf22dbb..65937fb 100644 --- a/src/sources/fedresurs/fedresurs-parser.ts +++ b/src/sources/fedresurs/fedresurs-parser.ts @@ -1,7 +1,11 @@ import { createHash } from "node:crypto"; import { load, type CheerioAPI, type Cheerio } from "cheerio"; import type { AnyNode } from "domhandler"; -import type { FedresursParsedMessage, FedresursSearchResultLink } from "./types"; +import type { + FedresursApiMessage, + FedresursParsedMessage, + FedresursSearchResultLink +} from "./types"; const SEARCH_LINK_PATTERNS = ["/MessageWindow.aspx", "/TradeLotInfo.aspx"]; @@ -108,6 +112,37 @@ export function parseFedresursDetailPage(html: string, detailUrl: string): Fedre }; } +export function parseFedresursApiMessage( + message: FedresursApiMessage, + options: { baseUrl: string } +): FedresursParsedMessage { + const externalId = cleanText(message.number) || message.guid; + const externalUrl = resolveUrl(options.baseUrl, `/sfactmessages/${message.guid}`) ?? options.baseUrl; + const subject = message.bankruptInfo?.data; + const subjectName = buildParticipantName(subject); + const description = extractTextFromHtml(message.content); + const checksum = createHash("sha256").update(JSON.stringify(message)).digest("hex"); + + return { + externalId, + externalUrl, + sourcePageUrl: externalUrl, + sourceName: "fedresurs", + sourceType: "bankruptcy", + messageType: cleanText(message.type) || undefined, + subjectName, + subjectInn: normalizeIdentifier(subject?.inn ?? undefined), + subjectOgrn: normalizeIdentifier(subject?.ogrn ?? subject?.ogrnip ?? undefined), + publishedAt: normalizeApiDateTime(message.datePublish), + eventDate: normalizeApiDateTime(message.dateEvent), + title: + cleanText(message.type) || + (cleanText(message.number) ? `Сообщение №${cleanText(message.number)}` : undefined), + description, + checksum + }; +} + function findFirstValue($: CheerioAPI, labels: string[]): string | undefined { for (const label of labels) { const value = findValueByLabel($, label); @@ -289,3 +324,48 @@ function normalizeIdentifier(value: string | undefined): string | undefined { const normalized = value.replace(/[^\d]/g, ""); return normalized || undefined; } + +function normalizeApiDateTime(value: string | null | undefined): string | undefined { + const normalized = cleanText(value); + if (!normalized) { + return undefined; + } + + const withoutMs = normalized.replace(/\.\d+/, ""); + if (/([+-]\d{2}:\d{2}|Z)$/i.test(withoutMs)) { + return withoutMs.replace(/Z$/i, "+00:00"); + } + + return `${withoutMs}+03:00`; +} + +function buildParticipantName( + participant: { + name?: string | null; + firstName?: string | null; + lastName?: string | null; + middleName?: string | null; + } | null | undefined +): string | undefined { + const directName = cleanText(participant?.name); + if (directName) { + return directName; + } + + const fullName = [participant?.lastName, participant?.firstName, participant?.middleName] + .map((value) => cleanText(value)) + .filter(Boolean) + .join(" "); + + return fullName || undefined; +} + +function extractTextFromHtml(value: string | null | undefined): string | undefined { + const html = cleanText(value); + if (!html) { + return undefined; + } + + const text = cleanText(load(`
${html}
`).text()); + return text || html; +} diff --git a/src/sources/fedresurs/fedresurs-source.ts b/src/sources/fedresurs/fedresurs-source.ts index 220d26b..bbe5b38 100644 --- a/src/sources/fedresurs/fedresurs-source.ts +++ b/src/sources/fedresurs/fedresurs-source.ts @@ -7,6 +7,10 @@ type FedresursSourceConfig = { searchUrl: string; maxItems: number; userAgent: string; + apiUrl?: string; + apiLogin?: string; + apiPassword?: string; + apiLookbackDays?: number; }; export function createFedresursSourceAdapter(config: FedresursSourceConfig): SourceAdapter { @@ -19,15 +23,48 @@ export function createFedresursSourceAdapter(config: FedresursSourceConfig): Sou context.logger.info( { searchUrl: config.searchUrl, - maxItems: config.maxItems + maxItems: config.maxItems, + officialApiEnabled: client.hasOfficialApiConfig() }, "fedresurs fetch started" ); + if (client.hasOfficialApiConfig()) { + const messages = await client.listRecentMessagesFromApi( + context.logger, + context.requestTimeoutMs + ); + + if (messages.length === 0) { + throw new Error( + "Fedresurs API не вернул ни одного сообщения за расчётное окно. Проверьте учётные данные и лимиты доступа." + ); + } + + context.logger.info( + { + source: "fedresurs", + itemsCollected: messages.length, + collectionMode: "official-api" + }, + "fedresurs messages collected" + ); + + return messages.map(({ message, rawJson }) => + mapFedresursMessageToCollectedRecord({ + message, + rawDocument: rawJson, + artifactKind: "RAW_JSON", + artifactContentType: "application/json; charset=utf-8", + artifactFileExtension: "json" + }) + ); + } + const links = await client.listMessageLinks(context.logger, context.requestTimeoutMs); if (links.length === 0) { throw new Error( - "Федресурс не вернул ни одной ссылки на сообщения. Вероятно, изменилась выдача или доступ к площадке ограничен." + "Федресурс больше не отдаёт публичный HTML-список сообщений. Для стабильного сбора задайте FEDRESURS_API_URL, FEDRESURS_API_LOGIN и FEDRESURS_API_PASSWORD для официального REST API." ); } @@ -45,7 +82,12 @@ export function createFedresursSourceAdapter(config: FedresursSourceConfig): Sou context.requestTimeoutMs ); - records.push(mapFedresursMessageToCollectedRecord({ message, html })); + records.push( + mapFedresursMessageToCollectedRecord({ + message, + rawDocument: html + }) + ); } catch (error) { context.logger.warn( { diff --git a/src/sources/fedresurs/types.ts b/src/sources/fedresurs/types.ts index 764e55d..0eb8a1a 100644 --- a/src/sources/fedresurs/types.ts +++ b/src/sources/fedresurs/types.ts @@ -3,6 +3,10 @@ export type FedresursClientConfig = { searchUrl: string; maxItems: number; userAgent: string; + apiUrl?: string; + apiLogin?: string; + apiPassword?: string; + apiLookbackDays?: number; }; export type FedresursSearchResultLink = { @@ -30,3 +34,42 @@ export type FedresursParsedMessage = { courtName?: string; checksum: string; }; + +export type FedresursApiAuthResponse = { + jwt: string; +}; + +export type FedresursApiParticipant = { + name?: string | null; + inn?: string | null; + ogrn?: string | null; + ogrnip?: string | null; + firstName?: string | null; + lastName?: string | null; + middleName?: string | null; +}; + +export type FedresursApiBankruptInfo = { + guid: string; + type?: string | null; + data?: FedresursApiParticipant | null; +}; + +export type FedresursApiMessage = { + guid: string; + bankruptGuid?: string | null; + annulmentMessageGuid?: string | null; + number?: string | null; + datePublish?: string | null; + dateEvent?: string | null; + content?: string | null; + type?: string | null; + lockReason?: string | null; + hasViolation?: boolean | null; + bankruptInfo?: FedresursApiBankruptInfo | null; +}; + +export type FedresursApiSearchResponse = { + pageData?: FedresursApiMessage[] | null; + total?: number; +}; diff --git a/src/sources/index.spec.ts b/src/sources/index.spec.ts index a55cbf7..fcc7c7f 100644 --- a/src/sources/index.spec.ts +++ b/src/sources/index.spec.ts @@ -26,6 +26,10 @@ describe("resolveEnabledSources", () => { FNS_USER_AGENT: "NPPWEB/1.0 (+https://example.test)", FNS_DOWNLOAD_EXTRACT: true, FEDRESURS_BASE_URL: "https://bankrot.fedresurs.ru", + FEDRESURS_API_URL: "https://bank-publications-prod.fedresurs.ru", + FEDRESURS_API_LOGIN: undefined, + FEDRESURS_API_PASSWORD: undefined, + FEDRESURS_API_LOOKBACK_DAYS: 31, FEDRESURS_SEARCH_URL: "https://bankrot.fedresurs.ru/Messages.aspx", FEDRESURS_MAX_ITEMS: 5, FEDRESURS_USER_AGENT: "NPPWEB/1.0 (+https://example.test)", @@ -84,6 +88,10 @@ describe("resolveEnabledSources", () => { FNS_USER_AGENT: "NPPWEB/1.0 (+https://example.test)", FNS_DOWNLOAD_EXTRACT: true, FEDRESURS_BASE_URL: "https://bankrot.fedresurs.ru", + FEDRESURS_API_URL: "https://bank-publications-prod.fedresurs.ru", + FEDRESURS_API_LOGIN: undefined, + FEDRESURS_API_PASSWORD: undefined, + FEDRESURS_API_LOOKBACK_DAYS: 31, FEDRESURS_SEARCH_URL: "https://bankrot.fedresurs.ru/Messages.aspx", FEDRESURS_MAX_ITEMS: 5, FEDRESURS_USER_AGENT: "NPPWEB/1.0 (+https://example.test)", diff --git a/src/sources/index.ts b/src/sources/index.ts index ebcc0cb..a77e8ce 100644 --- a/src/sources/index.ts +++ b/src/sources/index.ts @@ -27,6 +27,10 @@ type SourceResolverConfig = Pick< | "EIS_USER_AGENT" | "ENABLED_SOURCES" | "FEDRESURS_BASE_URL" + | "FEDRESURS_API_LOGIN" + | "FEDRESURS_API_LOOKBACK_DAYS" + | "FEDRESURS_API_PASSWORD" + | "FEDRESURS_API_URL" | "FEDRESURS_MAX_ITEMS" | "FEDRESURS_SEARCH_URL" | "FEDRESURS_USER_AGENT" @@ -100,6 +104,10 @@ function createSourceFactories(config: SourceResolverConfig): Record createFedresursSourceAdapter({ baseUrl: config.FEDRESURS_BASE_URL, + apiUrl: config.FEDRESURS_API_URL, + apiLogin: config.FEDRESURS_API_LOGIN, + apiPassword: config.FEDRESURS_API_PASSWORD, + apiLookbackDays: config.FEDRESURS_API_LOOKBACK_DAYS, searchUrl: config.FEDRESURS_SEARCH_URL, maxItems: config.FEDRESURS_MAX_ITEMS, userAgent: config.FEDRESURS_USER_AGENT