diff --git a/src/lib/http.ts b/src/lib/http.ts index b0d0aeb3..d196c6b8 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -19,6 +19,7 @@ */ import { getRequestId } from "./requestContext"; +import { logger } from "../config/logger"; /** The canonical X-Request-Id header name used throughout the application. */ export const REQUEST_ID_HEADER = "x-request-id"; @@ -26,6 +27,80 @@ export const REQUEST_ID_HEADER = "x-request-id"; /** The canonical X-Correlation-Id header name used throughout the application. */ export const CORRELATION_ID_HEADER = "x-correlation-id"; +export interface FetchRetryOptions { + maxAttempts?: number; + baseBackoffMs?: number; + maxBackoffMs?: number; + timeoutMs?: number; +} + +export class FetchError extends Error { + constructor(public readonly response: Response | undefined, message: string) { + super(message); + this.name = "FetchError"; + } +} + +/** + * Wraps `fetch` with a bounded retry policy, exponential backoff, and timeouts. + * Retries on 5xx status codes, 429 Too Many Requests, or network failures. + */ +export async function fetchWithRetry( + input: string | URL | globalThis.Request, + init?: RequestInit, + options: FetchRetryOptions = {} +): Promise { + const maxAttempts = options.maxAttempts ?? 3; + const baseBackoff = options.baseBackoffMs ?? 500; + const maxBackoff = options.maxBackoffMs ?? 10000; + const timeoutMs = options.timeoutMs ?? 10000; + + let attempt = 0; + let lastError: unknown; + + while (attempt < maxAttempts) { + attempt++; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + + const mergedSignal = init?.signal + ? AbortSignal.any([init.signal, controller.signal]) + : controller.signal; + + try { + const response = await fetch(input, { ...init, signal: mergedSignal }); + + if (!response.ok && (response.status >= 500 || response.status === 429)) { + throw new FetchError(response, `HTTP ${response.status}`); + } + + return response; + } catch (err: any) { + lastError = err; + + // Do not retry on explicit aborts from the caller's own signal + if (err.name === 'AbortError' && init?.signal?.aborted) { + throw err; + } + + if (attempt >= maxAttempts) { + break; + } + + const backoff = Math.min(maxBackoff, baseBackoff * Math.pow(2, attempt - 1)); + + const urlStr = typeof input === "string" ? input : input instanceof URL ? input.toString() : (input as Request).url; + logger.warn({ url: urlStr, attempt, error: err.message }, "fetchWithRetry: external provider call failed, retrying"); + + await new Promise(resolve => setTimeout(resolve, backoff)); + } finally { + clearTimeout(timeout); + } + } + + throw lastError; +} + /** * Wraps `fetch` and injects an `X-Request-Id` header derived from the current * AsyncLocalStorage context. All other arguments are forwarded unchanged. @@ -33,18 +108,19 @@ export const CORRELATION_ID_HEADER = "x-correlation-id"; export async function fetchWithRequestId( input: string | URL | globalThis.Request, init?: RequestInit, + options?: FetchRetryOptions ): Promise { const requestId = getRequestId(); if (!requestId) { // No active request context — call fetch as-is. - return fetch(input, init); + return fetchWithRetry(input, init, options); } const headers = new Headers(init?.headers); headers.set(REQUEST_ID_HEADER, requestId); - return fetch(input, { ...init, headers }); + return fetchWithRetry(input, { ...init, headers }, options); } /** @@ -61,17 +137,18 @@ export async function fetchWithRequestId( export async function fetchWithCorrelationId( input: string | URL | globalThis.Request, init?: RequestInit, + options?: FetchRetryOptions ): Promise { // Import lazily to avoid circular dependency at module evaluation time. const { getCorrelationId } = await import("../middleware/correlation"); const correlationId = getCorrelationId(); if (!correlationId) { - return fetch(input, init); + return fetchWithRetry(input, init, options); } const headers = new Headers(init?.headers); headers.set(CORRELATION_ID_HEADER, correlationId); - return fetch(input, { ...init, headers }); + return fetchWithRetry(input, { ...init, headers }, options); } diff --git a/src/services/healthProbes.ts b/src/services/healthProbes.ts index 411b7de2..0d8ab33a 100644 --- a/src/services/healthProbes.ts +++ b/src/services/healthProbes.ts @@ -78,7 +78,8 @@ async function probeHorizon(): Promise { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 4500); try { - await fetch(env.HORIZON_URL, { signal: controller.signal }); + const { fetchWithRetry } = await import("../lib/http"); + await fetchWithRetry(env.HORIZON_URL, { signal: controller.signal }); return { status: "ok", latencyMs: Date.now() - start }; } finally { clearTimeout(timeout); diff --git a/src/services/marketResolutionService.ts b/src/services/marketResolutionService.ts index 50bfe3c9..f2825079 100644 --- a/src/services/marketResolutionService.ts +++ b/src/services/marketResolutionService.ts @@ -229,7 +229,8 @@ export async function httpWebhookEmitter( const timer = setTimeout(() => controller.abort(), WEBHOOK_TIMEOUT_MS); try { - const response = await fetch(subscriber.url, { + const { fetchWithRetry } = await import("../lib/http"); + const response = await fetchWithRetry(subscriber.url, { method: "POST", headers: { "Content-Type": "application/json", diff --git a/src/services/settleConfirmerService.ts b/src/services/settleConfirmerService.ts index c5d1c255..fc4376b5 100644 --- a/src/services/settleConfirmerService.ts +++ b/src/services/settleConfirmerService.ts @@ -245,10 +245,10 @@ export class SettleConfirmerService { } } -// ─── Default Horizon client (production) ────────────────────────────────────── +import { fetchWithRetry } from "../lib/http"; /** - * Production Horizon client that uses `fetch` to call the Horizon REST API. + * Production Horizon client that uses `fetchWithRetry` to call the Horizon REST API. */ export class HttpHorizonClient implements HorizonClient { private readonly baseUrl: string; @@ -259,7 +259,7 @@ export class HttpHorizonClient implements HorizonClient { async getTransaction(txHash: string): Promise { const url = `${this.baseUrl}/transactions/${txHash}`; - const response = await fetch(url); + const response = await fetchWithRetry(url); if (!response.ok) { if (response.status === 404) { @@ -283,7 +283,7 @@ export class HttpHorizonClient implements HorizonClient { async getCurrentLedger(): Promise { const url = this.baseUrl; - const response = await fetch(url); + const response = await fetchWithRetry(url); if (!response.ok) { throw new Error( diff --git a/src/workers/backupVerifier.ts b/src/workers/backupVerifier.ts index 581832af..ddcda1d9 100644 --- a/src/workers/backupVerifier.ts +++ b/src/workers/backupVerifier.ts @@ -186,7 +186,8 @@ export class PgSmokeTestRunner implements SmokeTestRunner { */ export class HttpSlackReporter implements SlackReporter { async send(webhookUrl: string, message: SlackMessage): Promise { - const response = await fetch(webhookUrl, { + const { fetchWithRetry } = await import("../lib/http"); + const response = await fetchWithRetry(webhookUrl, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(message),