feat: Add bounded retry policy to external provider calls

src/lib/http.ts

@@ -19,32 +19,108 @@
  */
 
 import { getRequestId } from "./requestContext";
+import { logger } from "../config/logger";
 
 /** The canonical X-Request-Id header name used throughout the application. */
 export const REQUEST_ID_HEADER = "x-request-id";
 
 /** The canonical X-Correlation-Id header name used throughout the application. */
 export const CORRELATION_ID_HEADER = "x-correlation-id";
 
+export interface FetchRetryOptions {
+  maxAttempts?: number;
+  baseBackoffMs?: number;
+  maxBackoffMs?: number;
+  timeoutMs?: number;
+}
+
+export class FetchError extends Error {
+  constructor(public readonly response: Response | undefined, message: string) {
+    super(message);
+    this.name = "FetchError";
+  }
+}
+
+/**
+ * Wraps `fetch` with a bounded retry policy, exponential backoff, and timeouts.
+ * Retries on 5xx status codes, 429 Too Many Requests, or network failures.
+ */
+export async function fetchWithRetry(
+  input: string | URL | globalThis.Request,
+  init?: RequestInit,
+  options: FetchRetryOptions = {}
+): Promise<Response> {
+  const maxAttempts = options.maxAttempts ?? 3;
+  const baseBackoff = options.baseBackoffMs ?? 500;
+  const maxBackoff = options.maxBackoffMs ?? 10000;
+  const timeoutMs = options.timeoutMs ?? 10000;
+
+  let attempt = 0;
+  let lastError: unknown;
+
+  while (attempt < maxAttempts) {
+    attempt++;
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+    const mergedSignal = init?.signal 
+      ? AbortSignal.any([init.signal, controller.signal]) 
+      : controller.signal;
+
+    try {
+      const response = await fetch(input, { ...init, signal: mergedSignal });
+
+      if (!response.ok && (response.status >= 500 || response.status === 429)) {
+        throw new FetchError(response, `HTTP ${response.status}`);
+      }
+
+      return response;
+    } catch (err: any) {
+      lastError = err;
+
+      // Do not retry on explicit aborts from the caller's own signal
+      if (err.name === 'AbortError' && init?.signal?.aborted) {
+        throw err;
+      }
+
+      if (attempt >= maxAttempts) {
+        break;
+      }
+
+      const backoff = Math.min(maxBackoff, baseBackoff * Math.pow(2, attempt - 1));
+
+      const urlStr = typeof input === "string" ? input : input instanceof URL ? input.toString() : (input as Request).url;
+      logger.warn({ url: urlStr, attempt, error: err.message }, "fetchWithRetry: external provider call failed, retrying");
+
+      await new Promise(resolve => setTimeout(resolve, backoff));
+    } finally {
+      clearTimeout(timeout);
+    }
+  }
+
+  throw lastError;
+}
+
 /**
  * Wraps `fetch` and injects an `X-Request-Id` header derived from the current
  * AsyncLocalStorage context.  All other arguments are forwarded unchanged.
  */
 export async function fetchWithRequestId(
   input: string | URL | globalThis.Request,
   init?: RequestInit,
+  options?: FetchRetryOptions
 ): Promise<Response> {
   const requestId = getRequestId();
 
   if (!requestId) {
     // No active request context — call fetch as-is.
-    return fetch(input, init);
+    return fetchWithRetry(input, init, options);
   }
 
   const headers = new Headers(init?.headers);
   headers.set(REQUEST_ID_HEADER, requestId);
 
-  return fetch(input, { ...init, headers });
+  return fetchWithRetry(input, { ...init, headers }, options);
 }
 
 /**
@@ -61,17 +137,18 @@ export async function fetchWithRequestId(
 export async function fetchWithCorrelationId(
   input: string | URL | globalThis.Request,
   init?: RequestInit,
+  options?: FetchRetryOptions
 ): Promise<Response> {
   // Import lazily to avoid circular dependency at module evaluation time.
   const { getCorrelationId } = await import("../middleware/correlation");
   const correlationId = getCorrelationId();
 
   if (!correlationId) {
-    return fetch(input, init);
+    return fetchWithRetry(input, init, options);
   }
 
   const headers = new Headers(init?.headers);
   headers.set(CORRELATION_ID_HEADER, correlationId);
 
-  return fetch(input, { ...init, headers });
+  return fetchWithRetry(input, { ...init, headers }, options);
 }

src/services/healthProbes.ts

@@ -78,7 +78,8 @@ async function probeHorizon(): Promise<ProbeResult> {
     const controller = new AbortController();
     const timeout = setTimeout(() => controller.abort(), 4500);
     try {
-      await fetch(env.HORIZON_URL, { signal: controller.signal });
+      const { fetchWithRetry } = await import("../lib/http");
+      await fetchWithRetry(env.HORIZON_URL, { signal: controller.signal });
       return { status: "ok", latencyMs: Date.now() - start };
     } finally {
       clearTimeout(timeout);

src/services/marketResolutionService.ts

@@ -229,7 +229,8 @@ export async function httpWebhookEmitter(
   const timer = setTimeout(() => controller.abort(), WEBHOOK_TIMEOUT_MS);
 
   try {
-    const response = await fetch(subscriber.url, {
+    const { fetchWithRetry } = await import("../lib/http");
+    const response = await fetchWithRetry(subscriber.url, {
       method: "POST",
       headers: {
         "Content-Type": "application/json",

src/services/settleConfirmerService.ts

@@ -245,10 +245,10 @@ export class SettleConfirmerService {
   }
 }
 
-// ─── Default Horizon client (production) ──────────────────────────────────────
+import { fetchWithRetry } from "../lib/http";
 
 /**
- * Production Horizon client that uses `fetch` to call the Horizon REST API.
+ * Production Horizon client that uses `fetchWithRetry` to call the Horizon REST API.
  */
 export class HttpHorizonClient implements HorizonClient {
   private readonly baseUrl: string;
@@ -259,7 +259,7 @@ export class HttpHorizonClient implements HorizonClient {
 
   async getTransaction(txHash: string): Promise<TransactionInfo> {
     const url = `${this.baseUrl}/transactions/${txHash}`;
-    const response = await fetch(url);
+    const response = await fetchWithRetry(url);
 
     if (!response.ok) {
       if (response.status === 404) {
@@ -283,7 +283,7 @@ export class HttpHorizonClient implements HorizonClient {
 
   async getCurrentLedger(): Promise<number> {
     const url = this.baseUrl;
-    const response = await fetch(url);
+    const response = await fetchWithRetry(url);
 
     if (!response.ok) {
       throw new Error(

src/workers/backupVerifier.ts

@@ -186,7 +186,8 @@ export class PgSmokeTestRunner implements SmokeTestRunner {
  */
 export class HttpSlackReporter implements SlackReporter {
   async send(webhookUrl: string, message: SlackMessage): Promise<void> {
-    const response = await fetch(webhookUrl, {
+    const { fetchWithRetry } = await import("../lib/http");
+    const response = await fetchWithRetry(webhookUrl, {
       method: "POST",
       headers: { "Content-Type": "application/json" },
       body: JSON.stringify(message),