[Quality-2] Make readiness probes reflect required dependency classes

README.md

@@ -45,7 +45,7 @@ Once running:
 |---|---|---|
 | `GET /health` | None | Liveness check — returns `{ "status": "ok" }` immediately. Use this to verify the process is up. |
 | `GET /healthz/dependencies` | None | Shallow dependency probe — Postgres, Soroban RPC, Horizon, webhook queue (Redis). Cached for 5 s. Returns 200/207/503. |
-| `GET /api/health/ready` | None | **Deep readiness check** — runs four parallel probes with 1-second timeouts each. Returns 200 when ready, 503 when unready. |
+| `GET /api/health/ready` | None | **Deep readiness check** — runs five required dependency probes with 1-second timeouts each. Returns 200 when ready, 503 when unready. |
 | `GET /api/indexer/health` | None | Indexer health — probes external dependencies (Postgres + Soroban RPC) and compares the persisted cursor against the chain tip. Returns `"ok"` / `"degraded"` / `"down"` with dependency statuses in `dependencies` and lag data in `data`. Always HTTP 200. Supports [ETag / conditional GET](#etag--conditional-get-caching). |
 | `GET /api/recommendations/health` | None | Recommendations subsystem health — probes the two runtime dependencies the recommendations pipeline relies on (Postgres + Soroban RPC). Returns 200 when all pass, 503 when any is down. Response shape mirrors `GET /api/predictions/health`. |
 
@@ -60,12 +60,13 @@ Once running:
     "db":         { "status": "pass", "durationMs": 4,  "message": "Database connection healthy" },
     "sorobanRpc": { "status": "pass", "durationMs": 18, "message": "Soroban RPC healthy" },
     "indexerLag": { "status": "pass", "durationMs": 22, "message": "Indexer lag healthy: 12 ≤ 200 ledgers" },
-    "queue":      { "status": "pass", "durationMs": 2,  "message": "Queue (Redis) healthy" }
+    "queue":      { "status": "pass", "durationMs": 2,  "message": "Queue (Redis) healthy" },
+    "horizon":    { "status": "pass", "durationMs": 6,  "message": "Horizon healthy" }
   }
 }
 ```
 
-- `status` is `"ready"` only when **all four** probes pass; otherwise `"unready"`.
+- `status` is `"ready"` only when **all five** required dependency probes pass; otherwise `"unready"`.
 - HTTP 200 → ready, HTTP 503 → unready.
 - Pass `x-correlation-id` header to correlate log entries with the request.
 - `READINESS_MAX_LAG_LEDGERS` (env, default `200`) controls the indexer lag threshold.

docs/health-ready.md

@@ -9,7 +9,7 @@ Unlike `/healthz/dependencies` (cached 5 s, returns 207 for degraded), this endp
 
 - Runs **every call**, uncached
 - Returns only `200` (ready) or `503` (unready) — no 207
-- Declares ready only when **all four** probes pass
+- Declares ready only when **all five required dependency probes** pass
 
 ## Response shape
 
@@ -22,7 +22,8 @@ Unlike `/healthz/dependencies` (cached 5 s, returns 207 for degraded), this endp
     "db":         { "status": "pass", "durationMs": 4,  "message": "Database connection healthy" },
     "sorobanRpc": { "status": "pass", "durationMs": 18, "message": "Soroban RPC healthy" },
     "indexerLag": { "status": "pass", "durationMs": 22, "message": "Indexer lag healthy: 12 ≤ 200 ledgers" },
-    "queue":      { "status": "pass", "durationMs": 2,  "message": "Queue (Redis) healthy" }
+    "queue":      { "status": "pass", "durationMs": 2,  "message": "Queue (Redis) healthy" },
+    "horizon":    { "status": "pass", "durationMs": 6,  "message": "Horizon healthy" }
   }
 }
 ```
@@ -61,12 +62,18 @@ Mainnet produces ~1 ledger/5 s, so 200 ledgers ≈ ~17 minutes of tolerated lag.
 ### `queue` — Redis / BullMQ
 Issues a Redis `PING` with a 1 s timeout. Fails if the response is not `PONG`.
 
+### `horizon` — Horizon REST API
+Requests the configured Horizon root endpoint with a 1 s timeout. Fails when
+Horizon is unreachable or returns a non-success HTTP status. Horizon is required
+by the settlement confirmer worker.
+
 ## Environment variables
 
 | Variable | Default | Description |
 |---|---|---|
 | `REDIS_URL` | `redis://localhost:6379` | BullMQ Redis connection |
 | `SOROBAN_RPC_URL` | — | Soroban RPC endpoint (required) |
+| `HORIZON_URL` | — | Horizon endpoint (required) |
 | `READINESS_MAX_LAG_LEDGERS` | `200` | Max tolerated indexer lag ledgers |
 
 ## Correlation IDs
@@ -103,7 +110,7 @@ Each call emits one INFO entry on completion:
   "correlationId": "…",
   "status": "ready",
   "elapsedMs": 28,
-  "checks": { "db": { … }, "sorobanRpc": { … }, "indexerLag": { … }, "queue": { … } }
+  "checks": { "db": { … }, "sorobanRpc": { … }, "indexerLag": { … }, "queue": { … }, "horizon": { … } }
 }
 ```
 

src/routes/health/ready.ts

@@ -3,11 +3,12 @@
  *
  * GET /api/health/ready — deep readiness check
  *
- * Probes all four runtime dependencies in parallel:
+ * Probes all five required runtime dependencies in parallel:
  *  1. db          — Postgres (SELECT 1)
  *  2. sorobanRpc  — Soroban RPC (getLatestLedger)
  *  3. indexerLag  — compares indexer cursor to chain tip
  *  4. queue       — Redis / BullMQ (PING)
+ *  5. horizon     — Horizon REST API
  *
  * HTTP response codes
  * ───────────────────
@@ -24,7 +25,8 @@
  *     "db":         { "status": "pass"|"fail", "durationMs": <n>, "message": "…" },
  *     "sorobanRpc": { … },
  *     "indexerLag": { … },
- *     "queue":      { … }
+ *     "queue":      { … },
+ *     "horizon":    { … }
  *   }
  * }
  *

src/services/readinessService.ts

@@ -13,9 +13,10 @@
  *  • sorobanRpc  — getLatestLedger() call to the Soroban RPC node
  *  • indexerLag  — compares indexer cursor to chain tip; fails when lag > threshold
  *  • queue       — Redis PING via the BullMQ connection
+ *  • horizon     — Horizon root endpoint used by settlement workers
  *
  * Each probe has a 1-second timeout to prevent a single slow dependency from
- * blocking the readiness response. All four run in parallel via Promise.allSettled.
+ * blocking the readiness response. All five run in parallel via Promise.allSettled.
  */
 
 import { env } from "../config/env";
@@ -38,6 +39,7 @@ export interface ReadinessStatus {
   sorobanRpc: ReadinessCheck;
   indexerLag: ReadinessCheck;
   queue: ReadinessCheck;
+  horizon: ReadinessCheck;
 }
 
 export interface ReadinessResult {
@@ -228,10 +230,51 @@ export async function checkQueue(redis: RedisLike): Promise<ReadinessCheck> {
   }
 }
 
+/**
+ * Probe Horizon, which is required by the settlement confirmer worker.
+ *
+ * The response status is checked explicitly: a reachable Horizon instance
+ * returning an error page is not a healthy dependency.
+ */
+export async function checkHorizon(): Promise<ReadinessCheck> {
+  const start = Date.now();
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), PROBE_TIMEOUT_MS);
+
+  try {
+    const response = await fetch(env.HORIZON_URL, {
+      signal: controller.signal,
+    });
+
+    if (!response.ok) {
+      return {
+        status: "fail",
+        durationMs: Date.now() - start,
+        message: `Horizon returned HTTP ${response.status}`,
+      };
+    }
+
+    return {
+      status: "pass",
+      durationMs: Date.now() - start,
+      message: "Horizon healthy",
+    };
+  } catch (error) {
+    logger.error({ error }, "readiness_horizon_check_failed");
+    return {
+      status: "fail",
+      durationMs: Date.now() - start,
+      message: error instanceof Error ? error.message : "Horizon failed",
+    };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
 // ── Top-level orchestrator ────────────────────────────────────────────────────
 
 /**
- * Run all four readiness probes in parallel and return a consolidated result.
+ * Run all five readiness probes in parallel and return a consolidated result.
  *
  * A failed `Promise.allSettled` branch (i.e. an unexpected throw that bypasses
  * the probe's own try/catch) is mapped to a generic "fail" check so the
@@ -244,12 +287,13 @@ export async function performReadinessCheck(
   db: DbLike,
   redis: RedisLike,
 ): Promise<ReadinessResult> {
-  const [dbResult, rpcResult, lagResult, queueResult] =
+  const [dbResult, rpcResult, lagResult, queueResult, horizonResult] =
     await Promise.allSettled([
       checkDatabase(db),
       checkSorobanRpc(),
       checkIndexerLag(db),
       checkQueue(redis),
+      checkHorizon(),
     ]);
 
   const now = Date.now();
@@ -267,6 +311,7 @@ export async function performReadinessCheck(
     sorobanRpc: unwrap(rpcResult, "Soroban RPC check threw unexpectedly"),
     indexerLag: unwrap(lagResult, "Indexer lag check threw unexpectedly"),
     queue: unwrap(queueResult, "Queue check threw unexpectedly"),
+    horizon: unwrap(horizonResult, "Horizon check threw unexpectedly"),
   };
 
   const ready = Object.values(checks).every((c) => c.status === "pass");

tests/healthReady.test.ts

@@ -9,7 +9,7 @@
  *
  * Coverage targets
  * ────────────────
- *  • All four probes: db, sorobanRpc, indexerLag, queue
+ *  • All five required probes: db, sorobanRpc, indexerLag, queue, horizon
  *  • 200 when all pass / 503 when any fail
  *  • Response shape: status, correlationId, checkedAt, checks
  *  • correlationId echo + UUID generation
@@ -30,6 +30,16 @@ process.env.REDIS_URL = "redis://localhost:6379";
 
 // ── Mocks (must come before createApp / route imports) ───────────────────────
 
+const mockGetLatestLedger = jest.fn().mockResolvedValue({ sequence: 1100 });
+
+jest.mock("@stellar/stellar-sdk", () => ({
+  SorobanRpc: {
+    Server: jest.fn().mockImplementation(() => ({
+      getLatestLedger: mockGetLatestLedger,
+    })),
+  },
+}));
+
 // Prevent real DB pool from being opened.
 jest.mock("../src/db/client", () => ({
   db: {},
@@ -101,6 +111,7 @@ function allPass(): ReadinessResult {
       sorobanRpc: { status: "pass", durationMs: 10, message: "Soroban RPC healthy" },
       indexerLag: { status: "pass", durationMs: 8, message: "Indexer lag healthy: 50 ≤ 200 ledgers" },
       queue: { status: "pass", durationMs: 2, message: "Queue (Redis) healthy" },
+      horizon: { status: "pass", durationMs: 4, message: "Horizon healthy" },
     },
   };
 }
@@ -225,27 +236,78 @@ describe("readinessService — individual probes (unit)", () => {
     });
   });
 
-  describe("performReadinessCheck", () => {
-    // For these tests we mock the Soroban SDK at the module level so no real
-    // network call is made, and we control db/redis via injected stubs.
+  describe("checkHorizon", () => {
+    const fetchMock = jest.spyOn(globalThis, "fetch");
+
+    afterEach(() => {
+      fetchMock.mockReset();
+    });
+
+    it("returns pass for a successful Horizon response", async () => {
+      fetchMock.mockResolvedValue({ ok: true, status: 200 } as Response);
+
+      const result = await real.checkHorizon();
+
+      expect(result.status).toBe("pass");
+      expect(fetchMock).toHaveBeenCalledWith(
+        "https://horizon-testnet.stellar.org",
+        expect.objectContaining({ signal: expect.any(AbortSignal) }),
+      );
+    });
+
+    it("returns fail when Horizon responds with an error status", async () => {
+      fetchMock.mockResolvedValue({ ok: false, status: 503 } as Response);
+
+      const result = await real.checkHorizon();
+
+      expect(result).toMatchObject({
+        status: "fail",
+        message: "Horizon returned HTTP 503",
+      });
+    });
+
+    it("returns fail when Horizon is unreachable", async () => {
+      fetchMock.mockRejectedValue(new Error("network unavailable"));
 
+      const result = await real.checkHorizon();
+
+      expect(result).toMatchObject({
+        status: "fail",
+        message: "network unavailable",
+      });
+    });
+
+    it("returns fail when Horizon does not respond within the probe timeout", async () => {
+      jest.useFakeTimers();
+      fetchMock.mockImplementation((_input, init) =>
+        new Promise<Response>((_, reject) => {
+          init?.signal?.addEventListener("abort", () => {
+            reject(new Error("This operation was aborted"));
+          });
+        }),
+      );
+
+      const promise = real.checkHorizon();
+      jest.advanceTimersByTime(1_100);
+      const result = await promise;
+
+      expect(result.status).toBe("fail");
+      expect(result.message).toContain("aborted");
+      jest.useRealTimers();
+    });
+  });
+
+  describe("performReadinessCheck", () => {
     beforeEach(() => {
-      // Mock the entire Stellar SDK so checkSorobanRpc and checkIndexerLag
-      // don't try to hit the real RPC.
-      jest.doMock("@stellar/stellar-sdk", () => ({
-        SorobanRpc: {
-          Server: jest.fn().mockImplementation(() => ({
-            getLatestLedger: jest.fn().mockResolvedValue({ sequence: 1100 }),
-          })),
-        },
-      }));
+      jest.spyOn(globalThis, "fetch").mockResolvedValue({ ok: true, status: 200 } as Response);
+      mockGetLatestLedger.mockResolvedValue({ sequence: 1100 });
     });
 
     afterEach(() => {
-      jest.dontMock("@stellar/stellar-sdk");
+      jest.restoreAllMocks();
     });
 
-    it("returns ready when db and redis pass (rpc/indexer may vary)", async () => {
+    it("runs all required dependency probes", async () => {
       const db = makeDb();
       const redis = makeRedis("PONG");
 
@@ -254,6 +316,7 @@ describe("readinessService — individual probes (unit)", () => {
       // db and queue must pass; overall result depends on RPC reachability in CI
       expect(result.checks.db.status).toBe("pass");
       expect(result.checks.queue.status).toBe("pass");
+      expect(result.checks.horizon.status).toBe("pass");
       expect(result).toHaveProperty("status");
       expect(result).toHaveProperty("checks");
     });
@@ -304,7 +367,7 @@ describe("GET /api/health/ready — HTTP", () => {
     expect(res.body.status).toBe("ready");
   });
 
-  it("returns all four checks in the body", async () => {
+  it("returns all five required checks in the body", async () => {
     mockPerform.mockResolvedValue(allPass());
 
     const res = await request(makeApp()).get("/api/health/ready");
@@ -313,6 +376,7 @@ describe("GET /api/health/ready — HTTP", () => {
     expect(res.body.checks).toHaveProperty("sorobanRpc");
     expect(res.body.checks).toHaveProperty("indexerLag");
     expect(res.body.checks).toHaveProperty("queue");
+    expect(res.body.checks).toHaveProperty("horizon");
   });
 
   it("each check has status, durationMs, and message", async () => {
@@ -368,6 +432,15 @@ describe("GET /api/health/ready — HTTP", () => {
     expect(res.body.checks.queue.status).toBe("fail");
   });
 
+  it("returns 503 when Horizon probe fails", async () => {
+    mockPerform.mockResolvedValue(oneFailure("horizon"));
+
+    const res = await request(makeApp()).get("/api/health/ready");
+
+    expect(res.status).toBe(503);
+    expect(res.body.checks.horizon.status).toBe("fail");
+  });
+
   it("returns 503 when all probes fail", async () => {
     mockPerform.mockResolvedValue({
       status: "unready",
@@ -376,6 +449,7 @@ describe("GET /api/health/ready — HTTP", () => {
         sorobanRpc: { status: "fail", durationMs: 1001, message: "RPC down" },
         indexerLag: { status: "fail", durationMs: 1001, message: "Lag too high" },
         queue: { status: "fail", durationMs: 1001, message: "Redis down" },
+        horizon: { status: "fail", durationMs: 1001, message: "Horizon down" },
       },
     });