diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3b6a0cd801..b66df12186 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -85,11 +85,26 @@ updates: update-types: - minor - patch + # These lines are generated from packages/stack/src/Artifacts.ts (the single version table + # for slim-capable services); updates to them arrive through slim-release-published.yml, not + # Dependabot. Kong and pg14 (supabase/postgres) have no slim build either, but are bumped by + # hand, not by Dependabot. Dependabot keeps bumping only the images that remain genuinely + # upstream-only: migra, pg_prove, pgadmin-schema-diff. ignore: - dependency-name: "library/kong" - dependency-name: "axllent/mailpit" - dependency-name: "darthsim/imgproxy" - dependency-name: "timberio/vector" + - dependency-name: "supabase/postgres" + - dependency-name: "supabase/gotrue" + - dependency-name: "postgrest/postgrest" + - dependency-name: "supabase/realtime" + - dependency-name: "supabase/storage-api" + - dependency-name: "supabase/edge-runtime" + - dependency-name: "supabase/studio" + - dependency-name: "supabase/postgres-meta" + - dependency-name: "supabase/logflare" + - dependency-name: "supabase/supavisor" cooldown: default-days: 7 exclude: diff --git a/.github/scripts/slim-mirror-payload.ts b/.github/scripts/slim-mirror-payload.ts index 74191dfebf..5b61fff58e 100644 --- a/.github/scripts/slim-mirror-payload.ts +++ b/.github/scripts/slim-mirror-payload.ts @@ -4,7 +4,7 @@ */ const SERVICE_PATTERN = /^[a-z][a-z0-9-]*$/; -export const VERSION_PATTERN = /^[A-Za-z0-9._-]+$/; +const VERSION_PATTERN = /^[A-Za-z0-9._-]+$/; export const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; export const SOURCE_REGISTRY = "ghcr.io/supabase/cli"; diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index 809dce551f..1a266d12be 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -1,132 +1,1331 @@ import { describe, expect, test } from "bun:test"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; -import { InvalidPayloadError } from "./slim-mirror-payload.ts"; +import { InvalidPayloadError, nativeFileNames, nativeObjectUrl } from "./slim-mirror-payload.ts"; import { CATALOG_PATH, - planArtifactCatalogUpdate, - type ReleasePublication, + planSlimUpdates, + planUpdatesForService, + refreshCatalogPin, + resolveRevisionPin, + validateSlimReleasePublishedPayload, + waitForExpectedRelease, + type RevisionIo, } from "./sync-artifacts-catalog.ts"; -const DIGEST_B = `sha256:${"b".repeat(64)}`; -const POSTGRES_17 = `ghcr.io/supabase/cli/postgres:17.6.1.168@sha256:${"9".repeat(64)}`; -const POSTGRES_15 = `ghcr.io/supabase/cli/postgres:15.14.1.168@sha256:${"f".repeat(64)}`; +const NATIVE_TARGETS = ["darwin-arm64", "linux-amd64", "linux-arm64"] as const; + +const hex = (seed: string): string => new Bun.CryptoHasher("sha256").update(seed).digest("hex"); +const digest = (seed: string): string => `sha256:${hex(seed)}`; + +type NativeDigests = Record<(typeof NATIVE_TARGETS)[number], { archive: string; manifest: string }>; + +const nativeDigests = (seed: string): NativeDigests => ({ + "darwin-arm64": { archive: hex(`${seed}a`), manifest: hex(`${seed}A`) }, + "linux-amd64": { archive: hex(`${seed}b`), manifest: hex(`${seed}B`) }, + "linux-arm64": { archive: hex(`${seed}c`), manifest: hex(`${seed}C`) }, +}); + +/** Builds a `SHA256SUMS` body via the same file-name helper the sync script reads. */ +function checksumsFor(service: string, releaseVersion: string, digests: NativeDigests): string { + return NATIVE_TARGETS.map((target) => { + const files = nativeFileNames(service, releaseVersion, target); + return `${digests[target].archive} ${files.archive}\n${digests[target].manifest} ${files.manifest}`; + }).join("\n"); +} + +/** An `io` whose S3 copies always match the given checksums. */ +function matchingS3( + service: string, + releaseVersion: string, + digests: NativeDigests, +): RevisionIo["s3Sha256"] { + const byUrl = new Map(); + for (const target of NATIVE_TARGETS) { + const files = nativeFileNames(service, releaseVersion, target); + byUrl.set(nativeObjectUrl(service, releaseVersion, files.archive), digests[target].archive); + byUrl.set(nativeObjectUrl(service, releaseVersion, files.manifest), digests[target].manifest); + } + return async (url) => byUrl.get(url); +} + +/** + * Runs the repo's pinned `oxfmt` binary over `source`, the way `slim-release-published.yml` + * formats the catalog after every write, so parsing tests exercise real formatter output + * (line-wrapping, trailing commas) instead of a hand-written single-line literal. + */ +async function formatWithOxfmt(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "sync-artifacts-catalog-")); + const filePath = join(dir, "Artifacts.ts"); + try { + await writeFile(filePath, source); + const proc = Bun.spawn(["node_modules/.bin/oxfmt", "--config", ".oxfmtrc.json", filePath], { + stdout: "pipe", + stderr: "pipe", + }); + const [stderr, exitCode] = await Promise.all([new Response(proc.stderr).text(), proc.exited]); + if (exitCode !== 0) throw new Error(`oxfmt failed: ${stderr}`); + return await readFile(filePath, "utf8"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +/** + * A resolved `ArtifactPin` literal, for fixtures that exercise refreshing an already-resolved + * catalog entry to a new revision (the committed catalog only carries resolved pins). `seed` keys a real, distinct digest via the module's `digest` helper. + */ +function resolvedPinLiteral( + service: string, + version: string, + revision: number, + seed: string, +): string { + return `{ upstreamVersion: "${version}", revision: ${revision}, image: "ghcr.io/supabase/cli/${service}:${version}-r${revision}@${digest(seed)}", natives: {} }`; +} + +const postgres17Pin = resolvedPinLiteral("postgres", "17.6.1.168", 0, "postgres-17"); +/** + * The default pin's `image` field, quoted. A formatter may move a long property value onto its + * own line, but never breaks a string literal's contents — so this survives real formatting the + * way asserting the whole (potentially re-wrapped) `postgres17Pin` literal would not. + */ +const postgres17Image = `"ghcr.io/supabase/cli/postgres:17.6.1.168-r0@${digest("postgres-17")}"`; const fixture = `const workloadCatalog = { database: definition( "postgres", - "17.6.1.168", - "${POSTGRES_17}", + ${postgres17Pin}, "bin/supabase-postgres-start", ["bin/supabase-postgres-start"], - { "15.14.1.168": "${POSTGRES_15}" }, + { "15.14.1.168": ${resolvedPinLiteral("postgres", "15.14.1.168", 0, "postgres-15")} }, ), - rest: definition("postgrest", "v16.2", "ghcr.io/supabase/cli/postgrest:v16.2", "bin/postgrest"), + rest: definition("postgrest", ${resolvedPinLiteral("postgrest", "v16.2", 0, "postgrest")}, "bin/postgrest"), + storage: definition("storage", ${resolvedPinLiteral("storage", "v1.73.0", 0, "storage")}, "bin/storage", ["bin/storage"]), analytics: definition( "analytics", - "v1.50.9", - "ghcr.io/supabase/cli/analytics:v1.50.9@sha256:${"a".repeat(64)}", + ${resolvedPinLiteral("analytics", "v1.50.9", 0, "analytics")}, "bin/logflare", ), }; `; -const published = (digest?: string): ReleasePublication => ({ status: "published", digest }); - -describe("planArtifactCatalogUpdate", () => { - test("pins Dockerfile tags that changed and leaves the other postgres line", async () => { - const plan = await planArtifactCatalogUpdate({ - baseDockerfile: `FROM supabase/postgres:17.6.1.168 AS pg -FROM postgrest/postgrest:v16.2 AS postgrest -FROM supabase/logflare:1.50.9 AS logflare -`, - dockerfile: `FROM supabase/postgres:17.6.1.171 AS pg -FROM library/kong:2.8.1 AS kong -FROM postgrest/postgrest:v16.3 AS postgrest -FROM supabase/logflare:1.50.9 AS logflare -`, - catalog: fixture, - publication: async (service) => published(service === "postgres" ? DIGEST_B : undefined), +const vectorFixture = `const workloadCatalog = { + vector: definition("vector", ${resolvedPinLiteral("vector", "0.53.0", 0, "vector")}, "bin/vector"), +}; +`; + +/** + * Fixtures for `planSlimUpdates`: a committed catalog always carries resolved pins, so these + * give every pin a real revision. + */ +const plannerFixture = `const workloadCatalog = { + rest: definition( + "postgrest", + { + upstreamVersion: "v16.2", + revision: 0, + image: "ghcr.io/supabase/cli/postgrest:v16.2-r0@sha256:1111111111111111111111111111111111111111111111111111111111111", + natives: {}, + }, + "bin/postgrest", + ), + storage: definition( + "storage", + { + upstreamVersion: "v1.73.0", + revision: 0, + image: "ghcr.io/supabase/cli/storage:v1.73.0-r0@sha256:2222222222222222222222222222222222222222222222222222222222222", + natives: {}, + }, + "bin/storage", + ["bin/storage"], + ), +}; +`; + +/** Postgres carries two lines (17 default, 15 additional), both resolved. */ +const postgresPlannerFixture = `const workloadCatalog = { + database: definition( + "postgres", + { + upstreamVersion: "17.6.1.168", + revision: 1, + image: "ghcr.io/supabase/cli/postgres:17.6.1.168-r1@sha256:3333333333333333333333333333333333333333333333333333333333333", + natives: {}, + }, + "bin/supabase-postgres-start", + ["bin/supabase-postgres-start"], + { + "15.14.1.168": { + upstreamVersion: "15.14.1.168", + revision: 3, + image: "ghcr.io/supabase/cli/postgres:15.14.1.168-r3@sha256:4444444444444444444444444444444444444444444444444444444444444", + natives: {}, + }, + }, + ), +}; +`; + +describe("validateSlimReleasePublishedPayload", () => { + test("rejects a newline injected into upstream_version", () => { + expect(() => + validateSlimReleasePublishedPayload({ + service: "postgres", + upstream_version: "15.14.1.168\nrevision=999\ninjected=yes", + revision: "0", + release_version: "15.14.1.168\nrevision=999\ninjected=yes-r0", + }), + ).toThrow(InvalidPayloadError); + }); + + test("a newline-bearing service value produces a single-line error", () => { + let caught: unknown; + try { + validateSlimReleasePublishedPayload({ + service: "postgres\n::error ::injected", + upstream_version: "15.14.1.168", + revision: "0", + release_version: "15.14.1.168-r0", + }); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(InvalidPayloadError); + const message = (caught as InvalidPayloadError).message; + expect(message.split("\n")).toHaveLength(1); + expect(message).toContain(JSON.stringify("postgres\n::error ::injected")); + }); + + test("accepts a Studio-style calendar-versioned upstream", () => { + const payload = validateSlimReleasePublishedPayload({ + service: "studio", + upstream_version: "2026.09.04-sha-5a67366", + revision: "1", + release_version: "2026.09.04-sha-5a67366-r1", }); - expect(plan.updates.map((update) => `${update.service} ${update.version}`)).toEqual([ - "postgres 17.6.1.171", - "postgrest v16.3", + expect(payload).toEqual({ + service: "studio", + upstream_version: "2026.09.04-sha-5a67366", + revision: 1, + release_version: "2026.09.04-sha-5a67366-r1", + }); + }); + + test("accepts a Postgres four-part upstream version", () => { + const payload = validateSlimReleasePublishedPayload({ + service: "postgres", + upstream_version: "15.14.1.168", + revision: "3", + release_version: "15.14.1.168-r3", + }); + + expect(payload).toEqual({ + service: "postgres", + upstream_version: "15.14.1.168", + revision: 3, + release_version: "15.14.1.168-r3", + }); + }); + + test("rejects a release_version that does not match the derived value", () => { + expect(() => + validateSlimReleasePublishedPayload({ + service: "postgres", + upstream_version: "15.14.1.168", + revision: "3", + release_version: "15.14.1.168-r4", + }), + ).toThrow(InvalidPayloadError); + }); +}); + +describe("planSlimUpdates", () => { + test("hotfix only: a higher committed revision of the pinned upstream", () => { + const { updates, warnings } = planSlimUpdates(plannerFixture, "postgrest", [ + "postgrest-v16.2-r1", + ]); + + expect(warnings).toEqual([]); + expect(updates).toEqual([ + { + kind: "hotfix", + line: undefined, + branch: "slim-hotfix/postgrest", + title: "chore(stack): pin postgrest v16.2-r1", + fromRelease: "v16.2-r0", + toUpstream: "v16.2", + toRelease: "v16.2-r1", + }, ]); - expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgres:17.6.1.171@${DIGEST_B}"`); - expect(plan.source).toContain(`"${POSTGRES_15}"`); - expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgrest:v16.3"`); - expect(plan.source).toContain(`"v1.50.9"`); - expect(plan.skipped).toEqual([]); }); - test("a missing release blocks the commit and OrioleDB does not", async () => { - const movesPostgres = await planArtifactCatalogUpdate({ - baseDockerfile: "FROM supabase/postgres:17.6.1.168 AS pg\n", - dockerfile: `FROM supabase/postgres:17.6.1.171 AS pg -FROM postgrest/postgrest:v16.3 AS postgrest -`, - catalog: fixture, - publication: async () => ({ status: "missing" }), - }); - expect(movesPostgres.updates).toEqual([]); - expect(movesPostgres.source).toBe(fixture); - expect(movesPostgres.skipped.every((skip) => skip.blocking)).toBe(true); - - const oriole = await planArtifactCatalogUpdate({ - baseDockerfile: - "FROM supabase/postgres:17.6.1.168 AS pg\nFROM postgrest/postgrest:v16.2 AS postgrest\n", - dockerfile: `FROM supabase/postgres:16.0.0.1-orioledb AS pg -FROM postgrest/postgrest:v16.3 AS postgrest -`, - catalog: fixture, - publication: async () => published(), + test("upgrade only: a newer committed upstream on the same line", () => { + const { updates } = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.4-r0"]); + + expect(updates).toEqual([ + { + kind: "upgrade", + line: undefined, + branch: "slim-bump/postgrest", + title: "chore(stack): bump postgrest to v16.4-r0", + fromRelease: "v16.2-r0", + toUpstream: "v16.4", + toRelease: "v16.4-r0", + }, + ]); + }); + + test("both a hotfix and an upgrade can be planned in the same run", () => { + const { updates } = planSlimUpdates(plannerFixture, "postgrest", [ + "postgrest-v16.2-r1", + "postgrest-v16.4-r0", + ]); + + expect(updates).toEqual([ + { + kind: "hotfix", + line: undefined, + branch: "slim-hotfix/postgrest", + title: "chore(stack): pin postgrest v16.2-r1", + fromRelease: "v16.2-r0", + toUpstream: "v16.2", + toRelease: "v16.2-r1", + }, + { + kind: "upgrade", + line: undefined, + branch: "slim-bump/postgrest", + title: "chore(stack): bump postgrest to v16.4-r0", + fromRelease: "v16.2-r0", + toUpstream: "v16.4", + toRelease: "v16.4-r0", + }, + ]); + }); + + test("an older committed upstream is a backlog republish: it plans nothing", () => { + const { updates, warnings } = planSlimUpdates(plannerFixture, "postgrest", [ + "postgrest-v16.1-r0", + ]); + + expect(updates).toEqual([]); + expect(warnings).toEqual([]); + }); + + test("postgres: upgrade on the 17 line, hotfix on the 15 line, a major-18 release ignored and warned about", () => { + const { updates, warnings } = planSlimUpdates(postgresPlannerFixture, "postgres", [ + "postgres-17.11.0.002-r0", + "postgres-15.14.1.168-r4", + "postgres-18.0.0.001-r0", + ]); + + // The ignored major-18 tag is warned about, but doesn't block the two valid updates + // alongside it (P1: a warning must never corrupt or swallow the plan). + expect(warnings).toEqual([ + "::warning ::postgres 18.0.0.001 is not on a release line packages/stack/src/Artifacts.ts carries for it; ignoring postgres-18.0.0.001-r0.", + ]); + expect(updates).toEqual([ + { + kind: "upgrade", + line: "17", + branch: "slim-bump/postgres-17", + title: "chore(stack): bump postgres to 17.11.0.002-r0", + fromRelease: "17.6.1.168-r1", + toUpstream: "17.11.0.002", + toRelease: "17.11.0.002-r0", + }, + { + kind: "hotfix", + line: "15", + branch: "slim-hotfix/postgres-15", + title: "chore(stack): pin postgres 15.14.1.168-r4", + fromRelease: "15.14.1.168-r3", + toUpstream: "15.14.1.168", + toRelease: "15.14.1.168-r4", + }, + ]); + }); + + test("Studio: a newer date upgrades; the same date with a different sha does not", () => { + const studioFixture = `const workloadCatalog = { + studio: definition( + "studio", + { + upstreamVersion: "2026.09.14-sha-aaaaaaa", + revision: 0, + image: "ghcr.io/supabase/cli/studio:2026.09.14-sha-aaaaaaa-r0@sha256:1111111111111111111111111111111111111111111111111111111111111", + natives: {}, + }, + "bin/studio", + ), +}; +`; + + const sameDate = planSlimUpdates(studioFixture, "studio", ["studio-2026.09.14-sha-bbbbbbb-r0"]); + expect(sameDate.updates).toEqual([]); + + const newerDate = planSlimUpdates(studioFixture, "studio", [ + "studio-2026.09.28-sha-ccccccc-r0", + ]); + expect(newerDate.updates).toEqual([ + { + kind: "upgrade", + line: undefined, + branch: "slim-bump/studio", + title: "chore(stack): bump studio to 2026.09.28-sha-ccccccc-r0", + fromRelease: "2026.09.14-sha-aaaaaaa-r0", + toUpstream: "2026.09.28-sha-ccccccc", + toRelease: "2026.09.28-sha-ccccccc-r0", + }, + ]); + }); + + test("Studio: a year rollover upgrades even though releaseLine differs (single-pin services accept any upstream)", () => { + const studioFixture = `const workloadCatalog = { + studio: definition( + "studio", + { + upstreamVersion: "2026.09.28-sha-5e59b60", + revision: 0, + image: "ghcr.io/supabase/cli/studio:2026.09.28-sha-5e59b60-r0@sha256:5555555555555555555555555555555555555555555555555555555555555", + natives: {}, + }, + "bin/studio", + ), +}; +`; + + const { updates, warnings } = planSlimUpdates(studioFixture, "studio", [ + "studio-2027.01.01-sha-abcdef0-r0", + ]); + + expect(warnings).toEqual([]); + expect(updates).toEqual([ + { + kind: "upgrade", + line: undefined, + branch: "slim-bump/studio", + title: "chore(stack): bump studio to 2027.01.01-sha-abcdef0-r0", + fromRelease: "2026.09.28-sha-5e59b60-r0", + toUpstream: "2027.01.01-sha-abcdef0", + toRelease: "2027.01.01-sha-abcdef0-r0", + }, + ]); + }); + + test("postgrest: a major-version bump upgrades even though releaseLine differs (single-pin services accept any upstream)", () => { + const { updates, warnings } = planSlimUpdates(plannerFixture, "postgrest", [ + "postgrest-v17.0-r0", + ]); + + expect(warnings).toEqual([]); + expect(updates).toEqual([ + { + kind: "upgrade", + line: undefined, + branch: "slim-bump/postgrest", + title: "chore(stack): bump postgrest to v17.0-r0", + fromRelease: "v16.2-r0", + toUpstream: "v17.0", + toRelease: "v17.0-r0", + }, + ]); + }); + + test("a non-comparable version (OrioleDB-style suffix) is ignored and warned about", () => { + const { updates, warnings } = planSlimUpdates(plannerFixture, "postgrest", [ + "postgrest-v16.2-orioledb-r0", + ]); + + expect(updates).toEqual([]); + expect(warnings).toEqual([ + "::warning ::postgrest v16.2-orioledb is not a comparable version; ignoring.", + ]); + }); + + test("a `%` in a non-comparable version is encoded, staying a single workflow-command line", () => { + // `.` in the tag pattern allows `%` (only a real newline can't reach this far — the pattern + // can't match across one), so a real, legitimately-listed tag can still carry a `%` here. + const { updates, warnings } = planSlimUpdates(plannerFixture, "postgrest", [ + "postgrest-v16.2%off-r0", + ]); + + expect(updates).toEqual([]); + expect(warnings).toEqual([ + "::warning ::postgrest v16.2%25off is not a comparable version; ignoring.", + ]); + expect(warnings[0]?.split("\n")).toHaveLength(1); + }); + + test("a legacy tag with no -rN is ignored", () => { + const { updates, warnings } = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.4"]); + + expect(updates).toEqual([]); + expect(warnings).toEqual([]); + }); + + test("postgrest tags never match postgres, even as a prefix", () => { + const { updates } = planSlimUpdates(postgresPlannerFixture, "postgres", [ + "postgrest-v16.4-r0", + "postgres-17.11.0.002-r0", + ]); + + expect(updates).toEqual([ + { + kind: "upgrade", + line: "17", + branch: "slim-bump/postgres-17", + title: "chore(stack): bump postgres to 17.11.0.002-r0", + fromRelease: "17.6.1.168-r1", + toUpstream: "17.11.0.002", + toRelease: "17.11.0.002-r0", + }, + ]); + }); + + test("branch and title carry no - suffix for a service with a single line", () => { + const { updates } = planSlimUpdates(plannerFixture, "storage", ["storage-v1.74.0-r0"]); + + expect(updates).toEqual([ + { + kind: "upgrade", + line: undefined, + branch: "slim-bump/storage", + title: "chore(stack): bump storage to v1.74.0-r0", + fromRelease: "v1.73.0-r0", + toUpstream: "v1.74.0", + toRelease: "v1.74.0-r0", + }, + ]); + }); + + test("rejects an emitted value that fails the anchored patterns, even from a trusted-looking catalog", () => { + // The catalog's own pinned upstream carries a space here — never written by `refreshCatalogPin` + // (which validates every field it resolves), but this simulates a corrupted catalog, or a + // release tag whose upstream portion isn't newline-only-unsafe (`.` already excludes + // newlines) yet still fails `UPSTREAM_VERSION_PATTERN`. Every value the planner emits is + // checked, regardless of source. + const adversarial = `const workloadCatalog = { + rest: definition( + "postgrest", + { + upstreamVersion: "v16.2 injected", + revision: 0, + image: "ghcr.io/supabase/cli/postgrest:v16.2-r0@sha256:2222222222222222222222222222222222222222222222222222222222222", + natives: {}, + }, + "bin/postgrest", + ), +}; +`; + + expect(() => + planSlimUpdates(adversarial, "postgrest", ["postgrest-v16.2 injected-r1"]), + ).toThrow(InvalidPayloadError); + }); + + test("plans nothing for a service the catalog does not model, and warns about it", () => { + const { updates, warnings } = planSlimUpdates(plannerFixture, "no-such-service", [ + "no-such-service-v1.0-r0", + ]); + + expect(updates).toEqual([]); + expect(warnings).toEqual([ + "::warning ::packages/stack/src/Artifacts.ts has no slim entry for no-such-service; nothing to plan.", + ]); + }); +}); + +describe("planUpdatesForService (the plan-updates transport's IO seam)", () => { + test("an ignored tag alongside a valid update: the valid record comes back, plus a separate warning", async () => { + const result = await planUpdatesForService({ + catalog: postgresPlannerFixture, + service: "postgres", + listReleaseTags: async () => [ + "postgres-17.11.0.002-r0", + "postgres-18.0.0.001-r0", // ignored: no line 18 + ], }); - expect(oriole.updates.map((update) => update.service)).toEqual(["postgrest"]); - expect(oriole.skipped).toEqual([ + + expect(result.warnings).toEqual([ + "::warning ::postgres 18.0.0.001 is not on a release line packages/stack/src/Artifacts.ts carries for it; ignoring postgres-18.0.0.001-r0.", + ]); + expect(result.updates).toEqual([ { - alias: "pg", - reason: "pg supabase/postgres:16.0.0.1-orioledb has no slim image.", - blocking: false, + kind: "upgrade", + line: "17", + branch: "slim-bump/postgres-17", + title: "chore(stack): bump postgres to 17.11.0.002-r0", + fromRelease: "17.6.1.168-r1", + toUpstream: "17.11.0.002", + toRelease: "17.11.0.002-r0", }, ]); - expect(oriole.source).toContain(`"${POSTGRES_17}"`); + }); +}); + +describe("waitForExpectedRelease (item A's eventual-consistency wait, call counts a subprocess can't assert)", () => { + test("visible on the first listing: returns immediately without waiting", async () => { + let calls = 0; + const waits: number[] = []; + const result = await waitForExpectedRelease( + "postgrest", + "v16.4-r1", + { + listReleaseTags: async () => { + calls += 1; + return ["postgrest-v16.4-r1"]; + }, + wait: async (ms) => { + waits.push(ms); + }, + }, + 6, + 10_000, + ); + + expect(result).toEqual({ visible: true, tags: ["postgrest-v16.4-r1"] }); + expect(calls).toBe(1); + expect(waits).toEqual([]); + }); + + test("visible on a later listing: re-lists and waits between attempts until it appears", async () => { + let calls = 0; + const waits: number[] = []; + const result = await waitForExpectedRelease( + "postgrest", + "v16.4-r1", + { + listReleaseTags: async () => { + calls += 1; + return calls < 3 ? ["postgrest-v16.4-r0"] : ["postgrest-v16.4-r0", "postgrest-v16.4-r1"]; + }, + wait: async (ms) => { + waits.push(ms); + }, + }, + 6, + 10_000, + ); + + expect(result).toEqual({ visible: true, tags: ["postgrest-v16.4-r0", "postgrest-v16.4-r1"] }); + expect(calls).toBe(3); + expect(waits).toEqual([10_000, 10_000]); + }); + + test("never visible: exhausts every bounded attempt, waiting between but not after the last", async () => { + let calls = 0; + const waits: number[] = []; + const result = await waitForExpectedRelease( + "postgrest", + "v16.4-r1", + { + listReleaseTags: async () => { + calls += 1; + return ["postgrest-v16.4-r0"]; + }, + wait: async (ms) => { + waits.push(ms); + }, + }, + 3, + 10, + ); + + expect(result).toEqual({ visible: false, tags: ["postgrest-v16.4-r0"] }); + expect(calls).toBe(3); + expect(waits).toEqual([10, 10]); + }); +}); + +describe("runPlanUpdates (the actual plan-updates CLI mode, not just the pure planner)", () => { + test("an ignored tag, a valid upgrade, and the expected dispatched release: --output gets exactly the valid record, the warning reaches stdout, and the process exits 0", async () => { + // Real catalog, real "postgrest" pin — a local fixture server stands in for the releases API + // (`SLIM_SERVICES_RELEASES_API`), so this exercises the real subprocess: the CLI's argv + // parsing, `--expect-release`'s visibility wait, the network lister, and the file/stdout + // wiring together, not just the pure planner or an in-process stub. + const catalog = await Bun.file(CATALOG_PATH).text(); + const pinMatch = + /definition\(\s*"postgrest",\s*\{\s*upstreamVersion:\s*"([^"]+)",\s*revision:\s*(\d+)/.exec( + catalog, + ); + if (pinMatch === null) throw new Error("postgrest pin not found in the real catalog"); + const pinnedUpstream = pinMatch[1] as string; + const pinnedRevision = Number(pinMatch[2]); + // Higher than any real pinned postgrest version, so it always plans as an upgrade. + const dispatchedRelease = "v999.0-r0"; + + const server = Bun.serve({ + port: 0, + fetch: () => + Response.json([ + { tag_name: `postgrest-${pinnedUpstream}-orioledb-r0`, draft: false }, // ignored + { tag_name: `postgrest-${dispatchedRelease}`, draft: false }, // the valid upgrade + ]), + }); + const dir = await mkdtemp(join(tmpdir(), "plan-updates-subprocess-")); + const outputPath = join(dir, "slim-updates.tsv"); + try { + const proc = Bun.spawn( + [ + "bun", + ".github/scripts/sync-artifacts-catalog.ts", + "plan-updates", + "--service", + "postgrest", + "--format", + "lines", + "--expect-release", + dispatchedRelease, + "--output", + outputPath, + ], + { + stdout: "pipe", + stderr: "pipe", + env: { + ...globalThis.process.env, + SLIM_SERVICES_RELEASES_API: `http://127.0.0.1:${server.port}`, + }, + }, + ); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + + expect(exitCode).toBe(0); + expect(stdout).toContain( + `::warning ::postgrest ${pinnedUpstream}-orioledb is not a comparable version; ignoring.`, + ); + const content = await readFile(outputPath, "utf8"); + expect(content).toBe( + `upgrade\x1fslim-bump/postgrest\x1fchore(stack): bump postgrest to ${dispatchedRelease}\x1f${dispatchedRelease}\x1f${pinnedUpstream}-r${pinnedRevision}\n`, + ); + } finally { + await server.stop(true); + await rm(dir, { recursive: true, force: true }); + } + }); + + test("the expected dispatched release never becomes visible: the process exits non-zero with an actionable error, and writes no output file", async () => { + const server = Bun.serve({ + port: 0, + fetch: () => Response.json([{ tag_name: "postgrest-v16.4-r0", draft: false }]), + }); + const dir = await mkdtemp(join(tmpdir(), "plan-updates-subprocess-never-")); + const outputPath = join(dir, "slim-updates.tsv"); + try { + const proc = Bun.spawn( + [ + "bun", + ".github/scripts/sync-artifacts-catalog.ts", + "plan-updates", + "--service", + "postgrest", + "--format", + "lines", + "--expect-release", + "v999.0-r0", + "--output", + outputPath, + ], + { + stdout: "pipe", + stderr: "pipe", + env: { + ...globalThis.process.env, + SLIM_SERVICES_RELEASES_API: `http://127.0.0.1:${server.port}`, + SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS: "5", + }, + }, + ); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + + expect(exitCode).toBe(1); + expect(stdout).toContain("::error ::"); + expect(stdout).toContain("postgrest-v999.0-r0 is not visible yet"); + await expect(readFile(outputPath, "utf8")).rejects.toThrow(); + } finally { + await server.stop(true); + await rm(dir, { recursive: true, force: true }); + } + }); + + test("a missing --output exits non-zero before any network call", async () => { + const proc = Bun.spawn( + ["bun", ".github/scripts/sync-artifacts-catalog.ts", "plan-updates", "--service", "auth"], + { stdout: "pipe", stderr: "pipe" }, + ); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + + expect(exitCode).toBe(1); + expect(stdout).toContain( + "Usage: sync-artifacts-catalog.ts plan-updates --service --output ", + ); + }); + + test("a newline-bearing --service is rejected before any listing or wait", async () => { + const dir = await mkdtemp(join(tmpdir(), "plan-updates-bad-service-")); + const outputPath = join(dir, "slim-updates.tsv"); + try { + const proc = Bun.spawn( + [ + "bun", + ".github/scripts/sync-artifacts-catalog.ts", + "plan-updates", + "--service", + "auth\n::error ::injected", + "--output", + outputPath, + ], + { stdout: "pipe", stderr: "pipe" }, + ); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + + expect(exitCode).toBe(1); + expect(stdout).toContain("invalid --service"); + await expect(readFile(outputPath, "utf8")).rejects.toThrow(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("a newline-bearing --format is rejected, producing exactly one ::error :: line (not JSON.stringify'd; the boundary encoder is what protects it)", async () => { + const dir = await mkdtemp(join(tmpdir(), "plan-updates-bad-format-")); + const outputPath = join(dir, "slim-updates.tsv"); + try { + const proc = Bun.spawn( + [ + "bun", + ".github/scripts/sync-artifacts-catalog.ts", + "plan-updates", + "--service", + "auth", + "--output", + outputPath, + "--format", + "lines\n::error ::injected", + ], + { stdout: "pipe", stderr: "pipe" }, + ); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + + expect(exitCode).toBe(1); + const lines = stdout.trimEnd().split("\n"); + expect(lines).toHaveLength(1); + expect(lines[0]).toMatch(/^::error ::/); + expect(lines[0]).toContain("%0A"); + await expect(readFile(outputPath, "utf8")).rejects.toThrow(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test.each(["5ms", "9".repeat(309)])( + "an invalid SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS (%s) exits non-zero with a configuration error", + async (waitMs) => { + const proc = Bun.spawn( + [ + "bun", + ".github/scripts/sync-artifacts-catalog.ts", + "plan-updates", + "--service", + "postgrest", + "--expect-release", + "v999.0-r0", + "--output", + join(await mkdtemp(join(tmpdir(), "plan-updates-bad-wait-")), "slim-updates.tsv"), + ], + { + stdout: "pipe", + stderr: "pipe", + env: { ...globalThis.process.env, SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS: waitMs }, + }, + ); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + + expect(exitCode).toBe(1); + expect(stdout).toContain("SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS"); + expect(stdout).toContain("non-negative integer"); + }, + ); +}); + +describe("refreshCatalogPin", () => { + test("refreshes a service to the highest revision of its currently pinned upstream", async () => { + const digests = nativeDigests("g"); + const io: RevisionIo = { + listReleaseTags: async () => ["analytics-v1.50.9-r0", "analytics-v1.50.9-r1"], + fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r1", digests), + imageDigest: async () => digest("h"), + s3Sha256: matchingS3("analytics", "v1.50.9-r1", digests), + }; + + const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); + + expect(result.update).toEqual({ + service: "analytics", + version: "v1.50.9", + revision: 1, + previousVersion: "v1.50.9", + target: "default", + }); + expect(result.source).toContain("revision: 1"); + expect(result.source).toContain( + `image: "ghcr.io/supabase/cli/analytics:v1.50.9-r1@${digest("h")}"`, + ); + }); + + test("a resolved pin survives real formatting and can be refreshed again", async () => { + const first = nativeDigests("i"); + const firstIo: RevisionIo = { + listReleaseTags: async () => ["postgrest-v16.2-r0"], + fetchChecksums: async () => checksumsFor("postgrest", "v16.2-r0", first), + imageDigest: async () => digest("j"), + s3Sha256: matchingS3("postgrest", "v16.2-r0", first), + }; + const written = await refreshCatalogPin({ + catalog: fixture, + service: "postgrest", + io: firstIo, + }); + expect(written.update?.revision).toBe(0); + + const formatted = await formatWithOxfmt(written.source); + // The formatter actually wrapped the literal onto several lines with trailing commas; + // otherwise this test would not be exercising what it claims to. + expect(formatted.split("\n").length).toBeGreaterThan(written.source.split("\n").length); + + const second = nativeDigests("k"); + const secondIo: RevisionIo = { + listReleaseTags: async () => ["postgrest-v16.2-r0", "postgrest-v16.2-r1"], + fetchChecksums: async () => checksumsFor("postgrest", "v16.2-r1", second), + imageDigest: async () => digest("l"), + s3Sha256: matchingS3("postgrest", "v16.2-r1", second), + }; + const refreshed = await refreshCatalogPin({ + catalog: formatted, + service: "postgrest", + io: secondIo, + }); + + expect(refreshed.update).toEqual({ + service: "postgrest", + version: "v16.2", + revision: 1, + previousVersion: "v16.2", + target: "default", + }); + expect(refreshed.source).toContain("revision: 1"); + expect(refreshed.source).toContain(second["darwin-arm64"].archive); + }); + + test("refreshes the Postgres 15 additional pin, including after formatting", async () => { + const digests = nativeDigests("m"); + const io: RevisionIo = { + listReleaseTags: async () => ["postgres-15.14.1.168-r0"], + fetchChecksums: async () => checksumsFor("postgres", "15.14.1.168-r0", digests), + imageDigest: async () => digest("n"), + s3Sha256: matchingS3("postgres", "15.14.1.168-r0", digests), + }; + + const written = await refreshCatalogPin({ + catalog: fixture, + service: "postgres", + upstream: "15.14.1.168", + io, + }); + expect(written.update).toEqual({ + service: "postgres", + version: "15.14.1.168", + revision: 0, + previousVersion: "15.14.1.168", + target: "additional", + }); + // The default (17.x) postgres line is untouched. + expect(written.source).toContain(postgres17Image); + + const formatted = await formatWithOxfmt(written.source); + const nextDigests = nativeDigests("o"); + const nextIo: RevisionIo = { + listReleaseTags: async () => ["postgres-15.14.1.168-r0", "postgres-15.14.1.168-r1"], + fetchChecksums: async () => checksumsFor("postgres", "15.14.1.168-r1", nextDigests), + imageDigest: async () => digest("p"), + s3Sha256: matchingS3("postgres", "15.14.1.168-r1", nextDigests), + }; + + const refreshed = await refreshCatalogPin({ + catalog: formatted, + service: "postgres", + upstream: "15.14.1.168", + io: nextIo, + }); + + expect(refreshed.update).toEqual({ + service: "postgres", + version: "15.14.1.168", + revision: 1, + previousVersion: "15.14.1.168", + target: "additional", + }); + expect(refreshed.source).toContain(postgres17Image); + expect(refreshed.source).toContain(nextDigests["linux-arm64"].manifest); + }); + + test("moves the Postgres 15 additional pin to a new upstream version, updating its key", async () => { + const digests = nativeDigests("u"); + const io: RevisionIo = { + listReleaseTags: async () => ["postgres-15.19.0.002-r0"], + fetchChecksums: async () => checksumsFor("postgres", "15.19.0.002-r0", digests), + imageDigest: async () => digest("v"), + s3Sha256: matchingS3("postgres", "15.19.0.002-r0", digests), + }; + + const written = await refreshCatalogPin({ + catalog: fixture, + service: "postgres", + upstream: "15.19.0.002", + io, + }); + expect(written.update).toEqual({ + service: "postgres", + version: "15.19.0.002", + revision: 0, + previousVersion: "15.14.1.168", + target: "additional", + }); + // The old key is gone; the new key matches the resolved pin's `upstreamVersion`. + expect(written.source).not.toContain('"15.14.1.168"'); + expect(written.source).toContain('"15.19.0.002": { upstreamVersion: "15.19.0.002"'); + // The default (17.x) postgres line is untouched. + expect(written.source).toContain(postgres17Image); + + // The renamed key resolves the entry again after real formatting, e.g. for a later hotfix. + const formatted = await formatWithOxfmt(written.source); + const nextDigests = nativeDigests("w"); + const nextIo: RevisionIo = { + listReleaseTags: async () => ["postgres-15.19.0.002-r0", "postgres-15.19.0.002-r1"], + fetchChecksums: async () => checksumsFor("postgres", "15.19.0.002-r1", nextDigests), + imageDigest: async () => digest("x"), + s3Sha256: matchingS3("postgres", "15.19.0.002-r1", nextDigests), + }; + const refreshed = await refreshCatalogPin({ + catalog: formatted, + service: "postgres", + upstream: "15.19.0.002", + io: nextIo, + }); + + expect(refreshed.update).toEqual({ + service: "postgres", + version: "15.19.0.002", + revision: 1, + previousVersion: "15.19.0.002", + target: "additional", + }); + expect(refreshed.source).toContain(nextDigests["linux-arm64"].manifest); + }); +}); + +describe("resolveRevisionPin waits for the S3 mirror", () => { + test("waits while one S3 object is missing, then resolves once it appears", async () => { + const digests = nativeDigests("s3-wait"); + const releaseVersion = "v16.2-r0"; + const missingUrl = nativeObjectUrl( + "postgrest", + releaseVersion, + nativeFileNames("postgrest", releaseVersion, "darwin-arm64").archive, + ); + const present = matchingS3("postgrest", releaseVersion, digests); + let missingCalls = 0; + const waits: number[] = []; + const io: RevisionIo = { + listReleaseTags: async () => [`postgrest-${releaseVersion}`], + fetchChecksums: async () => checksumsFor("postgrest", releaseVersion, digests), + imageDigest: async () => digest("s3-wait-digest"), + s3Sha256: async (url) => { + if (url === missingUrl) { + missingCalls += 1; + if (missingCalls < 2) return undefined; + } + return present(url); + }, + wait: async (ms) => { + waits.push(ms); + }, + }; + + const resolution = await resolveRevisionPin("postgrest", "v16.2", io); + + expect(resolution.status).toBe("resolved"); + expect(missingCalls).toBe(2); + expect(waits).toHaveLength(1); }); - test("refuses a backward pin and a tag that would escape the catalog string", async () => { - const backward = await planArtifactCatalogUpdate({ - baseDockerfile: "FROM supabase/postgres:17.6.1.170 AS pg\n", - dockerfile: "FROM supabase/postgres:17.6.1.171 AS pg\n", - catalog: fixture.replaceAll("17.6.1.168", "17.6.1.173"), - publication: async () => { - throw new Error("publication lookup"); + test("fails immediately on a digest mismatch, without waiting", async () => { + const digests = nativeDigests("s3-mismatch"); + const releaseVersion = "v16.2-r0"; + const waits: number[] = []; + const io: RevisionIo = { + listReleaseTags: async () => [`postgrest-${releaseVersion}`], + fetchChecksums: async () => checksumsFor("postgrest", releaseVersion, digests), + imageDigest: async () => digest("s3-mismatch-digest"), + s3Sha256: async () => hex("wrong-bytes"), + wait: async (ms) => { + waits.push(ms); }, + }; + + const resolution = await resolveRevisionPin("postgrest", "v16.2", io); + + expect(resolution.status).toBe("stale"); + if (resolution.status !== "stale") throw new Error("expected status 'stale'"); + expect(resolution.message).toContain("digest mismatch"); + expect(waits).toEqual([]); + }); + + test("fails after the bounded attempts when an S3 object never appears, with an actionable message", async () => { + const digests = nativeDigests("s3-never"); + const releaseVersion = "v16.2-r0"; + const waits: number[] = []; + const io: RevisionIo = { + listReleaseTags: async () => [`postgrest-${releaseVersion}`], + fetchChecksums: async () => checksumsFor("postgrest", releaseVersion, digests), + imageDigest: async () => digest("s3-never-digest"), + s3Sha256: async () => undefined, + wait: async (ms) => { + waits.push(ms); + }, + }; + + const resolution = await resolveRevisionPin("postgrest", "v16.2", io); + + expect(resolution.status).toBe("stale"); + if (resolution.status !== "stale") throw new Error("expected status 'stale'"); + expect(resolution.message).toContain("mirror-slim-image.yml"); + expect(resolution.message).toContain("hasn't finished"); + expect(waits.length).toBeGreaterThan(0); + }); +}); + +describe("refreshCatalogPin --release", () => { + test("pins exactly the requested committed release, not the highest one", async () => { + const digests = nativeDigests("release-0"); + const io: RevisionIo = { + listReleaseTags: async () => ["postgrest-v16.2-r0", "postgrest-v16.2-r1"], + fetchChecksums: async () => checksumsFor("postgrest", "v16.2-r0", digests), + imageDigest: async () => digest("release-digest-0"), + s3Sha256: matchingS3("postgrest", "v16.2-r0", digests), + }; + + const result = await refreshCatalogPin({ + catalog: fixture, + service: "postgrest", + release: "v16.2-r0", + io, }); - expect(backward.updates).toEqual([]); - expect(backward.skipped.map((skip) => skip.blocking)).toEqual([true]); - expect(backward.source).toContain(`"17.6.1.173"`); + + expect(result.update).toEqual({ + service: "postgrest", + version: "v16.2", + revision: 0, + previousVersion: "v16.2", + target: "default", + }); + }); + + test("fails when the requested release is not committed", async () => { + const io: RevisionIo = { + listReleaseTags: async () => ["postgrest-v16.2-r0"], + fetchChecksums: async () => undefined, + imageDigest: async () => undefined, + s3Sha256: async () => undefined, + }; await expect( - planArtifactCatalogUpdate({ - dockerfile: 'FROM supabase/gotrue:v1" AS gotrue\n', + refreshCatalogPin({ catalog: fixture, service: "postgrest", release: "v16.2-r1", io }), + ).rejects.toThrow(InvalidPayloadError); + }); + + test("rejects --upstream and --release given together", async () => { + const io: RevisionIo = { + listReleaseTags: async () => [], + fetchChecksums: async () => undefined, + imageDigest: async () => undefined, + s3Sha256: async () => undefined, + }; + + await expect( + refreshCatalogPin({ catalog: fixture, - publication: async () => published(), + service: "postgrest", + upstream: "v16.2", + release: "v16.2-r0", + io, }), ).rejects.toThrow(InvalidPayloadError); }); }); +describe("refreshCatalogPin backfills upstreamImage", () => { + test("resolves a derived service's upstreamImage from its per-target manifests", async () => { + const digests = nativeDigests("y"); + const io: RevisionIo = { + listReleaseTags: async () => ["analytics-v1.50.9-r0", "analytics-v1.50.9-r1"], + fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r1", digests), + imageDigest: async () => digest("z"), + s3Sha256: matchingS3("analytics", "v1.50.9-r1", digests), + fetchManifest: async () => JSON.stringify({ upstream_image: "supabase/logflare:1.50.9" }), + }; + + const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); + + expect(result.source).toContain('upstreamImage: "supabase/logflare:1.50.9"'); + }); + + test("falls back to source_image when a target's manifest has no upstream_image (image-derived build)", async () => { + const digests = nativeDigests("aa"); + const io: RevisionIo = { + listReleaseTags: async () => ["analytics-v1.50.9-r0"], + fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r0", digests), + imageDigest: async () => digest("ab"), + s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), + fetchManifest: async () => JSON.stringify({ source_image: "supabase/logflare:1.50.9" }), + }; + + const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); + + expect(result.source).toContain('upstreamImage: "supabase/logflare:1.50.9"'); + }); + + test("strips a docker.io prefix and a digest from the resolved upstreamImage", async () => { + const digests = nativeDigests("ac"); + const io: RevisionIo = { + listReleaseTags: async () => ["analytics-v1.50.9-r0"], + fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r0", digests), + imageDigest: async () => digest("ad"), + s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), + fetchManifest: async () => + JSON.stringify({ upstream_image: "docker.io/supabase/logflare:1.50.9@sha256:deadbeef" }), + }; + + const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); + + expect(result.source).toContain('upstreamImage: "supabase/logflare:1.50.9"'); + }); + + test("fails loudly when a derived service's manifests disagree across native targets", async () => { + const digests = nativeDigests("ae"); + let call = 0; + const io: RevisionIo = { + listReleaseTags: async () => ["analytics-v1.50.9-r0"], + fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r0", digests), + imageDigest: async () => digest("af"), + s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), + fetchManifest: async () => { + call += 1; + return JSON.stringify({ upstream_image: `supabase/logflare:1.50.${call}` }); + }, + }; + + await expect(refreshCatalogPin({ catalog: fixture, service: "analytics", io })).rejects.toThrow( + /manifests disagree/, + ); + }); + + test("resolves a mirrored service's upstreamImage from its oci-provenance source", async () => { + const digests = nativeDigests("ag"); + const io: RevisionIo = { + listReleaseTags: async () => ["vector-0.53.0-r0"], + fetchChecksums: async () => checksumsFor("vector", "0.53.0-r0", digests), + imageDigest: async () => digest("ah"), + s3Sha256: matchingS3("vector", "0.53.0-r0", digests), + fetchProvenance: async () => + JSON.stringify({ source: "docker.io/timberio/vector:0.53.0-alpine" }), + }; + + const result = await refreshCatalogPin({ catalog: vectorFixture, service: "vector", io }); + + expect(result.source).toContain('upstreamImage: "timberio/vector:0.53.0-alpine"'); + }); + + test("leaves upstreamImage absent when io has no manifest/provenance fetchers", async () => { + const digests = nativeDigests("ai"); + const io: RevisionIo = { + listReleaseTags: async () => ["analytics-v1.50.9-r0"], + fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r0", digests), + imageDigest: async () => digest("aj"), + s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), + }; + + const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); + + expect(result.source).not.toContain("upstreamImage"); + }); + + test("rejects a manifest upstream_image carrying a quote or template expression, writing nothing", async () => { + const digests = nativeDigests("ak"); + const io: RevisionIo = { + listReleaseTags: async () => ["analytics-v1.50.9-r0"], + fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r0", digests), + imageDigest: async () => digest("al"), + s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), + fetchManifest: async () => + JSON.stringify({ upstream_image: 'supabase/logflare:1.50.9"] }; import("evil"); //' }), + }; + + await expect(refreshCatalogPin({ catalog: fixture, service: "analytics", io })).rejects.toThrow( + InvalidPayloadError, + ); + }); +}); + describe("against the real catalog", () => { - test("every slim Dockerfile alias is a catalog entry", async () => { - const dockerfile = await Bun.file("apps/cli/src/shared/services/Dockerfile").text(); + test("a real catalog entry survives real formatting and can be refreshed again", async () => { const catalog = await Bun.file(CATALOG_PATH).text(); - const plan = await planArtifactCatalogUpdate({ - dockerfile, + const pinnedVersion = /definition\(\s*"auth",\s*\{\s*upstreamVersion:\s*"([^"]+)"/.exec( catalog, - publication: async () => published(`sha256:${"a".repeat(64)}`), + )?.[1]; + if (pinnedVersion === undefined) throw new Error("auth pin not found in the real catalog"); + + const first = nativeDigests("q"); + const firstIo: RevisionIo = { + listReleaseTags: async () => [`auth-${pinnedVersion}-r0`], + fetchChecksums: async () => checksumsFor("auth", `${pinnedVersion}-r0`, first), + imageDigest: async () => digest("r"), + s3Sha256: matchingS3("auth", `${pinnedVersion}-r0`, first), + }; + const written = await refreshCatalogPin({ catalog, service: "auth", io: firstIo }); + expect(written.update?.revision).toBe(0); + + const formatted = await formatWithOxfmt(written.source); + expect(formatted).toContain(`upstreamVersion: "${pinnedVersion}"`); + + const second = nativeDigests("s"); + const secondIo: RevisionIo = { + listReleaseTags: async () => [`auth-${pinnedVersion}-r0`, `auth-${pinnedVersion}-r1`], + fetchChecksums: async () => checksumsFor("auth", `${pinnedVersion}-r1`, second), + imageDigest: async () => digest("t"), + s3Sha256: matchingS3("auth", `${pinnedVersion}-r1`, second), + }; + const refreshed = await refreshCatalogPin({ + catalog: formatted, + service: "auth", + io: secondIo, }); - expect( - plan.skipped.filter((skip) => skip.blocking && !skip.reason.includes("older than")), - ).toEqual([]); + expect(refreshed.update).toEqual({ + service: "auth", + version: pinnedVersion, + revision: 1, + previousVersion: pinnedVersion, + target: "default", + }); + expect(refreshed.source).toContain("revision: 1"); + expect(refreshed.source).toContain(second["darwin-arm64"].manifest); }); }); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index 838f2c5b75..0ec85a6b02 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -1,28 +1,145 @@ /** - * Pins `packages/stack/src/Artifacts.ts` to the slim workloads named by - * `apps/cli/src/shared/services/Dockerfile`. Native archive URLs are derived - * from service + version. An entry that already carries a digest keeps one: - * the published `ghcr.io/supabase/cli/:` manifest digest. + * Pins `packages/stack/src/Artifacts.ts` to committed `supabase/slim-services` + * revisions (`-r`). Every entry is pinned by content: the GHCR + * image digest, plus an archive and manifest sha256 per native target. * - * Run: `bun .github/scripts/sync-artifacts-catalog.ts [base-dockerfile]` + * The catalog is the single version table (supabase/cli#6883): the Dockerfile's + * slim-capable lines are a generated view of it + * (`apps/cli/scripts/render-service-dockerfile.ts`), never the other way + * around. Updates land through two modes: + * + * Manual mode: refreshes one catalog entry, either to a specific committed + * release (`--release`) or to the highest committed revision of a given + * upstream version, or of its currently pinned upstream version when neither + * is given. `--upstream` and `--release` are mutually exclusive. + * + * bun .github/scripts/sync-artifacts-catalog.ts --service [--upstream | --release -r] + * + * Plan-updates mode (used by the `slim-release-published` dispatch workflow): + * lists a service's committed slim-services releases and computes, per + * release line, the hotfix and/or upgrade a workflow should apply. Pure + * planning: `planSlimUpdates` takes the release tag list as an argument, + * makes no network calls, and never logs — it returns `{ updates, warnings }`. + * Records go only to `--output ` (never stdout); warnings print to + * stdout as `::warning ::…` lines, so the two channels can't corrupt one + * another when a caller redirects stdout separately from the records file. + * + * bun .github/scripts/sync-artifacts-catalog.ts plan-updates --service \ + * --output [--format lines] [--expect-release -r] + * + * `--expect-release` handles the releases API lagging behind the dispatch that triggered this + * run: before planning, the listed committed tags must include `-`, or + * this mode re-lists a bounded number of times before giving up (`waitForExpectedRelease`). + * + * Validate-payload mode (used by the same workflow, before anything else): + * checks an untrusted `slim-release-published` dispatch payload against + * anchored charsets and prints it back as `key=value` lines, so a value that + * fails validation is never written to `$GITHUB_OUTPUT` in the first place. + * + * bun .github/scripts/sync-artifacts-catalog.ts validate-payload --service \ + * --upstream --revision --release */ -import { parseDockerfileServiceImages } from "../../apps/cli/src/shared/services/parse-dockerfile-service-images.ts"; -import { isOrioleImage, slimCatalogPin } from "../../apps/cli/src/shared/services/slim-images.ts"; import { DIGEST_PATTERN, InvalidPayloadError, SOURCE_REGISTRY, - VERSION_PATTERN, + checksumFor, escapeRegExp, + nativeFileNames, + nativeObjectUrl, } from "./slim-mirror-payload.ts"; export const CATALOG_PATH = "packages/stack/src/Artifacts.ts"; -const DOCKERFILE_PATH = "apps/cli/src/shared/services/Dockerfile"; -const NATIVE_RELEASES = "https://api.github.com/repos/supabase/slim-services/releases/tags"; const SLIM_IMAGE_PREFIX = `${SOURCE_REGISTRY}/`; +/** The three native targets the catalog pins per revision. Kept self-contained; see module docs. */ +const NATIVE_TARGETS = ["darwin-arm64", "linux-amd64", "linux-arm64"] as const; +type NativeTargetName = (typeof NATIVE_TARGETS)[number]; + +/** Overridable so an integration test can point at a local fixture server instead of GitHub. */ +const RELEASES_API = + process.env.SLIM_SERVICES_RELEASES_API ?? + "https://api.github.com/repos/supabase/slim-services/releases"; +const RELEASE_DOWNLOAD_BASE = "https://github.com/supabase/slim-services/releases/download"; + +/** Bounded retry for `waitForExpectedRelease`: 6 attempts, 10s apart, by default. */ +const EXPECT_RELEASE_ATTEMPTS = 6; +const DEFAULT_EXPECT_RELEASE_INTERVAL_MS = 10_000; +/** Bounded retry for the S3-mirror wait in `resolveRevisionPin`: ~10 min, covering a native upload of all three targets (`mirror-slim-image.yml`'s `upload-natives-s3`). */ +const S3_WAIT_ATTEMPTS = 20; +const DEFAULT_S3_WAIT_INTERVAL_MS = 30_000; +const MAX_TIMER_DELAY_MS = 2 ** 31 - 1; // setTimeout's own ceiling. + +/** + * Parses a millisecond wait-interval override from `envVar` fresh on every call, not once at + * module load, so a test can set it right before spawning. Unset keeps `fallback`; anything else + * must be a non-negative integer of at most `MAX_TIMER_DELAY_MS`, or this throws. + */ +function waitIntervalMs(envVar: string, fallback: number): number { + const raw = process.env[envVar]; + if (raw === undefined) return fallback; + const value = Number(raw); + if (!/^(0|[1-9][0-9]*)$/.test(raw) || value > MAX_TIMER_DELAY_MS) { + throw new InvalidPayloadError( + `invalid ${envVar} ${JSON.stringify(raw)}: expected a non-negative integer of at most ${MAX_TIMER_DELAY_MS}`, + ); + } + return value; +} + +const expectReleaseIntervalMs = (): number => + waitIntervalMs("SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS", DEFAULT_EXPECT_RELEASE_INTERVAL_MS); +const s3WaitIntervalMs = (): number => + waitIntervalMs("SLIM_UPDATES_S3_WAIT_MS", DEFAULT_S3_WAIT_INTERVAL_MS); + +/** Real delay, for a caller that didn't override `wait`. */ +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +type BoundedCheck = + | { readonly kind: "done"; readonly value: T } + | { readonly kind: "retry" } + | { readonly kind: "failed"; readonly message: string }; + +type BoundedWaitOutcome = + | { readonly status: "done"; readonly value: T } + | { readonly status: "timed-out" } + | { readonly status: "failed"; readonly message: string }; + +/** + * Generic bounded retry-with-wait: `waitForExpectedRelease`'s release-visibility wait and + * `resolveRevisionPin`'s S3-mirror wait both build on this. Calls `check` up to `attempts` + * times, `wait`-ing `intervalMs` between, until it reports `"done"` or an unrecoverable + * `"failed"` (never waited out). Exhausting every attempt on `"retry"` yields `"timed-out"`. + */ +async function boundedWait( + check: () => Promise>, + wait: (ms: number) => Promise, + attempts: number, + intervalMs: number, +): Promise> { + for (let attempt = 0; attempt < attempts; attempt++) { + const result = await check(); + if (result.kind === "done") return { status: "done", value: result.value }; + if (result.kind === "failed") return { status: "failed", message: result.message }; + if (attempt < attempts - 1) await wait(intervalMs); + } + return { status: "timed-out" }; +} + +/** + * A GitHub Actions workflow-command line, `message` run through the workflow-command data + * encoding (`%` first, so encoding `\r`/`\n` never gets re-escaped) — the boundary every + * `::error ::`/`::warning ::` this script emits goes through. + */ +function workflowCommand(kind: "error" | "warning", message: string): string { + const encoded = message.replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A"); + return `::${kind} ::${encoded}`; +} + /** Leading numeric component, `v` stripped. Only postgres carries more than one line. */ function releaseLine(version: string): string { const withoutPrefix = version.replace(/^[vV]/, ""); @@ -30,369 +147,1240 @@ function releaseLine(version: string): string { return separator === -1 ? withoutPrefix : withoutPrefix.slice(0, separator); } -/** True when every numeric prefix of `next` is older than `current`. Equal prefixes are not older. */ -function isOlderRelease(next: string, current: string): boolean { - const nextParts = next.replace(/^[vV]/, "").split("."); - const currentParts = current.replace(/^[vV]/, "").split("."); - const count = Math.max(nextParts.length, currentParts.length); - for (let index = 0; index < count; index++) { - const nextValue = leadingInteger(nextParts[index] ?? "0"); - const currentValue = leadingInteger(currentParts[index] ?? "0"); - if (nextValue === undefined || currentValue === undefined) return false; - if (nextValue !== currentValue) return nextValue < currentValue; - } - return false; +/** + * Matches a `--r` release tag. With `upstream` given, matches only that + * exact upstream (what `resolveRevisionPin` needs); with it omitted, captures any upstream as + * group 1 and the revision as group 2 (what the planner needs to enumerate every committed + * revision of every upstream a service carries). A tag with no `-r` suffix — legacy — never + * matches either shape. + */ +function releaseTagPattern(service: string, upstream?: string): RegExp { + const upstreamPart = upstream === undefined ? "(.+)" : escapeRegExp(upstream); + return new RegExp(`^${escapeRegExp(service)}-${upstreamPart}-r(0|[1-9][0-9]*)$`); +} + +/** + * Strips a leading `v`/`V` and one trailing `-sha-`, then parses the remainder as dot- + * separated integers. Returns undefined when the remainder isn't `^\d+(\.\d+)*$` — a version that + * isn't comparable this way, such as an OrioleDB-style suffix. + */ +function comparableVersion(version: string): ReadonlyArray | undefined { + const stripped = version.replace(/^[vV]/, "").replace(/-sha-[0-9a-f]+$/i, ""); + if (!/^\d+(\.\d+)*$/.test(stripped)) return undefined; + return stripped.split(".").map(Number); } -function leadingInteger(part: string): number | undefined { - const match = /^(\d+)/.exec(part); - return match === null ? undefined : Number(match[1]); +/** + * Numeric ordering of two upstream versions: `-1` when `a` is older, `0` when neither is newer + * (including two versions that only differ in a stripped `-sha-` suffix, e.g. two Studio + * builds on the same date), `1` when `a` is newer. Undefined when either isn't comparable. + */ +function compareVersions(a: string, b: string): number | undefined { + const av = comparableVersion(a); + const bv = comparableVersion(b); + if (av === undefined || bv === undefined) return undefined; + const length = Math.max(av.length, bv.length); + for (let index = 0; index < length; index++) { + const left = av[index] ?? 0; + const right = bv[index] ?? 0; + if (left !== right) return left < right ? -1 : 1; + } + return 0; } export interface CatalogPinUpdate { readonly service: string; readonly version: string; + readonly revision: number; readonly previousVersion: string; readonly target: "default" | "additional"; } -export interface SkippedCatalogPin { - readonly alias: string; - readonly reason: string; - /** OrioleDB has no slim image and must not fail the other pins. */ - readonly blocking: boolean; +/** Content pin for one resolved `-r` revision, ready to serialize into the catalog. */ +interface ResolvedPin { + readonly upstreamVersion: string; + readonly revision: number; + readonly image: string; + /** + * The pin's `ArtifactPin.upstreamImage`. Populated by manual mode (`refreshCatalogPin`) via + * `resolveUpstreamImage` below, when `io` carries `fetchManifest`/`fetchProvenance`. + */ + readonly upstreamImage?: string; + readonly natives: Readonly< + Record + >; } -export interface CatalogPlan { - readonly source: string; - readonly updates: ReadonlyArray; - readonly skipped: ReadonlyArray; +export type RevisionResolution = + | { readonly status: "resolved"; readonly pin: ResolvedPin } + | { + readonly status: "missing" | "incomplete" | "stale" | "lookup-failed"; + readonly message: string; + }; + +/** + * Network ports the resolver needs: the release list (for revision allocation), a release's + * `SHA256SUMS` body, the GHCR manifest digest, and a byte source's sha256. Tests stub these + * directly. + */ +export interface RevisionIo { + /** Tags of every published, non-draft release — a draft is not yet a committed revision. */ + readonly listReleaseTags: () => Promise>; + readonly fetchChecksums: (service: string, releaseVersion: string) => Promise; + readonly imageDigest: (service: string, releaseVersion: string) => Promise; + readonly s3Sha256: (url: string) => Promise; + /** Overridable real delay between the S3-mirror wait's bounded attempts. */ + readonly wait?: (ms: number) => Promise; + /** + * Raw contents of a native target's `.manifest.json` release asset, for a derived service's + * `upstream_image` (or `source_image` on an image-derived build, e.g. postgrest's Linux + * targets). Optional: only manual mode's `upstreamImage` backfill (`resolveUpstreamImage`) + * calls this, so a test `io` that doesn't exercise that path can omit it. + */ + readonly fetchManifest?: ( + service: string, + releaseVersion: string, + target: NativeTargetName, + ) => Promise; + /** + * Raw contents of a mirrored service's `-.oci-provenance.json` + * release asset, whose `source` field is the mirrored upstream image. Optional for the same + * reason as `fetchManifest`. + */ + readonly fetchProvenance?: ( + service: string, + releaseVersion: string, + upstreamVersion: string, + ) => Promise; } -export type ReleasePublication = - | { readonly status: "published"; readonly digest?: string } - | { readonly status: "missing" | "lookup-failed" }; +function desiredImage(service: string, releaseVersion: string, digest: string): string { + if (!DIGEST_PATTERN.test(digest)) { + throw new InvalidPayloadError(`missing slim digest for ${service}:${releaseVersion}`); + } + return `${SLIM_IMAGE_PREFIX}${service}:${releaseVersion}@${digest}`; +} -/** `definition("", "", ""`. */ -function defaultEntryPattern(service: string): RegExp { - const s = escapeRegExp(service); - return new RegExp( - `(definition\\(\\s*"${s}",\\s*")([^"]+)("\\s*,\\s*")(${escapeRegExp(SLIM_IMAGE_PREFIX)}${s}:[^"]+)(")`, - ); +/** + * Resolves `service`'s committed `-r` revision, pinned by content: the GHCR + * manifest digest, and every native target's archive and manifest sha256, cross-checked against + * the S3 mirror copy. + * + * With `requiredRevision` given, that exact revision must already be committed — this is what + * pins exactly a planned release (`--release`), never "highest at apply time". Without it, the + * highest committed revision of `upstream` is used (`--upstream`, and the hotfix/upgrade default). + */ +export async function resolveRevisionPin( + service: string, + upstream: string, + io: RevisionIo, + requiredRevision?: number, +): Promise { + const tagPattern = releaseTagPattern(service, upstream); + const seenRevisions = new Set(); + let highest: number | undefined; + for (const tag of await io.listReleaseTags()) { + const match = tagPattern.exec(tag); + if (match === null) continue; + const revision = Number(match[1]); + seenRevisions.add(revision); + if (highest === undefined || revision > highest) highest = revision; + } + + let target: number; + if (requiredRevision !== undefined) { + if (!seenRevisions.has(requiredRevision)) { + return { + status: "missing", + message: `${service}:${upstream}-r${requiredRevision} is not a committed slim-services release.`, + }; + } + target = requiredRevision; + } else { + if (highest === undefined) { + return { + status: "missing", + message: `${service}:${upstream} has no published slim-services revision.`, + }; + } + target = highest; + } + + const releaseVersion = `${upstream}-r${target}`; + const checksums = await io.fetchChecksums(service, releaseVersion); + if (checksums === undefined) { + return { + status: "incomplete", + message: `${service}-${releaseVersion} has no SHA256SUMS asset.`, + }; + } + + const natives: Record = {}; + for (const target of NATIVE_TARGETS) { + const files = nativeFileNames(service, releaseVersion, target); + const archive = checksumFor(checksums, files.archive); + const manifest = checksumFor(checksums, files.manifest); + if (archive === undefined || manifest === undefined) { + return { + status: "incomplete", + message: `${service}-${releaseVersion} SHA256SUMS has no line for ${ + archive === undefined ? files.archive : files.manifest + }.`, + }; + } + natives[target] = { archive, manifest }; + } + + const digest = await io.imageDigest(service, releaseVersion); + if (digest === undefined || !DIGEST_PATTERN.test(digest)) { + return { + status: "lookup-failed", + message: `${SLIM_IMAGE_PREFIX}${service}:${releaseVersion} has no published manifest.`, + }; + } + + // `mirror-slim-image.yml`'s S3 upload runs in parallel, best-effort, so a freshly committed + // revision's S3 copy can briefly lag GitHub: a missing object waits, bounded; a present object + // with the wrong bytes fails immediately instead of waiting. + for (const target of NATIVE_TARGETS) { + const files = nativeFileNames(service, releaseVersion, target); + for (const part of ["archive", "manifest"] as const) { + const url = nativeObjectUrl(service, releaseVersion, files[part]); + const expected = natives[target]?.[part] as string; + const outcome = await boundedWait( + async () => { + const actual = await io.s3Sha256(url); + if (actual === undefined) { + console.log( + `Waiting for the S3 mirror of ${service}-${releaseVersion} ${target} (${part})...`, + ); + return { kind: "retry" }; + } + if (actual !== expected) { + return { + kind: "failed", + message: `S3 copy of ${releaseVersion} ${target} (${part}) is stale (digest mismatch); run the slim-services mirror backfill.`, + }; + } + return { kind: "done", value: true }; + }, + io.wait ?? sleep, + S3_WAIT_ATTEMPTS, + s3WaitIntervalMs(), + ); + if (outcome.status === "failed") { + return { status: "stale", message: outcome.message }; + } + if (outcome.status === "timed-out") { + return { + status: "stale", + message: `S3 copy of ${releaseVersion} ${target} (${part}) never appeared; the S3 mirror hasn't finished — check mirror-slim-image.yml for this release, backfill if needed, and re-run.`, + }; + } + } + } + + return { + status: "resolved", + pin: { + upstreamVersion: upstream, + revision: target, + image: desiredImage(service, releaseVersion, digest), + natives: natives as Record, + }, + }; } -/** Additional release entries: `"": ""`. */ -function additionalEntryPattern(service: string, version?: string): RegExp { - const s = escapeRegExp(service); - const key = version === undefined ? `[^"]+` : escapeRegExp(version); - return new RegExp( - `"(${key})"(\\s*:\\s*)"(${escapeRegExp(SLIM_IMAGE_PREFIX)}${s}:[^"]+)"`, - version === undefined ? "g" : "", - ); +/** + * Services slim-services mirrors from an unmodified upstream image rather than building from + * source: their `upstreamImage` comes from the release's `oci-provenance.json` `source` field, + * not a native target's manifest. Keep this in sync with `SlimServicesSource`'s mirror-mode + * services. + */ +const MIRROR_MODE_SOURCE_SERVICES: ReadonlySet = new Set(["vector", "mailpit", "imgproxy"]); + +function parseJsonRecord(raw: string): Record | undefined { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return undefined; + } + return typeof parsed === "object" && parsed !== null + ? (parsed as Record) + : undefined; +} + +function stringField(record: Record | undefined, key: string): string | undefined { + const value = record?.[key]; + return typeof value === "string" && value.length > 0 ? value : undefined; } -function imageHasDigest(image: string): boolean { - return /@sha256:[0-9a-f]{64}$/.test(image); +/** Strips a `docker.io/` prefix and any `@sha256:…` digest, to match the Dockerfile's `FROM` form. */ +function normalizeUpstreamImage(image: string): string { + const withoutDigest = image.split("@")[0] ?? image; + return withoutDigest.startsWith("docker.io/") + ? withoutDigest.slice("docker.io/".length) + : withoutDigest; } -function desiredImage( +/** + * A normalized `upstreamImage`: one or more lowercase `registry`/`repository` path segments + * (each `[a-z0-9]`, optionally separated internally by `.`/`_`/`-`), a `:`, and a tag matching + * Docker's own tag grammar. Anchored, so no whitespace, quote, or template/expression syntax can + * slip through — this value gets embedded as a TypeScript string literal in `Artifacts.ts`. + */ +const IMAGE_REFERENCE_PATTERN = + /^[a-z0-9]+(?:[._-][a-z0-9]+)*(?:\/[a-z0-9]+(?:[._-][a-z0-9]+)*)*:[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$/; + +/** Validates a normalized `upstreamImage` before it's ever written to `Artifacts.ts`. */ +function validateUpstreamImage(image: string, context: string): string { + if (!IMAGE_REFERENCE_PATTERN.test(image)) { + throw new InvalidPayloadError( + `${context} has an invalid upstreamImage ${JSON.stringify(image)}.`, + ); + } + return image; +} + +/** + * Resolves `service`'s `upstreamImage` for the release `releaseVersion` (`-r`): + * a mirrored service's `oci-provenance.json` `source`, or a derived service's per-target + * manifest `upstream_image` (falling back to `source_image` for an image-derived build, e.g. + * postgrest's Linux targets) — cross-checked across every native target, so a derived service + * whose manifests disagree fails instead of silently picking one. Only manual mode + * (`refreshCatalogPin`) calls this; `io` must carry `fetchManifest`/`fetchProvenance`. + */ +async function resolveUpstreamImage( service: string, - version: string, - currentImage: string, - digest: string, -): string { - const tagged = `${SLIM_IMAGE_PREFIX}${service}:${version}`; - if (!imageHasDigest(currentImage)) return tagged; - if (!DIGEST_PATTERN.test(digest)) { - throw new InvalidPayloadError(`missing slim digest for ${service}:${version}`); + releaseVersion: string, + upstreamVersion: string, + io: RevisionIo, +): Promise { + if (MIRROR_MODE_SOURCE_SERVICES.has(service)) { + if (io.fetchProvenance === undefined) { + throw new InvalidPayloadError( + `${service} needs an io.fetchProvenance to resolve upstreamImage.`, + ); + } + const provenance = await io.fetchProvenance(service, releaseVersion, upstreamVersion); + if (provenance === undefined) { + throw new InvalidPayloadError(`${service}-${releaseVersion} has no oci-provenance asset.`); + } + const source = stringField(parseJsonRecord(provenance), "source"); + if (source === undefined) { + throw new InvalidPayloadError( + `${service}-${releaseVersion} oci-provenance has no 'source' field.`, + ); + } + return validateUpstreamImage(normalizeUpstreamImage(source), `${service}-${releaseVersion}`); + } + + if (io.fetchManifest === undefined) { + throw new InvalidPayloadError(`${service} needs an io.fetchManifest to resolve upstreamImage.`); + } + const values = new Set(); + for (const target of NATIVE_TARGETS) { + const manifest = await io.fetchManifest(service, releaseVersion, target); + if (manifest === undefined) { + throw new InvalidPayloadError( + `${service}-${releaseVersion}-${target} has no manifest asset.`, + ); + } + const record = parseJsonRecord(manifest); + const value = stringField(record, "upstream_image") ?? stringField(record, "source_image"); + if (value === undefined) { + throw new InvalidPayloadError( + `${service}-${releaseVersion}-${target} manifest has neither 'upstream_image' nor 'source_image'.`, + ); + } + values.add(normalizeUpstreamImage(value)); + } + if (values.size !== 1) { + throw new InvalidPayloadError( + `${service}-${releaseVersion} manifests disagree on the upstream image: ${[...values] + .sort() + .map((value) => JSON.stringify(value)) + .join(", ")}.`, + ); } - return `${tagged}@${digest}`; + return validateUpstreamImage([...values][0] as string, `${service}-${releaseVersion}`); +} + +/** + * Serializes a resolved pin into the object literal `Artifacts.ts` embeds, in catalog order. + * Every string field goes through `JSON.stringify`, not manual `"${…}"` interpolation — this + * text is written straight into TypeScript source that later gets imported, so an unescaped + * quote or template expression in any field (release metadata included) would inject code. + */ +function serializePin(pin: ResolvedPin): string { + const natives = NATIVE_TARGETS.map((target) => { + const native = pin.natives[target]; + return `${JSON.stringify(target)}: { archive: ${JSON.stringify(native.archive)}, manifest: ${JSON.stringify(native.manifest)} }`; + }).join(", "); + const upstreamImage = + pin.upstreamImage === undefined ? "" : ` upstreamImage: ${JSON.stringify(pin.upstreamImage)},`; + return `{ upstreamVersion: ${JSON.stringify(pin.upstreamVersion)}, revision: ${pin.revision}, image: ${JSON.stringify(pin.image)},${upstreamImage} natives: { ${natives} } }`; +} + +interface PinSpan { + readonly start: number; + readonly end: number; + readonly version: string; + /** + * Span of an additional (release-line-keyed) pin's own string key, content only (no quotes). + * Absent for the default pin, which carries no separate key to keep in sync. + */ + readonly key?: { readonly start: number; readonly end: number }; +} + +/** + * Index right after the matching close-bracket for the open-bracket character at `openIndex` + * (which must itself be `open`). Skips string contents, so a formatter's line-wrapping, trailing + * commas, or reordered properties never confuse the boundary — only bracket balance matters. + */ +function scanBalanced(source: string, openIndex: number, open: string, close: string): number { + let depth = 0; + let index = openIndex; + for (; index < source.length; index++) { + const ch = source[index]; + if (ch === '"' || ch === "'" || ch === "`") { + const quote = ch; + index++; + while (index < source.length && source[index] !== quote) { + if (source[index] === "\\") index++; + index++; + } + continue; + } + if (ch === open) depth++; + else if (ch === close) { + depth--; + if (depth === 0) return index + 1; + } + } + throw new InvalidPayloadError(`unterminated '${open}' while parsing ${CATALOG_PATH}`); +} + +/** Loosely finds `upstreamVersion` anywhere inside a resolved pin literal's text. */ +const PIN_UPSTREAM_VERSION = /upstreamVersion:\s*"([^"]+)"/; +/** Loosely finds `revision` anywhere inside a resolved pin literal's text. */ +const PIN_REVISION = /revision:\s*(\d+)/; + +/** + * Matches a resolved `ArtifactPin` object literal starting exactly at `index`. The span is found + * by bracket balance, then the version is pulled out with a loose field search — so a formatter's + * whitespace, line breaks, trailing commas, or property order never break matching, only the + * literal shape itself would. + */ +function matchPinAt(source: string, index: number): PinSpan | undefined { + if (source[index] === "{") { + const end = scanBalanced(source, index, "{", "}"); + const version = PIN_UPSTREAM_VERSION.exec(source.slice(index, end))?.[1]; + return version === undefined ? undefined : { start: index, end, version }; + } + return undefined; +} + +interface ServicePins { + readonly defaultPin: PinSpan; + readonly additional: ReadonlyArray; +} + +/** + * Finds every pin expression `service` carries in `source`: the `definition("", , + * ...)` default pin, plus any additional (release-line-keyed) pins in its trailing object + * argument. Both `selectEntry` and `planSlimUpdates` build on this single traversal. + */ +function collectServicePins(source: string, service: string): ServicePins | undefined { + const prefix = new RegExp(`definition\\(\\s*"${escapeRegExp(service)}"\\s*,\\s*`); + const prefixMatch = prefix.exec(source); + if (prefixMatch === null) return undefined; + const pinIndex = prefixMatch.index + prefixMatch[0].length; + const defaultPin = matchPinAt(source, pinIndex); + if (defaultPin === undefined) return undefined; + + const openParenIndex = prefixMatch.index + prefixMatch[0].indexOf("("); + const callEnd = scanBalanced(source, openParenIndex, "(", ")"); + + const additional: PinSpan[] = []; + const keyPattern = /"([^"]+)"\s*:\s*/g; + keyPattern.lastIndex = defaultPin.end; + for ( + let keyMatch = keyPattern.exec(source); + keyMatch !== null; + keyMatch = keyPattern.exec(source) + ) { + if (keyMatch.index >= callEnd) break; + const valueStart = keyMatch.index + keyMatch[0].length; + const pin = matchPinAt(source, valueStart); + if (pin === undefined) { + keyPattern.lastIndex = valueStart; + continue; + } + const keyText = keyMatch[1] ?? ""; + const keyStart = keyMatch.index + keyMatch[0].indexOf(keyText); + additional.push({ ...pin, key: { start: keyStart, end: keyStart + keyText.length } }); + keyPattern.lastIndex = pin.end; + } + + return { defaultPin, additional }; } type SelectedEntry = - | { readonly kind: "default" | "additional"; readonly version: string; readonly image: string } + | { readonly kind: "default" | "additional"; readonly version: string; readonly span: PinSpan } | { readonly kind: "unmodelled-service" } | { readonly kind: "unmodelled-release-line"; readonly known: ReadonlyArray }; -function selectEntry(source: string, service: string, version: string): SelectedEntry { - const defaultMatch = defaultEntryPattern(service).exec(source); - if (defaultMatch === null) return { kind: "unmodelled-service" }; +/** + * Locates `service`'s pin expression in `source`: the `definition("", , ...)` + * default pin, and, when `version` is given, whichever pin (default or additional) sits on its + * release line. With no `version`, returns the default pin unconditionally. + */ +function selectEntry(source: string, service: string, version: string | undefined): SelectedEntry { + const pins = collectServicePins(source, service); + if (pins === undefined) return { kind: "unmodelled-service" }; + const { defaultPin, additional } = pins; + + if (version === undefined) { + return { kind: "default", version: defaultPin.version, span: defaultPin }; + } - const currentDefaultVersion = defaultMatch[2] ?? ""; - const currentDefaultImage = defaultMatch[4] ?? ""; - const additional = [...source.matchAll(additionalEntryPattern(service))].map((match) => ({ - version: match[1] ?? "", - image: match[3] ?? "", - })); const bumpsDefault = - additional.length === 0 || releaseLine(version) === releaseLine(currentDefaultVersion); + additional.length === 0 || releaseLine(version) === releaseLine(defaultPin.version); if (bumpsDefault) { - return { kind: "default", version: currentDefaultVersion, image: currentDefaultImage }; + return { kind: "default", version: defaultPin.version, span: defaultPin }; } - - const sameLine = additional.find((entry) => releaseLine(entry.version) === releaseLine(version)); + const sameLine = additional.find((pin) => releaseLine(pin.version) === releaseLine(version)); if (sameLine === undefined) { return { kind: "unmodelled-release-line", - known: [currentDefaultVersion, ...additional.map((entry) => entry.version)], + known: [defaultPin.version, ...additional.map((pin) => pin.version)], }; } - return { kind: "additional", version: sameLine.version, image: sameLine.image }; + return { kind: "additional", version: sameLine.version, span: sameLine }; } -function skipReason(alias: string, version: string, entry: SelectedEntry): string | undefined { - if (entry.kind === "unmodelled-service") { - return `${CATALOG_PATH} has no slim entry for ${alias}.`; +/** The `{service, upstream_version, revision, release_version}` payload a `slim-release-published` dispatch carries. */ +export interface SlimReleasePublishedPayload { + readonly service: string; + readonly upstream_version: string; + readonly revision: number; + readonly release_version: string; +} + +/** Every upstream tag format the catalog carries, including Studio's `2026.09.04-sha-5a67366`. */ +const UPSTREAM_VERSION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +const SERVICE_NAME_PATTERN = /^[a-z0-9-]+$/; +const RELEASE_VERSION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*-r(0|[1-9][0-9]*)$/; + +/** + * Validates an untrusted `slim-release-published` dispatch payload before any of its fields are + * written anywhere (a `$GITHUB_OUTPUT` line, a branch name, a PR title, …). Every field is checked + * against an anchored charset — `upstream_version` and `release_version` included, not just + * `service` and `revision` — so a value carrying a newline or shell/Actions metacharacter is + * rejected here instead of being echoed downstream. + */ +export function validateSlimReleasePublishedPayload(input: { + readonly service: string; + readonly upstream_version: string; + readonly revision: string; + readonly release_version: string; +}): SlimReleasePublishedPayload { + if (!SERVICE_NAME_PATTERN.test(input.service)) { + throw new InvalidPayloadError(`invalid service: ${JSON.stringify(input.service)}`); } - if (entry.kind === "unmodelled-release-line") { - return `${alias} ${version} is not on a release line ${CATALOG_PATH} carries (${entry.known.join(", ")}).`; + if (!UPSTREAM_VERSION_PATTERN.test(input.upstream_version)) { + throw new InvalidPayloadError( + `invalid upstream_version: ${JSON.stringify(input.upstream_version)}`, + ); } - return undefined; + if (!/^(0|[1-9][0-9]*)$/.test(input.revision)) { + throw new InvalidPayloadError(`invalid revision: ${JSON.stringify(input.revision)}`); + } + if (!RELEASE_VERSION_PATTERN.test(input.release_version)) { + throw new InvalidPayloadError( + `invalid release_version: ${JSON.stringify(input.release_version)}`, + ); + } + const revision = Number(input.revision); + const expected = `${input.upstream_version}-r${revision}`; + if (input.release_version !== expected) { + throw new InvalidPayloadError( + `release_version ${JSON.stringify(input.release_version)} does not match derived ${JSON.stringify(expected)}`, + ); + } + return { + service: input.service, + upstream_version: input.upstream_version, + revision, + release_version: input.release_version, + }; } -function slimVersions(dockerfile: string): ReadonlyMap { - const versions = new Map(); - for (const from of parseDockerfileServiceImages(dockerfile)) { - const pin = slimCatalogPin(from.alias, from.image); - if (pin !== undefined) versions.set(from.alias, pin.version); +const normalizeText = (text: string): string => text.replace(/\s+/g, " ").trim(); + +/** + * Writes `pin` over `entry`'s span in `source`, or leaves it unchanged when it already matches. + * An additional (release-line-keyed) pin also gets its own string key rewritten to `pin`'s + * `upstreamVersion` when it moves to a different upstream version, so the key an additional pin + * is looked up by never drifts from the `upstreamVersion` its resolved pin literal carries. + */ +function writePin( + source: string, + entry: Extract, + pin: ResolvedPin, +): { readonly source: string; readonly changed: boolean } { + const desired = serializePin(pin); + const current = source.slice(entry.span.start, entry.span.end); + let next = source; + let changed = false; + if (normalizeText(current) !== normalizeText(desired)) { + next = next.slice(0, entry.span.start) + desired + next.slice(entry.span.end); + changed = true; } - return versions; + const key = entry.span.key; + if (key !== undefined && source.slice(key.start, key.end) !== pin.upstreamVersion) { + next = next.slice(0, key.start) + pin.upstreamVersion + next.slice(key.end); + changed = true; + } + return { source: next, changed }; } -type PinResult = - | { - readonly kind: "updated"; - readonly source: string; - readonly previousVersion: string; - readonly target: "default" | "additional"; - } - | { readonly kind: "unchanged" }; +/** + * One hotfix or upgrade a `slim-release-published` run should apply, on one of `service`'s + * release lines. `line` is the leading numeric component (`releaseLine`), present only when the + * service carries more than one line (only postgres does today) — it's already folded into + * `branch`, so a caller never needs to consume it separately. + */ +export interface SlimUpdate { + readonly kind: "hotfix" | "upgrade"; + readonly line?: string; + readonly branch: string; + readonly title: string; + /** The release version pinned before this update, e.g. `v2.195.0-r0`. */ + readonly fromRelease: string; + readonly toUpstream: string; + /** The release version this update pins, e.g. `v2.195.0-r1`. */ + readonly toRelease: string; +} -function pinService( - source: string, +/** + * Builds one `SlimUpdate`, validating every value it emits — `service`, `toUpstream` and + * `toRelease` — against the same anchored patterns `validateSlimReleasePublishedPayload` uses, + * before any of them reaches a branch name or PR title. Release tag names (the ultimate source of + * `toUpstream`) come from an external API, so this check applies even to values derived from the + * catalog itself (`fromRelease`'s upstream), not only to values freshly parsed from a tag. + */ +function buildUpdate( + kind: "hotfix" | "upgrade", service: string, - version: string, - digest: string | undefined, -): PinResult { - const entry = selectEntry(source, service, version); - if (entry.kind !== "default" && entry.kind !== "additional") { - throw new InvalidPayloadError(`${service} ${version} is not a catalog entry.`); + line: string, + hasLines: boolean, + pinned: { readonly upstream: string; readonly revision: number }, + toUpstream: string, + toRevision: number, +): SlimUpdate { + if (!SERVICE_NAME_PATTERN.test(service)) { + throw new InvalidPayloadError(`invalid service: ${JSON.stringify(service)}`); } - if (imageHasDigest(entry.image) && (digest === undefined || !DIGEST_PATTERN.test(digest))) { - throw new InvalidPayloadError(`missing slim digest for ${service}:${version}`); + if (!UPSTREAM_VERSION_PATTERN.test(toUpstream)) { + throw new InvalidPayloadError(`invalid upstream version: ${JSON.stringify(toUpstream)}`); } - const desired = desiredImage(service, version, entry.image, digest ?? ""); - if (entry.version === version && entry.image === desired) return { kind: "unchanged" }; - - if (entry.kind === "default") { - return { - kind: "updated", - source: source.replace( - defaultEntryPattern(service), - (_full, prefix: string, _version: string, mid: string, _image: string, suffix: string) => - `${prefix}${version}${mid}${desired}${suffix}`, - ), - previousVersion: entry.version, - target: "default", - }; + const toRelease = `${toUpstream}-r${toRevision}`; + if (!RELEASE_VERSION_PATTERN.test(toRelease)) { + throw new InvalidPayloadError(`invalid release version: ${JSON.stringify(toRelease)}`); } + const fromRelease = `${pinned.upstream}-r${pinned.revision}`; + const suffix = hasLines ? `-${line}` : ""; + const branch = + kind === "hotfix" ? `slim-hotfix/${service}${suffix}` : `slim-bump/${service}${suffix}`; + const title = + kind === "hotfix" + ? `chore(stack): pin ${service} ${toRelease}` + : `chore(stack): bump ${service} to ${toRelease}`; return { - kind: "updated", - source: source.replace( - additionalEntryPattern(service, entry.version), - (_full, _key: string, separator: string) => `"${version}"${separator}"${desired}"`, - ), - previousVersion: entry.version, - target: "additional", + kind, + line: hasLines ? line : undefined, + branch, + title, + fromRelease, + toUpstream, + toRelease, }; } +export interface PlanSlimUpdatesResult { + readonly updates: ReadonlyArray; + /** `::warning ::…` workflow-command lines, for the caller to print to stdout. */ + readonly warnings: ReadonlyArray; +} + +/** Groups every candidate release tag under one bucket, for a service with no additional pins. */ +const SINGLE_LINE_KEY = "*"; + /** - * Rewrites `catalog` from Dockerfile tags that differ from `baseDockerfile`. - * With no base, every slim tag is in scope. A pin that cannot be applied is - * reported in `skipped`. OrioleDB is the only non-blocking skip. + * The hotfix and/or upgrade a `slim-release-published` run should apply for `service`, computed + * against `catalog`'s current pins and `releaseTags` (every committed — published, non-draft — + * slim-services release tag; the caller filters drafts before calling this). Pure: no network, no + * file I/O, no logging — every diagnostic comes back in `warnings` instead, so a caller (the CLI, + * or a test) decides where it goes. This matters because `plan-updates` writes `updates` straight + * into a file the workflow parses as records; a `console.log`'d warning on the same stdout the + * workflow captures would corrupt that file instead of just being informational. + * + * Per release line (the default pin's line, plus one per additional pin — only postgres + * has more than one) a **hotfix** fires when the pinned upstream has a committed revision higher + * than the pinned one; an **upgrade** fires when the newest committed upstream on the line is + * newer than the pinned one (ties, e.g. two Studio builds dated the same day, are not newer). + * Both can fire in the same run. A service with no additional pins has exactly one line and + * accepts any comparable upstream on it — a Studio year rollover or a postgrest major bump is + * the same line moving forward, not a different one, so it is never filtered by `releaseLine`. + * Only a service that does carry additional pins (postgres) filters a release tag to the line its + * `releaseLine` names; a tag on no such line, or whose version isn't comparable, is warned about + * and ignored — it never causes a plan-updates run to fail. */ -export async function planArtifactCatalogUpdate(input: { - readonly dockerfile: string; - readonly baseDockerfile?: string; - readonly catalog: string; - readonly publication: (service: string, version: string) => Promise; -}): Promise { - let source = input.catalog; - const updates: CatalogPinUpdate[] = []; - const skipped: SkippedCatalogPin[] = []; - const baseVersions = - input.baseDockerfile === undefined ? undefined : slimVersions(input.baseDockerfile); - const changed = (alias: string, version: string | undefined): boolean => - baseVersions === undefined || baseVersions.get(alias) !== version; - - for (const from of parseDockerfileServiceImages(input.dockerfile)) { - if (isOrioleImage(from.image)) { - if (!changed(from.alias, undefined)) continue; - skipped.push({ - alias: from.alias, - reason: `${from.alias} ${from.image} has no slim image.`, - blocking: false, - }); - continue; +export function planSlimUpdates( + catalog: string, + service: string, + releaseTags: ReadonlyArray, +): PlanSlimUpdatesResult { + const warnings: string[] = []; + const pins = collectServicePins(catalog, service); + if (pins === undefined) { + warnings.push( + workflowCommand( + "warning", + `${CATALOG_PATH} has no slim entry for ${service}; nothing to plan.`, + ), + ); + return { updates: [], warnings }; + } + + const allPins = [pins.defaultPin, ...pins.additional]; + const hasLines = pins.additional.length > 0; + + const pinnedByLine = new Map(); + for (const span of allPins) { + const line = hasLines ? releaseLine(span.version) : SINGLE_LINE_KEY; + const text = catalog.slice(span.start, span.end); + const revisionMatch = PIN_REVISION.exec(text); + if (revisionMatch === null) { + throw new InvalidPayloadError( + `${CATALOG_PATH} has no revision for ${service} ${span.version}; a committed catalog always pins a resolved revision.`, + ); } - const pin = slimCatalogPin(from.alias, from.image); - if (pin === undefined || !changed(from.alias, pin.version)) continue; - if (!VERSION_PATTERN.test(pin.version)) { - throw new InvalidPayloadError(`invalid version for ${from.alias}: '${pin.version}'`); + pinnedByLine.set(line, { upstream: span.version, revision: Number(revisionMatch[1]) }); + } + + const tagPattern = releaseTagPattern(service); + const candidatesByLine = new Map>(); + for (const tag of releaseTags) { + const match = tagPattern.exec(tag); + if (match === null) continue; // not this service, or a legacy tag with no `-rN`: ignored. + const upstream = match[1] as string; + const revision = Number(match[2]); + let line: string; + if (hasLines) { + line = releaseLine(upstream); + if (!pinnedByLine.has(line)) { + warnings.push( + workflowCommand( + "warning", + `${service} ${upstream} is not on a release line ${CATALOG_PATH} carries for it; ignoring ${tag}.`, + ), + ); + continue; + } + } else { + line = SINGLE_LINE_KEY; } + const list = candidatesByLine.get(line) ?? []; + list.push({ upstream, revision }); + candidatesByLine.set(line, list); + } - const entry = selectEntry(source, pin.service, pin.version); - const reason = skipReason(from.alias, pin.version, entry); - if (reason !== undefined) { - skipped.push({ alias: from.alias, reason, blocking: true }); - continue; + const updates: SlimUpdate[] = []; + for (const [line, pinned] of pinnedByLine) { + const candidates = candidatesByLine.get(line) ?? []; + + const sameUpstreamRevisions = candidates + .filter((candidate) => candidate.upstream === pinned.upstream) + .map((candidate) => candidate.revision); + if (sameUpstreamRevisions.length > 0) { + const highest = Math.max(...sameUpstreamRevisions); + if (highest > pinned.revision) { + updates.push( + buildUpdate("hotfix", service, line, hasLines, pinned, pinned.upstream, highest), + ); + } } - if (entry.kind !== "default" && entry.kind !== "additional") continue; - if (isOlderRelease(pin.version, entry.version)) { - skipped.push({ - alias: from.alias, - reason: `${pin.service} ${pin.version} is older than the catalog pin ${entry.version}.`, - blocking: true, - }); - continue; + + const highestRevisionByUpstream = new Map(); + for (const candidate of candidates) { + const current = highestRevisionByUpstream.get(candidate.upstream); + if (current === undefined || candidate.revision > current) { + highestRevisionByUpstream.set(candidate.upstream, candidate.revision); + } } - if (entry.version === pin.version && !imageHasDigest(entry.image)) continue; - - const release = await input.publication(pin.service, pin.version); - if (release.status !== "published") { - skipped.push({ - alias: from.alias, - reason: - release.status === "missing" - ? `${pin.service}:${pin.version} has no published slim image and native release.` - : `${pin.service}:${pin.version} publication check failed.`, - blocking: true, - }); - continue; + + let bestUpstream: string | undefined; + for (const upstream of highestRevisionByUpstream.keys()) { + const comparedToPinned = compareVersions(upstream, pinned.upstream); + if (comparedToPinned === undefined) { + warnings.push( + workflowCommand( + "warning", + `${service} ${upstream} is not a comparable version; ignoring.`, + ), + ); + continue; + } + if (bestUpstream === undefined || (compareVersions(upstream, bestUpstream) ?? 0) > 0) { + bestUpstream = upstream; + } } - const digest = imageHasDigest(entry.image) ? release.digest : undefined; - if (imageHasDigest(entry.image) && (digest === undefined || !DIGEST_PATTERN.test(digest))) { - skipped.push({ - alias: from.alias, - reason: `${pin.service}:${pin.version} has no published slim manifest.`, - blocking: true, - }); - continue; + if (bestUpstream !== undefined && compareVersions(bestUpstream, pinned.upstream) === 1) { + const revision = highestRevisionByUpstream.get(bestUpstream) as number; + updates.push(buildUpdate("upgrade", service, line, hasLines, pinned, bestUpstream, revision)); } - const result = pinService(source, pin.service, pin.version, digest); - if (result.kind === "unchanged") continue; - source = result.source; - updates.push({ - service: pin.service, - version: pin.version, - previousVersion: result.previousVersion, - target: result.target, - }); } - return { source, updates, skipped }; + return { updates, warnings }; } -type Probe = "published" | "missing" | "lookup-failed"; +/** + * Reads every committed release tag through `listReleaseTags` and plans against `catalog`. The + * injectable lister is the seam a dry run or an end-to-end test uses to exercise the whole + * `plan-updates` transport — this function plus the CLI's file/stdout wiring — without a network + * call, the same pattern `RevisionIo.listReleaseTags` already uses. + */ +export async function planUpdatesForService(input: { + readonly catalog: string; + readonly service: string; + readonly listReleaseTags: () => Promise>; +}): Promise { + const releaseTags = await input.listReleaseTags(); + return planSlimUpdates(input.catalog, input.service, releaseTags); +} + +export interface WaitForExpectedReleaseResult { + readonly visible: boolean; + /** The last listing `listReleaseTags` returned, whichever attempt it came from. */ + readonly tags: ReadonlyArray; +} + +/** + * Eventual-consistency handling of the releases API lagging behind the `slim-release-published` + * dispatch that triggered this run: re-lists up to `attempts` times, `io.wait`-ing between + * attempts, until `-` appears. Not a test retry — a real, bounded wait + * for an external API to catch up. `io.wait` is the seam a unit test overrides to skip the real + * delay; an integration test instead shrinks the real delay via `SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS` + * (parsed fresh by `expectReleaseIntervalMs` on every call this default reaches), since a + * subprocess can't be handed a function. + */ +export async function waitForExpectedRelease( + service: string, + releaseVersion: string, + io: { + readonly listReleaseTags: () => Promise>; + readonly wait: (ms: number) => Promise; + }, + attempts: number = EXPECT_RELEASE_ATTEMPTS, + intervalMs: number = expectReleaseIntervalMs(), +): Promise { + const expectedTag = `${service}-${releaseVersion}`; + let tags: ReadonlyArray = []; + const outcome = await boundedWait( + async () => { + tags = await io.listReleaseTags(); + return tags.includes(expectedTag) ? { kind: "done", value: true } : { kind: "retry" }; + }, + io.wait, + attempts, + intervalMs, + ); + return { visible: outcome.status === "done", tags }; +} + +export interface CatalogRefreshResult { + readonly source: string; + readonly update?: CatalogPinUpdate; +} + +/** + * Refreshes one catalog entry: to exactly the committed release named by `release` (`-r`, + * required to already be committed — never "highest at apply time"), or to the highest committed + * revision of `upstream` (or, when both are omitted, of the entry's currently pinned upstream + * version). `upstream` and `release` are mutually exclusive. Pure aside from `io`: callers own + * reading and writing `Artifacts.ts`. + */ +export async function refreshCatalogPin(input: { + readonly catalog: string; + readonly service: string; + readonly upstream?: string; + readonly release?: string; + readonly io: RevisionIo; +}): Promise { + if (input.upstream !== undefined && input.release !== undefined) { + throw new InvalidPayloadError("--upstream and --release are mutually exclusive."); + } + if (input.upstream !== undefined && !UPSTREAM_VERSION_PATTERN.test(input.upstream)) { + throw new InvalidPayloadError(`invalid --upstream '${input.upstream}'`); + } + + let upstream = input.upstream; + let requiredRevision: number | undefined; + if (input.release !== undefined) { + if (!RELEASE_VERSION_PATTERN.test(input.release)) { + throw new InvalidPayloadError(`invalid --release '${input.release}'`); + } + const match = /^(.+)-r(0|[1-9][0-9]*)$/.exec(input.release) as RegExpExecArray; + upstream = match[1]; + requiredRevision = Number(match[2]); + } + + const entry = selectEntry(input.catalog, input.service, upstream); + if (entry.kind === "unmodelled-service") { + throw new InvalidPayloadError(`${CATALOG_PATH} has no slim entry for ${input.service}.`); + } + if (entry.kind === "unmodelled-release-line") { + throw new InvalidPayloadError( + `${input.service} ${upstream ?? ""} is not on a release line ${CATALOG_PATH} carries (${entry.known.join(", ")}).`, + ); + } + const resolvedUpstream = upstream ?? entry.version; + const resolution = await resolveRevisionPin( + input.service, + resolvedUpstream, + input.io, + requiredRevision, + ); + if (resolution.status !== "resolved") { + throw new InvalidPayloadError(resolution.message); + } + // Manual mode also backfills `upstreamImage`, so a plain `io` (most tests) can still exercise + // revision resolution without stubbing the extra fetchers. + const pin = + input.io.fetchManifest === undefined && input.io.fetchProvenance === undefined + ? resolution.pin + : { + ...resolution.pin, + upstreamImage: await resolveUpstreamImage( + input.service, + `${resolution.pin.upstreamVersion}-r${resolution.pin.revision}`, + resolvedUpstream, + input.io, + ), + }; + const written = writePin(input.catalog, entry, pin); + if (!written.changed) return { source: input.catalog }; + return { + source: written.source, + update: { + service: input.service, + version: resolvedUpstream, + revision: resolution.pin.revision, + previousVersion: entry.version, + target: entry.kind, + }, + }; +} + +function githubHeaders(): Record { + const headers: Record = { + Accept: "application/vnd.github+json", + "User-Agent": "supabase-cli-catalog-sync", + }; + const token = process.env.GITHUB_TOKEN; + if (token !== undefined && token !== "") headers.Authorization = `Bearer ${token}`; + return headers; +} -async function probeManifest(reference: string): Promise<{ probe: Probe; digest?: string }> { +/** Tags of every published, non-draft `supabase/slim-services` release. A draft is never a committed revision. */ +async function listReleaseTags(): Promise> { + const tags: string[] = []; + for (let page = 1; ; page++) { + const response = await fetch(`${RELEASES_API}?per_page=100&page=${page}`, { + headers: githubHeaders(), + }); + if (!response.ok) { + throw new InvalidPayloadError( + `slim-services releases list failed (HTTP ${response.status}).`, + ); + } + const batch: unknown = await response.json(); + if (!Array.isArray(batch)) { + throw new InvalidPayloadError("Malformed slim-services releases response."); + } + for (const item of batch) { + const record = item as { tag_name?: unknown; draft?: unknown } | null; + if (record?.draft === true) continue; + const tagName = record?.tag_name; + if (typeof tagName === "string") tags.push(tagName); + } + if (batch.length < 100) break; + } + return tags; +} + +async function fetchChecksums( + service: string, + releaseVersion: string, +): Promise { + const response = await fetch(`${RELEASE_DOWNLOAD_BASE}/${service}-${releaseVersion}/SHA256SUMS`); + if (response.status === 404) return undefined; + if (!response.ok) { + throw new InvalidPayloadError( + `SHA256SUMS download failed for ${service}-${releaseVersion} (HTTP ${response.status}).`, + ); + } + return response.text(); +} + +async function imageDigest(service: string, releaseVersion: string): Promise { + const reference = `${SLIM_IMAGE_PREFIX}${service}:${releaseVersion}`; const proc = Bun.spawn(["regctl", "manifest", "head", reference], { stdout: "pipe", stderr: "pipe", }); - const [stdout, stderr, exit] = await Promise.all([ + const [stdout, , exit] = await Promise.all([ new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited, ]); const digest = stdout.trim(); - if (exit === 0 && DIGEST_PATTERN.test(digest)) return { probe: "published", digest }; - const detail = stderr.toLowerCase(); - if ( - detail.includes("manifest unknown") || - detail.includes("name unknown") || - detail.includes("not found") || - detail.includes("404") - ) { - return { probe: "missing" }; + return exit === 0 && DIGEST_PATTERN.test(digest) ? digest : undefined; +} + +async function s3Sha256(url: string): Promise { + const response = await fetch(url); + if (!response.ok) return undefined; + const hasher = new Bun.CryptoHasher("sha256"); + hasher.update(new Uint8Array(await response.arrayBuffer())); + return hasher.digest("hex"); +} + +async function fetchManifest( + service: string, + releaseVersion: string, + target: NativeTargetName, +): Promise { + const files = nativeFileNames(service, releaseVersion, target); + const response = await fetch( + `${RELEASE_DOWNLOAD_BASE}/${service}-${releaseVersion}/${files.manifest}`, + ); + if (response.status === 404) return undefined; + if (!response.ok) { + throw new InvalidPayloadError( + `manifest download failed for ${service}-${releaseVersion} ${target} (HTTP ${response.status}).`, + ); } - const message = stderr.trim(); - console.log( - `::warning ::${reference} publication check failed${message === "" ? "" : `: ${message}`}`, + return response.text(); +} + +async function fetchProvenance( + service: string, + releaseVersion: string, + upstreamVersion: string, +): Promise { + const response = await fetch( + `${RELEASE_DOWNLOAD_BASE}/${service}-${releaseVersion}/${service}-${upstreamVersion}.oci-provenance.json`, ); - return { probe: "lookup-failed" }; + if (response.status === 404) return undefined; + if (!response.ok) { + throw new InvalidPayloadError( + `oci-provenance download failed for ${service}-${releaseVersion} (HTTP ${response.status}).`, + ); + } + return response.text(); } -async function probeNativeRelease(service: string, version: string): Promise { - const tag = `${service}-${version}`; - const headers: Record = { - Accept: "application/vnd.github+json", - "User-Agent": "supabase-cli-catalog-sync", - }; - const token = process.env.GITHUB_TOKEN; - if (token !== undefined && token !== "") headers.Authorization = `Bearer ${token}`; - try { - const response = await fetch(`${NATIVE_RELEASES}/${encodeURIComponent(tag)}`, { headers }); - if (response.status === 200) return "published"; - if (response.status === 404) return "missing"; - console.log(`::warning ::${tag} native release check returned HTTP ${response.status}.`); - return "lookup-failed"; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.log(`::warning ::${tag} native release check failed: ${message}`); - return "lookup-failed"; - } -} - -async function lookupPublication(service: string, version: string): Promise { - const reference = `${SLIM_IMAGE_PREFIX}${service}:${version}`; - const [manifest, native] = await Promise.all([ - probeManifest(reference), - probeNativeRelease(service, version), - ]); - if (manifest.probe === "lookup-failed" || native === "lookup-failed") { - return { status: "lookup-failed" }; +function defaultRevisionIo(): RevisionIo { + return { listReleaseTags, fetchChecksums, imageDigest, s3Sha256, fetchManifest, fetchProvenance }; +} + +function parseFlags(argv: ReadonlyArray): ReadonlyMap { + const flags = new Map(); + for (let index = 0; index < argv.length; index++) { + const arg = argv[index]; + if (arg === undefined || !arg.startsWith("--")) { + throw new InvalidPayloadError(`unexpected argument '${arg ?? ""}'`); + } + const key = arg.slice(2); + const value = argv[index + 1]; + if (value === undefined || value.startsWith("--")) { + throw new InvalidPayloadError(`missing value for --${key}`); + } + flags.set(key, value); + index++; } - if (manifest.probe === "missing" || native === "missing") return { status: "missing" }; - return { status: "published", digest: manifest.digest }; + return flags; } -async function main(argv: ReadonlyArray): Promise { - const [dockerfilePath = DOCKERFILE_PATH, catalogPath = CATALOG_PATH, baseDockerfilePath] = argv; - const dockerfile = await Bun.file(dockerfilePath).text(); +async function runManual(argv: ReadonlyArray): Promise { + const flags = parseFlags(argv); + const service = flags.get("service"); + if (service === undefined) { + throw new InvalidPayloadError( + "Usage: sync-artifacts-catalog.ts --service [--upstream | --release -r]", + ); + } + const catalogPath = CATALOG_PATH; const catalog = await Bun.file(catalogPath).text(); - const baseDockerfile = - baseDockerfilePath === undefined ? undefined : await Bun.file(baseDockerfilePath).text(); - const plan = await planArtifactCatalogUpdate({ - dockerfile, - baseDockerfile, + const result = await refreshCatalogPin({ catalog, - publication: lookupPublication, + service, + upstream: flags.get("upstream"), + release: flags.get("release"), + io: defaultRevisionIo(), }); - for (const skip of plan.skipped) { - console.log(`::warning ::Left ${skip.alias} unchanged: ${skip.reason}`); + if (result.update === undefined) { + console.log(`${service} already pins the highest committed revision.`); + return; } - if (plan.skipped.some((skip) => skip.blocking)) { - console.log("::error ::Refusing to commit a partial catalog update."); - process.exit(1); + await Bun.write(catalogPath, result.source); + console.log( + `Pinned ${service} ${result.update.target} ${result.update.previousVersion} -> ${result.update.version} r${result.update.revision}.`, + ); +} + +/** + * Line-oriented encoding of one `SlimUpdate`, for a bash loop: `kind`, `branch`, `title` and + * `release` (the loop's four required fields, driving the checkout/pin/commit/PR steps) plus + * `from` (the release replaced, for the PR body's "from -> to"). Every field is guaranteed + * non-empty by construction. `\x1f` (unit separator) is the delimiter, not a tab: a title can + * carry ordinary whitespace, and `IFS=$'\t' read` would collapse it. + */ +function updateLine(update: SlimUpdate): string { + return [update.kind, update.branch, update.title, update.toRelease, update.fromRelease].join( + "\x1f", + ); +} + +/** + * `plan-updates`' records go only into `--output `, never stdout: a caller (the workflow) + * reads warnings from stdout as `::warning ::…` lines, and would otherwise mistake one for a + * malformed record and abort a run that had valid updates alongside it. + * + * `io.listReleaseTags` defaults to the real network lister but is overridable — the minimal seam + * a test uses to exercise this CLI mode's own file/stdout wiring (not just the pure planner) + * without a network call, the same pattern `RevisionIo` already uses. `io.wait` likewise defaults + * to a real delay but is overridable, the seam `waitForExpectedRelease`'s own unit tests use. The + * `--service`/`--output` usage check runs before either the catalog read or the lister, so a test + * can also assert this mode fails fast on a missing flag without touching the network. + * + * With `--expect-release -r` given, `--r` must be among the listed tags + * before planning runs at all — otherwise this dispatch's own release could be missing from a + * releases API that hasn't caught up yet, and the run would plan and pass quietly without it + * (`waitForExpectedRelease` handles the resulting eventual-consistency wait). Still missing after + * the bounded retries: this mode throws instead of planning, so `main()`'s handler reports an + * actionable `::error ::` and exits non-zero, and no `--output` file is written. + */ +export async function runPlanUpdates( + argv: ReadonlyArray, + io: { + readonly listReleaseTags: () => Promise>; + readonly wait?: (ms: number) => Promise; + } = { listReleaseTags }, +): Promise { + const flags = parseFlags(argv); + const service = flags.get("service"); + const output = flags.get("output"); + const expectRelease = flags.get("expect-release"); + if (service === undefined || output === undefined) { + throw new InvalidPayloadError( + "Usage: sync-artifacts-catalog.ts plan-updates --service --output " + + "[--format lines] [--expect-release -r]", + ); } - await Bun.write(catalogPath, plan.source); - if (plan.updates.length === 0) { - console.log("Workload catalog already matches the Dockerfile."); - return; + if (!SERVICE_NAME_PATTERN.test(service)) { + throw new InvalidPayloadError(`invalid --service ${JSON.stringify(service)}`); + } + const format = flags.get("format") ?? "json"; + if (format !== "json" && format !== "lines") { + throw new InvalidPayloadError(`invalid --format '${format}' (expected 'json' or 'lines')`); } - for (const update of plan.updates) { - console.log( - `Pinned ${update.service} ${update.target} ${update.previousVersion} -> ${update.version}.`, + if (expectRelease !== undefined && !RELEASE_VERSION_PATTERN.test(expectRelease)) { + throw new InvalidPayloadError(`invalid --expect-release ${JSON.stringify(expectRelease)}`); + } + + let listReleaseTags = io.listReleaseTags; + if (expectRelease !== undefined) { + const waited = await waitForExpectedRelease(service, expectRelease, { + listReleaseTags: io.listReleaseTags, + wait: io.wait ?? sleep, + }); + if (!waited.visible) { + throw new InvalidPayloadError( + `${service}-${expectRelease} is not visible yet from the slim-services releases API; re-run this workflow once it has propagated.`, + ); + } + // Reuses the listing the successful attempt already fetched, instead of listing again. + listReleaseTags = async () => waited.tags; + } + + const catalog = await Bun.file(CATALOG_PATH).text(); + const { updates, warnings } = await planUpdatesForService({ catalog, service, listReleaseTags }); + for (const warning of warnings) console.log(warning); + const content = + format === "lines" + ? updates.map((update) => `${updateLine(update)}\n`).join("") + : `${JSON.stringify(updates)}\n`; + await Bun.write(output, content); +} + +async function runValidatePayload(argv: ReadonlyArray): Promise { + const flags = parseFlags(argv); + const service = flags.get("service"); + const upstream = flags.get("upstream"); + const revision = flags.get("revision"); + const release = flags.get("release"); + if ( + service === undefined || + upstream === undefined || + revision === undefined || + release === undefined + ) { + throw new InvalidPayloadError( + "Usage: sync-artifacts-catalog.ts validate-payload --service --upstream --revision --release ", ); } + const payload = validateSlimReleasePublishedPayload({ + service, + upstream_version: upstream, + revision, + release_version: release, + }); + console.log(`service=${payload.service}`); + console.log(`upstream_version=${payload.upstream_version}`); + console.log(`revision=${payload.revision}`); + console.log(`release_version=${payload.release_version}`); +} + +async function main(argv: ReadonlyArray): Promise { + if (argv[0] === "validate-payload") { + await runValidatePayload(argv.slice(1)); + return; + } + if (argv[0] === "plan-updates") { + await runPlanUpdates(argv.slice(1)); + return; + } + if (argv[0]?.startsWith("--") === true) { + await runManual(argv); + return; + } + + throw new InvalidPayloadError( + "Usage: sync-artifacts-catalog.ts --service [--upstream | --release -r], " + + "or validate-payload / plan-updates --service ", + ); } if (import.meta.main) { main(process.argv.slice(2)).catch((error: unknown) => { - console.log(`::error ::${error instanceof Error ? error.message : String(error)}`); + console.log(workflowCommand("error", error instanceof Error ? error.message : String(error))); process.exit(1); }); } diff --git a/.github/workflows/mirror-template-images.yml b/.github/workflows/mirror-template-images.yml index 3b8cb03676..dabc95430e 100644 --- a/.github/workflows/mirror-template-images.yml +++ b/.github/workflows/mirror-template-images.yml @@ -3,9 +3,10 @@ name: Mirror template images # Keeps the ghcr.io/ECR mirror in sync with the image versions pinned in # apps/cli/src/shared/services/Dockerfile, which `dockerfileServiceImages` # parses as the single source of truth for local service images. When the -# Dockerfile changes on develop — most often via a merged dependabot `docker` -# bump — this workflow detects any tag that is not yet mirrored and backfills -# it the same way `cli-go-mirror-image.yml` does. +# Dockerfile changes on develop — most often via a merged catalog bump or +# hotfix PR from slim-release-published.yml, or a Dependabot `docker` bump for +# an upstream-only image — this workflow detects any tag that is not yet +# mirrored and backfills it the same way `cli-go-mirror-image.yml` does. # # It runs on `push` to develop (not on the PR) on purpose: mirroring needs the # AWS role + packages:write, which a dependabot-triggered `pull_request` run diff --git a/.github/workflows/slim-release-published.yml b/.github/workflows/slim-release-published.yml new file mode 100644 index 0000000000..9e90d9a362 --- /dev/null +++ b/.github/workflows/slim-release-published.yml @@ -0,0 +1,227 @@ +name: Slim Release Published + +# supabase/slim-services sends this dispatch after it mints an immutable release +# `--r`. The stack catalog +# (packages/stack/src/Artifacts.ts) is the single version table for the service; this workflow +# treats the dispatch as a trigger and reconciles the catalog against every committed +# (published, non-draft) `-...-r` release, opening or updating one pull request per +# hotfix and/or upgrade it finds (`.github/scripts/sync-artifacts-catalog.ts`'s `planSlimUpdates`). +# +# Plan and apply run from the same checkout of the default branch, in a single job: a re-run +# always recomputes from the latest develop, so a stale plan can never be applied, and a +# superseded PR's branch is rewritten in place (force-pushed) instead of racing a fresh one. A +# backlog republish of an older upstream version naturally plans nothing. +# +# The payload arrives with whatever authority holds the dispatch token, so it is treated as +# untrusted: fields are pattern- and shape-checked before use +# (`validateSlimReleasePublishedPayload`), and never interpolated directly into a `run:` script +# (only passed through `env:`). Release tag names come from an external API too, so every value +# `planSlimUpdates` emits is re-validated against the same anchored patterns before it can reach a +# branch name or PR title. + +on: + repository_dispatch: + types: + - slim-release-published + +permissions: + contents: read + +concurrency: + group: slim-release-published-${{ github.event.client_payload.service }} + cancel-in-progress: false + +jobs: + pickup: + runs-on: ubuntu-latest + # Setup (~10 min) + release visibility (<1 min) + the S3-mirror wait (up to ~10 min, normally only + # for the first planned item) + pin, render and push per item. + timeout-minutes: 45 + permissions: + contents: read + steps: + - name: Checkout the default branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + # Installs the locked workspace dependencies the scripts need: `Artifacts.ts` imports + # `effect`, and `render-service-dockerfile.ts` imports `@supabase/stack/internal/artifacts`. + # Replaces the bare mise step the pre-single-table version of this workflow used. + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + + - name: Record the base commit + id: base + run: echo "sha=$(git rev-parse HEAD)" >>"$GITHUB_OUTPUT" + + # Delegates to `validateSlimReleasePublishedPayload` (sync-artifacts-catalog.ts) so every + # field — including `upstream_version` and `release_version`, not just `service` and + # `revision` — is checked against an anchored charset before it is ever written to + # `$GITHUB_OUTPUT`, a branch name, or a PR title. A value that fails validation makes the + # script exit non-zero and print a single `::error ::…` line to stdout (the rejected value + # is JSON-escaped in that message, so it can never smuggle in a newline or workflow command), + # which this step re-echoes so it still surfaces as an annotation. + - name: Validate payload + id: validate + env: + SERVICE: ${{ github.event.client_payload.service }} + UPSTREAM_VERSION: ${{ github.event.client_payload.upstream_version }} + REVISION: ${{ github.event.client_payload.revision }} + RELEASE_VERSION: ${{ github.event.client_payload.release_version }} + run: | + set -euo pipefail + if ! output="$(bun .github/scripts/sync-artifacts-catalog.ts validate-payload \ + --service "$SERVICE" --upstream "$UPSTREAM_VERSION" --revision "$REVISION" --release "$RELEASE_VERSION")"; then + echo "$output" + exit 1 + fi + echo "$output" >>"$GITHUB_OUTPUT" + + # Reconciles the service against every committed slim-services release (`planSlimUpdates`), + # not just the release that triggered this dispatch — this run is idempotent and safe to + # receive out of order. Records go only to `--output` (never stdout): each line is + # field-separated with `\x1f` (see `updateLine`), which a later step reads directly. Any + # `::warning ::…` the planner emits (an ignored tag, say) prints to this step's own stdout + # instead, so it can never be mistaken for a malformed record. + # + # `--expect-release` guards against the releases API lagging behind this very dispatch: it + # requires `-` to be a listed tag before planning runs at all, + # re-listing a bounded number of times first (`waitForExpectedRelease`). Without it, a slow + # API would make this run plan without the release that triggered it, and pass quietly. + - name: Plan updates + id: plan + env: + SERVICE: ${{ steps.validate.outputs.service }} + RELEASE_VERSION: ${{ steps.validate.outputs.release_version }} + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + bun .github/scripts/sync-artifacts-catalog.ts plan-updates --service "$SERVICE" --format lines \ + --expect-release "$RELEASE_VERSION" --output "${RUNNER_TEMP}/slim-updates.tsv" + count="$(wc -l < "${RUNNER_TEMP}/slim-updates.tsv" | tr -d ' ')" + echo "count=${count}" >>"$GITHUB_OUTPUT" + + - name: Nothing to do + if: steps.plan.outputs.count == '0' + env: + SERVICE: ${{ steps.validate.outputs.service }} + RELEASE_VERSION: ${{ steps.validate.outputs.release_version }} + run: echo "No catalog updates for ${SERVICE} from ${RELEASE_VERSION}; nothing to do." + + - name: Install regctl + if: steps.plan.outputs.count != '0' + run: | + set -euo pipefail + install -d "${RUNNER_TEMP}/regctl-bin" + curl -fsSLo "${RUNNER_TEMP}/regctl-bin/regctl" \ + https://github.com/regclient/regclient/releases/download/v0.11.5/regctl-linux-amd64 + echo "c93aa7638749f5aaac1a8e01787321889c78f0101809bb2880343478d0ba0467 ${RUNNER_TEMP}/regctl-bin/regctl" | sha256sum -c - + chmod +x "${RUNNER_TEMP}/regctl-bin/regctl" + echo "${RUNNER_TEMP}/regctl-bin" >>"$GITHUB_PATH" + "${RUNNER_TEMP}/regctl-bin/regctl" version + + - name: Generate token + if: steps.plan.outputs.count != '0' + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.GH_APP_CLIENT_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + + # One branch per planned item, built fresh from the base commit recorded above (the default + # branch's head at the start of this run) — never from wherever a previous loop iteration + # left HEAD. Pins exactly the planned release (`--release`, never "highest at apply time"), + # so plan and pin agree by construction: a revision minted a second later arrives with its + # own dispatch, not by racing this one. + # + # The app token (contents + pull-requests write) never reaches third-party code or disk: + # `git push` takes it only as part of an explicit URL, computed once as `push_url` + # (overridable via `PUSH_REMOTE_URL`, the seam a dry run uses to target a local bare repo + # instead), and `gh` gets it inline per call. Every `bun`/`pnpm` command below — the sync + # script, the Dockerfile generator, the formatter — runs under `env -u APP_TOKEN` so a + # compromised transitive dependency of any of them (they import `effect`) can't read it. + - name: Apply planned updates + if: steps.plan.outputs.count != '0' + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + GITHUB_TOKEN: ${{ github.token }} + SERVICE: ${{ steps.validate.outputs.service }} + RELEASE_VERSION: ${{ steps.validate.outputs.release_version }} + BASE_SHA: ${{ steps.base.outputs.sha }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + push_url="${PUSH_REMOTE_URL:-https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}.git}" + manual_prefix="bun .github/scripts/sync-artifacts-catalog.ts --service ${SERVICE} --release" + + # Read the records on fd 3, not stdin: every command the loop body runs (`bun`, `gh`, + # `git`) otherwise shares stdin with the `read`, and any of them consuming a byte of it + # would swallow the rest of the file's records. + while IFS=$'\x1f' read -r -u 3 kind branch title release from; do + if [ -z "$kind" ] || [ -z "$branch" ] || [ -z "$title" ] || [ -z "$release" ] || [ -z "$from" ]; then + echo "::error ::Malformed plan-updates line: kind='${kind}' branch='${branch}' title='${title}' release='${release}' from='${from}'." + exit 1 + fi + + git checkout -B "$branch" "$BASE_SHA" + + env -u APP_TOKEN bun .github/scripts/sync-artifacts-catalog.ts --service "$SERVICE" --release "$release" + env -u APP_TOKEN -u GITHUB_TOKEN pnpm exec oxfmt --config .oxfmtrc.json packages/stack/src/Artifacts.ts + env -u APP_TOKEN -u GITHUB_TOKEN bun apps/cli/scripts/render-service-dockerfile.ts + + generated_files=( + packages/stack/src/Artifacts.ts + apps/cli/src/shared/services/Dockerfile + apps/cli-go/pkg/config/templates/Dockerfile + ) + if git diff --quiet -- "${generated_files[@]}"; then + echo "${branch}: catalog and Dockerfiles already match ${release}; skipping." + continue + fi + + git add -- "${generated_files[@]}" + git commit -m "$title" + + if ! git push --force "$push_url" "HEAD:refs/heads/${branch}"; then + echo "::error ::Failed to push ${branch}. Run manually: ${manual_prefix} ${release}, regenerate the Dockerfiles with \`bun apps/cli/scripts/render-service-dockerfile.ts\`, then open a pull request by hand." + exit 1 + fi + + if [ "$kind" = "upgrade" ]; then + action="bumps" + else + action="pins" + fi + # Names the release this PR actually pins as the subject; the dispatch's triggering + # release (RELEASE_VERSION) can differ from it (a hotfix dispatch commonly also + # yields an unrelated upgrade on the same line), so it's only mentioned, not implied + # to be what changed. + body="$(printf 'This %s %s from %s to %s.\n\nhttps://github.com/supabase/slim-services/releases/tag/%s-%s\n\nPlanned after supabase/slim-services published %s. This branch is rewritten from %s whenever a newer relevant release arrives.\n' \ + "$action" "$SERVICE" "$from" "$release" "$SERVICE" "$release" "$RELEASE_VERSION" "$DEFAULT_BRANCH")" + + # `--head` matches by branch name only and ignores the owner, so a fork PR with the + # same head branch name would otherwise match too; `isCrossRepository` excludes it. + existing="$(GH_TOKEN="$APP_TOKEN" gh pr list --head "$branch" --base "$DEFAULT_BRANCH" --state open --json number,isCrossRepository --jq 'map(select(.isCrossRepository | not)) | .[0].number // empty')" + if [ -n "$existing" ]; then + if ! GH_TOKEN="$APP_TOKEN" gh pr edit "$existing" --title "$title" --body "$body"; then + echo "::error ::Pushed ${branch} but failed to edit pull request #${existing}. Run manually: ${manual_prefix} ${release}, then edit the pull request from ${branch} by hand." + exit 1 + fi + elif ! GH_TOKEN="$APP_TOKEN" gh pr create \ + --title "$title" \ + --body "$body" \ + --assignee jgoux \ + --head "$branch" \ + --base "$DEFAULT_BRANCH"; then + echo "::error ::Pushed ${branch} but failed to open a pull request. Run manually: ${manual_prefix} ${release}, then open a pull request from ${branch} against ${DEFAULT_BRANCH} by hand." + exit 1 + fi + done 3< "${RUNNER_TEMP}/slim-updates.tsv" diff --git a/.github/workflows/sync-artifacts-catalog.yml b/.github/workflows/sync-artifacts-catalog.yml deleted file mode 100644 index d57b5d7b28..0000000000 --- a/.github/workflows/sync-artifacts-catalog.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: Sync Artifacts catalog - -# The script is the base revision. The pull request only supplies the Dockerfile -# and catalog text, and the job runs for Dependabot so that pull request cannot -# edit this workflow and then execute it with a write token. The catalog commit -# is pushed with the app token so the new head triggers CI. Dependabot's -# GITHUB_TOKEN would not. -on: - pull_request: - paths: - - apps/cli/src/shared/services/Dockerfile - -permissions: - contents: read - -concurrency: - group: sync-artifacts-catalog-${{ github.event.pull_request.number }} - cancel-in-progress: false - -jobs: - sync: - name: Pin workload catalog - if: github.actor == 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest - timeout-minutes: 15 - permissions: - contents: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.event.pull_request.base.sha }} - persist-credentials: true - fetch-depth: 1 - - # Bun only. The script imports no workspace packages. The firewall token - # is empty on a Dependabot run; setup is not required to format one file. - - name: Install toolchains - uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0 - with: - version: 2026.9.0 - - - name: Install regctl - run: | - set -euo pipefail - install -d "${RUNNER_TEMP}/regctl-bin" - curl -fsSLo "${RUNNER_TEMP}/regctl-bin/regctl" \ - https://github.com/regclient/regclient/releases/download/v0.11.5/regctl-linux-amd64 - echo "c93aa7638749f5aaac1a8e01787321889c78f0101809bb2880343478d0ba0467 ${RUNNER_TEMP}/regctl-bin/regctl" | sha256sum -c - - chmod +x "${RUNNER_TEMP}/regctl-bin/regctl" - echo "${RUNNER_TEMP}/regctl-bin" >> "$GITHUB_PATH" - "${RUNNER_TEMP}/regctl-bin/regctl" version - - - name: Generate token - id: app-token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - client-id: ${{ vars.GH_APP_CLIENT_ID }} - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - permission-contents: write - - - name: Pin the catalog to the Dockerfile - env: - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - HEAD_REF: ${{ github.event.pull_request.head.ref }} - PR_NUMBER: ${{ github.event.pull_request.number }} - APP_TOKEN: ${{ steps.app-token.outputs.token }} - run: | - set -euo pipefail - git fetch --no-tags origin "pull/${PR_NUMBER}/head" - fetched="$(git rev-parse FETCH_HEAD)" - if [ "$fetched" != "$HEAD_SHA" ]; then - echo "::error ::pull/${PR_NUMBER} head is ${fetched}, expected ${HEAD_SHA}." - exit 1 - fi - dockerfile="$(mktemp)" - catalog="$(mktemp)" - original="$(mktemp)" - git show "$HEAD_SHA:apps/cli/src/shared/services/Dockerfile" > "$dockerfile" - git show "$HEAD_SHA:packages/stack/src/Artifacts.ts" > "$catalog" - cp "$catalog" "$original" - bun .github/scripts/sync-artifacts-catalog.ts "$dockerfile" "$catalog" apps/cli/src/shared/services/Dockerfile - if cmp -s "$original" "$catalog"; then - echo "Workload catalog already matches the Dockerfile." - exit 0 - fi - git checkout --detach "$HEAD_SHA" - cp "$catalog" packages/stack/src/Artifacts.ts - bun x oxfmt@0.65.0 --config .oxfmtrc.json packages/stack/src/Artifacts.ts - git add -- packages/stack/src/Artifacts.ts - if git diff --cached --quiet -- packages/stack/src/Artifacts.ts; then - echo "Workload catalog already matches the Dockerfile." - exit 0 - fi - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git commit -m "chore(stack): pin the workload catalog to the Dockerfile image tags" - git remote set-url origin "https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git push origin "HEAD:refs/heads/${HEAD_REF}" diff --git a/apps/cli-go/pkg/config/config.go b/apps/cli-go/pkg/config/config.go index 0491495db9..5c2eca797e 100644 --- a/apps/cli-go/pkg/config/config.go +++ b/apps/cli-go/pkg/config/config.go @@ -883,11 +883,11 @@ func (c *config) Load(path string, fsys fs.FS, overrides ...ConfigEditor) error // Update image versions switch c.Db.MajorVersion { case 13: - c.Db.Image = pg15 + c.Db.Image = Images.Pg15 case 14: - c.Db.Image = pg14 + c.Db.Image = Images.Pg14 case 15: - c.Db.Image = pg15 + c.Db.Image = Images.Pg15 } if c.Db.MajorVersion > 14 { if version, err := fs.ReadFile(fsys, builder.PostgresVersionPath); err == nil { diff --git a/apps/cli-go/pkg/config/constants.go b/apps/cli-go/pkg/config/constants.go index 08d572d2da..93bb482f69 100644 --- a/apps/cli-go/pkg/config/constants.go +++ b/apps/cli-go/pkg/config/constants.go @@ -9,14 +9,15 @@ import ( ) const ( - pg13 = "supabase/postgres:13.3.0" - pg14 = "supabase/postgres:14.1.0.89" - pg15 = "supabase/postgres:15.8.1.085" deno1 = "supabase/edge-runtime:v1.68.4" ) type images struct { Pg string `mapstructure:"pg"` + // PG13/15 and PG14 come from the Dockerfile's generated `pg15`/hand-pinned `pg14` stages + // (the single version table), not hardcoded constants. + Pg15 string `mapstructure:"pg15"` + Pg14 string `mapstructure:"pg14"` // Append to Services when adding new dependencies below Kong string `mapstructure:"kong"` Inbucket string `mapstructure:"mailpit"` diff --git a/apps/cli-go/pkg/config/templates/Dockerfile b/apps/cli-go/pkg/config/templates/Dockerfile index a9c85ee0ca..aee8c87f83 100644 --- a/apps/cli-go/pkg/config/templates/Dockerfile +++ b/apps/cli-go/pkg/config/templates/Dockerfile @@ -1,21 +1,29 @@ -# Exposed for updates by .github/dependabot.yml -FROM supabase/postgres:17.6.1.171 AS pg +# The tag pinned below is generated from packages/stack/src/Artifacts.ts by +# apps/cli/scripts/render-service-dockerfile.ts, for these aliases only: pg, pg15, mailpit, +# postgrest, pgmeta, studio, imgproxy, edgeruntime, vector, supavisor, gotrue, realtime, storage, +# logflare. Do not hand-edit those tags; a CI drift check enforces this. Run the generator after +# a catalog update. Everything else in this file (this comment, kong, pg14, the one-shot job +# images) is hand- or Dependabot-managed. +FROM supabase/postgres:17.11.0.002 AS pg +FROM supabase/postgres:15.19.0.002 AS pg15 +# Postgres 14 has no slim build; bumped by hand (Dependabot ignores supabase/postgres). +FROM supabase/postgres:14.1.0.89 AS pg14 # New always-on service alias: extend SERVICE_IMAGE_ALIASES in # shared/services/services.shared.ts and DOCKERFILE_ALIAS_BY_SERVICE in # commands/start/start.gates.ts. FROM library/kong:2.8.1 AS kong -FROM axllent/mailpit:v1.30.2 AS mailpit -FROM postgrest/postgrest:v16.3 AS postgrest +FROM axllent/mailpit:v1.31.3 AS mailpit +FROM postgrest/postgrest:v16.4 AS postgrest FROM supabase/postgres-meta:v0.99.0 AS pgmeta -FROM supabase/studio:2026.09.14-sha-4dd8a95 AS studio -FROM darthsim/imgproxy:v3.8.0 AS imgproxy +FROM supabase/studio:2026.09.28-sha-5e59b60 AS studio +FROM darthsim/imgproxy:v3.26.0 AS imgproxy FROM supabase/edge-runtime:v1.77.1 AS edgeruntime -FROM timberio/vector:0.53.0-alpine AS vector +FROM timberio/vector:0.58.0-alpine AS vector FROM supabase/supavisor:2.9.13 AS supavisor FROM supabase/gotrue:v2.197.0 AS gotrue -FROM supabase/realtime:v2.135.3 AS realtime -FROM supabase/storage-api:v1.77.0 AS storage -FROM supabase/logflare:1.50.12 AS logflare +FROM supabase/realtime:v2.140.3 AS realtime +FROM supabase/storage-api:v1.79.28 AS storage +FROM supabase/logflare:1.50.15 AS logflare # One-shot job images (not started as long-running containers); differ's alias # is resolved via dockerfileServiceImage("differ") in commands/db/diff/pgadmin-diff.ts. FROM supabase/pgadmin-schema-diff:cli-0.0.5 AS differ diff --git a/apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts b/apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts new file mode 100644 index 0000000000..73080b1628 --- /dev/null +++ b/apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, test, vi } from "vitest"; + +import { renderDockerfile } from "./render-service-dockerfile.ts"; + +// A minimal fixture catalog covering every slim-capable alias, so `renderDockerfile`'s +// "every alias needs exactly one line" check is satisfied by default; each test only mutates +// what it's exercising. `vi.mock` factories are hoisted above every other top-level statement, +// so the fixture pins are built inline here rather than imported from an outer module. +vi.mock("@supabase/stack/internal/artifacts", () => { + const nativePin = { archive: "a".repeat(64), manifest: "b".repeat(64) }; + const natives = { "darwin-arm64": nativePin, "linux-amd64": nativePin, "linux-arm64": nativePin }; + const pin = (sourceService: string, upstreamImage: string, isDefault: boolean) => ({ + service: sourceService, + sourceService, + isDefault, + pin: { + upstreamVersion: upstreamImage.split(":").at(-1), + revision: 0, + image: `ghcr.io/supabase/cli/${sourceService}:fixture-r0@sha256:${"c".repeat(64)}`, + upstreamImage, + natives, + }, + }); + return { + catalogPins: () => [ + pin("postgres", "supabase/postgres:17.0.0", true), + pin("postgres", "supabase/postgres:15.0.0", false), + pin("mailpit", "axllent/mailpit:v1.0.0", true), + pin("postgrest", "postgrest/postgrest:v1.0.0", true), + pin("pgmeta", "supabase/postgres-meta:v1.0.0", true), + pin("studio", "supabase/studio:1.0.0", true), + pin("imgproxy", "ghcr.io/imgproxy/imgproxy:v1.0.0", true), + pin("edge-runtime", "supabase/edge-runtime:v1.0.0", true), + pin("vector", "timberio/vector:1.0.0-alpine", true), + pin("pooler", "supabase/supavisor:1.0.0", true), + pin("auth", "supabase/gotrue:v1.0.0", true), + pin("realtime", "supabase/realtime:v1.0.0", true), + pin("storage", "supabase/storage-api:v1.0.0", true), + pin("analytics", "supabase/logflare:1.0.0", true), + ], + }; +}); + +/** A minimal Dockerfile with every slim-capable alias, plus kong and pg14 (hand-managed). */ +function baseDockerfile(overrides: Readonly> = {}): string { + const lines: Readonly> = { + pg: "FROM supabase/postgres:16.9.9 AS pg", + pg15: "FROM supabase/postgres:14.9.9 AS pg15", + mailpit: "FROM axllent/mailpit:v0.9.9 AS mailpit", + postgrest: "FROM postgrest/postgrest:v0.9.9 AS postgrest", + pgmeta: "FROM supabase/postgres-meta:v0.9.9 AS pgmeta", + studio: "FROM supabase/studio:0.9.9 AS studio", + imgproxy: "FROM darthsim/imgproxy:v0.9.9 AS imgproxy", + edgeruntime: "FROM supabase/edge-runtime:v0.9.9 AS edgeruntime", + vector: "FROM timberio/vector:0.9.9-alpine AS vector", + supavisor: "FROM supabase/supavisor:0.9.9 AS supavisor", + gotrue: "FROM supabase/gotrue:v0.9.9 AS gotrue", + realtime: "FROM supabase/realtime:v0.9.9 AS realtime", + storage: "FROM supabase/storage-api:v0.9.9 AS storage", + logflare: "FROM supabase/logflare:0.9.9 AS logflare", + kong: "FROM library/kong:2.8.1 AS kong", + }; + const merged = { ...lines, ...overrides }; + return `# hand-authored header\n${Object.values(merged).join("\n")}\n# hand-authored footer\n`; +} + +describe("renderDockerfile: pin selection", () => { + test("pg takes the default pin and pg15 takes the additional one, by isDefault, not a version-string check", () => { + const rendered = renderDockerfile(baseDockerfile()); + expect(rendered).toContain("FROM supabase/postgres:17.0.0 AS pg\n"); + expect(rendered).toContain("FROM supabase/postgres:15.0.0 AS pg15\n"); + }); +}); + +describe("renderDockerfile: repository rule", () => { + test("keeps the existing repository and only replaces the tag", () => { + const rendered = renderDockerfile(baseDockerfile()); + expect(rendered).toContain("FROM supabase/gotrue:v1.0.0 AS gotrue\n"); + }); + + test("fails loudly when the Dockerfile's repository no longer matches the catalog's, for a non-overridden alias", () => { + expect(() => + renderDockerfile(baseDockerfile({ gotrue: "FROM some-other-org/gotrue:v0.9.9 AS gotrue" })), + ).toThrow(/gotrue.*repository/i); + }); + + test("tolerates imgproxy's documented repository mismatch (the allowlisted override)", () => { + const rendered = renderDockerfile(baseDockerfile()); + // Catalog upstreamImage repo is ghcr.io/imgproxy/imgproxy; the Dockerfile keeps darthsim. + expect(rendered).toContain("FROM darthsim/imgproxy:v1.0.0 AS imgproxy\n"); + }); + + test("still fails loudly when imgproxy's Dockerfile repository drifts from its override entry", () => { + expect(() => + renderDockerfile( + baseDockerfile({ imgproxy: "FROM some-other-org/imgproxy:v0.9.9 AS imgproxy" }), + ), + ).toThrow(/imgproxy.*REPOSITORY_OVERRIDES/i); + }); +}); + +describe("renderDockerfile: line preservation", () => { + test("never inserts a line for a missing slim-capable alias", () => { + const withoutPg15 = baseDockerfile().replace(/FROM supabase\/postgres:14\.9\.9 AS pg15\n/, ""); + expect(() => renderDockerfile(withoutPg15)).toThrow(/pg15/); + }); + + test("leaves non-slim-capable lines (kong) and comments byte-for-byte untouched", () => { + const current = baseDockerfile(); + const rendered = renderDockerfile(current); + expect(rendered).toContain("FROM library/kong:2.8.1 AS kong"); + expect(rendered).toContain("# hand-authored header"); + expect(rendered).toContain("# hand-authored footer"); + }); +}); diff --git a/apps/cli/scripts/render-service-dockerfile.ts b/apps/cli/scripts/render-service-dockerfile.ts new file mode 100644 index 0000000000..6c6c114eb6 --- /dev/null +++ b/apps/cli/scripts/render-service-dockerfile.ts @@ -0,0 +1,192 @@ +// Rewrites the slim-capable `FROM ... AS ` lines of +// apps/cli/src/shared/services/Dockerfile (and its byte-identical Go copy, +// apps/cli-go/pkg/config/templates/Dockerfile) in place, from the stack catalog +// (packages/stack/src/Artifacts.ts): each such line's tag comes from that service's catalog pin +// `upstreamImage`. Every other line — comments, kong, the Postgres 14 pin (no slim build), and +// the one-shot job images — is hand- or Dependabot-managed and left byte-for-byte untouched. +// +// bun apps/cli/scripts/render-service-dockerfile.ts # writes both files +// bun apps/cli/scripts/render-service-dockerfile.ts --check # fails on drift, writes nothing +// +// The `--check` mode is what CI runs (as a `render-service-dockerfile.unit.test.ts` assertion) to +// catch a hand-edited generated line, or a catalog change that hasn't been regenerated yet. +import { fileURLToPath } from "node:url"; +import { catalogPins, type ArtifactPin } from "@supabase/stack/internal/artifacts"; + +// Resolved from this module's own URL, so both the CLI invocation (cwd = repo root) and the +// vitest unit test (cwd = apps/cli) read the same files. +export const TS_DOCKERFILE_PATH = fileURLToPath( + new URL("../src/shared/services/Dockerfile", import.meta.url), +); +export const GO_DOCKERFILE_PATH = fileURLToPath( + new URL("../../cli-go/pkg/config/templates/Dockerfile", import.meta.url), +); + +type CatalogEntry = ReturnType[number]; + +/** + * Dockerfile repository overrides, for the one documented exception to the repository rule + * below. Keep this list to that exception; add a reason alongside any addition. + */ +const REPOSITORY_OVERRIDES: Readonly> = { + // slim-services mirrors ghcr.io/imgproxy/imgproxy (its `upstreamImage`), but the Dockerfile — + // and `mirror-template-images.yml` and the docker.io registry rewrite — keep darthsim/imgproxy, + // which carries the same tags on Docker Hub. + imgproxy: "darthsim/imgproxy", +}; + +interface SlimAliasSpec { + readonly sourceService: string; + /** Selects the catalog's default pin (every alias but `pg15`) or its lone additional pin. */ + readonly wantDefault: boolean; +} + +/** Every slim-capable Dockerfile alias, mapped to the catalog pin it tracks. */ +const SLIM_ALIAS_SPECS: ReadonlyMap = new Map([ + ["pg", { sourceService: "postgres", wantDefault: true }], + ["pg15", { sourceService: "postgres", wantDefault: false }], + ["mailpit", { sourceService: "mailpit", wantDefault: true }], + ["postgrest", { sourceService: "postgrest", wantDefault: true }], + ["pgmeta", { sourceService: "pgmeta", wantDefault: true }], + ["studio", { sourceService: "studio", wantDefault: true }], + ["imgproxy", { sourceService: "imgproxy", wantDefault: true }], + ["edgeruntime", { sourceService: "edge-runtime", wantDefault: true }], + ["vector", { sourceService: "vector", wantDefault: true }], + ["supavisor", { sourceService: "pooler", wantDefault: true }], + ["gotrue", { sourceService: "auth", wantDefault: true }], + ["realtime", { sourceService: "realtime", wantDefault: true }], + ["storage", { sourceService: "storage", wantDefault: true }], + ["logflare", { sourceService: "analytics", wantDefault: true }], +]); + +function selectPin( + alias: string, + spec: SlimAliasSpec, + pins: ReadonlyArray, +): ArtifactPin { + const candidates = pins.filter( + (entry) => entry.sourceService === spec.sourceService && entry.isDefault === spec.wantDefault, + ); + if (candidates.length !== 1) { + throw new Error( + `expected exactly one ${spec.wantDefault ? "default" : "additional"} catalog pin for ` + + `'${spec.sourceService}' (alias '${alias}'), found ${candidates.length}`, + ); + } + return candidates[0]!.pin; +} + +function tagOf(upstreamImage: string): string { + const sep = upstreamImage.lastIndexOf(":"); + if (sep === -1) throw new Error(`upstreamImage has no tag: ${upstreamImage}`); + return upstreamImage.slice(sep + 1); +} + +function repositoryOf(upstreamImage: string): string { + const sep = upstreamImage.lastIndexOf(":"); + if (sep === -1) throw new Error(`upstreamImage has no tag: ${upstreamImage}`); + return upstreamImage.slice(0, sep); +} + +/** Matches a `FROM : AS ` line, keeping every other byte for reuse. */ +const FROM_LINE = /^(FROM\s+)(.+):([^:\s]+)(\s+AS\s+)([^\s#]+)(.*)$/i; + +/** + * Rewrites only the slim-capable `FROM ... AS ` lines of `currentDockerfile`, in place: + * the repository is read from the existing line (and asserted to still match the catalog pin's + * `upstreamImage` repository, except for `REPOSITORY_OVERRIDES`); only the tag is replaced, from + * that pin's `upstreamImage`. Every other line — comments, kong, `pg14`, the job images — passes + * through untouched. A slim-capable alias missing from the file is an error; the generator never + * inserts a line. + */ +export function renderDockerfile(currentDockerfile: string): string { + const pins = catalogPins(); + const seen = new Set(); + + const lines = currentDockerfile.split("\n").map((line) => { + const match = FROM_LINE.exec(line); + if (match === null) return line; + const [, fromKeyword, repository, , asKeyword, alias, rest] = match as unknown as [ + string, + string, + string, + string, + string, + string, + string, + ]; + const spec = SLIM_ALIAS_SPECS.get(alias); + if (spec === undefined) return line; + seen.add(alias); + + const pin = selectPin(alias, spec, pins); + const pinRepository = repositoryOf(pin.upstreamImage); + const override = REPOSITORY_OVERRIDES[alias]; + if (override === undefined) { + if (repository !== pinRepository) { + throw new Error( + `${alias}'s Dockerfile repository '${repository}' no longer matches the catalog's ` + + `'${pinRepository}'. If this is intentional, add '${alias}' to REPOSITORY_OVERRIDES with a reason.`, + ); + } + } else if (repository !== override) { + throw new Error( + `${alias}'s Dockerfile repository '${repository}' no longer matches its ` + + `REPOSITORY_OVERRIDES entry '${override}'. Update the override if this is intentional.`, + ); + } + + return `${fromKeyword}${repository}:${tagOf(pin.upstreamImage)}${asKeyword}${alias}${rest}`; + }); + + for (const alias of SLIM_ALIAS_SPECS.keys()) { + if (!seen.has(alias)) { + throw new Error( + `no Dockerfile line for slim-capable alias '${alias}'; the generator never inserts one`, + ); + } + } + + return lines.join("\n"); +} + +async function main(argv: ReadonlyArray): Promise { + const check = argv.includes("--check"); + const current = await Bun.file(TS_DOCKERFILE_PATH).text(); + const rendered = renderDockerfile(current); + + if (check) { + const go = await Bun.file(GO_DOCKERFILE_PATH).text(); + let failed = false; + if (current !== rendered) { + console.log( + `::error ::${TS_DOCKERFILE_PATH} has drifted from packages/stack/src/Artifacts.ts. Run ` + + "`bun apps/cli/scripts/render-service-dockerfile.ts` and commit the result.", + ); + failed = true; + } + if (go !== rendered) { + console.log( + `::error ::${GO_DOCKERFILE_PATH} is not a byte copy of ${TS_DOCKERFILE_PATH}. Run ` + + "`bun apps/cli/scripts/render-service-dockerfile.ts` and commit the result.", + ); + failed = true; + } + if (failed) { + process.exit(1); + } + console.log("Both Dockerfiles match the catalog."); + return; + } + + await Bun.write(TS_DOCKERFILE_PATH, rendered); + await Bun.write(GO_DOCKERFILE_PATH, rendered); + console.log(`Regenerated ${TS_DOCKERFILE_PATH} and ${GO_DOCKERFILE_PATH} from the catalog.`); +} + +if (import.meta.main) { + main(process.argv.slice(2)).catch((error: unknown) => { + console.log(`::error ::${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + }); +} diff --git a/apps/cli/scripts/render-service-dockerfile.unit.test.ts b/apps/cli/scripts/render-service-dockerfile.unit.test.ts new file mode 100644 index 0000000000..0c4f9036d9 --- /dev/null +++ b/apps/cli/scripts/render-service-dockerfile.unit.test.ts @@ -0,0 +1,66 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, test } from "vitest"; + +import { + GO_DOCKERFILE_PATH, + TS_DOCKERFILE_PATH, + renderDockerfile, +} from "./render-service-dockerfile.ts"; + +// Both against the real, checked-in Dockerfile and the real catalog — no hardcoded version +// literal, so a catalog bump never makes this drift-detection test itself go stale. +const currentTsDockerfile = readFileSync(TS_DOCKERFILE_PATH, "utf8"); + +describe("renderDockerfile against the real catalog and Dockerfile", () => { + test("round-trips: rewriting the checked-in Dockerfile reproduces it byte for byte (no drift)", () => { + expect(renderDockerfile(currentTsDockerfile)).toBe(currentTsDockerfile); + }); + + test("rendering twice is stable", () => { + expect(renderDockerfile(currentTsDockerfile)).toBe(renderDockerfile(currentTsDockerfile)); + }); + + // The Go tree still `go:embed`s its own copy for a dependency that hasn't been removed yet; + // this is the single place that keeps the two copies in sync (folded from the former + // dockerfile-go-sync.unit.test.ts), until apps/cli-go is deleted. + test("the Go tree's embedded Dockerfile is a byte copy of the TS-owned one", () => { + const goDockerfile = readFileSync(GO_DOCKERFILE_PATH, "utf8"); + expect(goDockerfile).toBe(currentTsDockerfile); + }); + + test("detects drift when a generated line is hand-edited", () => { + // A hand-edit to a generated tag (bumping gotrue without touching the catalog) must not + // round-trip back to itself — this is the drift check's whole point. + const tampered = currentTsDockerfile.replace( + /FROM supabase\/gotrue:v[^\s]+ AS gotrue/, + "FROM supabase/gotrue:v9.9.9 AS gotrue", + ); + expect(tampered).not.toBe(currentTsDockerfile); + expect(renderDockerfile(tampered)).not.toBe(tampered); + // It heals back to the catalog-pinned tag, not just "detects a difference". + expect(renderDockerfile(tampered)).toBe(currentTsDockerfile); + }); + + test("preserves hand-/Dependabot-managed lines verbatim, even when their own tags change", () => { + // Simulates a Dependabot bump of kong/differ/migra/pgprove: give each a synthetic marker + // tag (never a real pin) and confirm the generator leaves those exact lines alone rather + // than reverting or otherwise touching them. + const marker = "99.99.99-fixture-marker"; + let tampered = currentTsDockerfile; + for (const alias of ["kong", "differ", "migra", "pgprove"]) { + const pattern = new RegExp(`^FROM (\\S+):(\\S+) AS ${alias}$`, "m"); + const match = pattern.exec(tampered); + expect(match, `no line for hand-managed alias '${alias}'`).not.toBeNull(); + tampered = tampered.replace(pattern, `FROM $1:${marker} AS ${alias}`); + } + expect(tampered).not.toBe(currentTsDockerfile); + + const rendered = renderDockerfile(tampered); + for (const alias of ["kong", "differ", "migra", "pgprove"]) { + const tamperedLine = tampered.split("\n").find((line) => line.endsWith(`AS ${alias}`)); + expect(tamperedLine).toBeDefined(); + expect(tamperedLine).toContain(marker); + expect(rendered).toContain(tamperedLine); + } + }); +}); diff --git a/apps/cli/scripts/sweep-live-projects.integration.test.ts b/apps/cli/scripts/sweep-live-projects.integration.test.ts index ac124dce97..faaaac4c2b 100644 --- a/apps/cli/scripts/sweep-live-projects.integration.test.ts +++ b/apps/cli/scripts/sweep-live-projects.integration.test.ts @@ -6,6 +6,13 @@ import path from "node:path"; const script = path.resolve(import.meta.dirname, "sweep-live-projects.sh"); const directories: string[] = []; +// Per-sweep hang guard: a spawned run gets this long before the safety net kills +// it. A test's own timeout must cover every sweep it runs plus report margin, so +// derive it from the sweep count rather than a single shared budget. +const SWEEP_GUARD_MS = 30_000; +const REPORT_MARGIN_MS = 10_000; +const sweepBudget = (sweeps: number) => sweeps * SWEEP_GUARD_MS + REPORT_MARGIN_MS; + afterEach(async () => { await Promise.all(directories.splice(0).map((directory) => rm(directory, { recursive: true }))); }); @@ -73,13 +80,22 @@ async function runSweep(scenario: Scenario) { stdout: "pipe", stderr: "pipe", }); - const timeout = setTimeout(() => child.kill(), 10_000); + let hung = false; + const timeout = setTimeout(() => { + hung = true; + child.kill(); + }, SWEEP_GUARD_MS); const [exitCode, stdout, stderr] = await Promise.all([ child.exited, new Response(child.stdout).text(), new Response(child.stderr).text(), ]); clearTimeout(timeout); + if (hung) { + throw new Error( + `sweep script hung: killed after ${SWEEP_GUARD_MS}ms without exiting (stdout: ${stdout}, stderr: ${stderr})`, + ); + } return { exitCode, stdout, stderr, deletes }; } finally { await server.stop(true); @@ -88,75 +104,91 @@ async function runSweep(scenario: Scenario) { const active = (ref: string, name = `e2e-${ref}`) => ({ ref, name, status: "ACTIVE" }); -describe.skipIf(process.platform === "win32")("sweep-live-projects.sh", { timeout: 40_000 }, () => { - test("accepts refused deletion when a fresh authenticated list shows absence", async () => { - const result = await runSweep({ - lists: [[active("gone")], []], - deletes: { gone: { status: 403, body: { message: "already removed" } } }, +describe.skipIf(process.platform === "win32")( + "sweep-live-projects.sh", + { timeout: sweepBudget(1) }, + () => { + test("accepts refused deletion when a fresh authenticated list shows absence", async () => { + const result = await runSweep({ + lists: [[active("gone")], []], + deletes: { gone: { status: 403, body: { message: "already removed" } } }, + }); + expect(result.exitCode).toBe(0); + expect(result.deletes).toEqual(["gone"]); + expect(`${result.stdout}\n${result.stderr}`).not.toContain("test-token"); }); - expect(result.exitCode).toBe(0); - expect(result.deletes).toEqual(["gone"]); - expect(`${result.stdout}\n${result.stderr}`).not.toContain("test-token"); - }); - test("accepts a terminal status and converges after a stale list", async () => { - const result = await runSweep({ - lists: [[active("stale")], [active("stale")], [{ ...active("stale"), status: "REMOVED" }]], - deletes: { stale: { status: 202 } }, + test("accepts a terminal status and converges after a stale list", async () => { + const result = await runSweep({ + lists: [[active("stale")], [active("stale")], [{ ...active("stale"), status: "REMOVED" }]], + deletes: { stale: { status: 202 } }, + }); + expect(result.exitCode).toBe(0); }); - expect(result.exitCode).toBe(0); - }); - test("retries a transient read failure but rejects malformed evidence", async () => { - const transient = await runSweep({ - lists: [503, [active("retry")], []], - deletes: { retry: { status: 202 } }, - }); - expect(transient.exitCode).toBe(0); + test( + "retries a transient read failure but rejects malformed evidence", + async () => { + const transient = await runSweep({ + lists: [503, [active("retry")], []], + deletes: { retry: { status: 202 } }, + }); + expect(transient.exitCode).toBe(0); - const malformed = await runSweep({ lists: [{ projects: [] }], deletes: {} }); - expect(malformed.exitCode).not.toBe(0); - expect(malformed.deletes).toEqual([]); - }); + const malformed = await runSweep({ lists: [{ projects: [] }], deletes: {} }); + expect(malformed.exitCode).not.toBe(0); + expect(malformed.deletes).toEqual([]); + }, + sweepBudget(2), + ); - test("fails terminal listing errors without retrying or deleting", async () => { - const forbidden = await runSweep({ lists: [403, []], deletes: {} }); - expect(forbidden.exitCode).not.toBe(0); - expect(forbidden.deletes).toEqual([]); + test( + "fails terminal listing errors without retrying or deleting", + async () => { + const forbidden = await runSweep({ lists: [403, []], deletes: {} }); + expect(forbidden.exitCode).not.toBe(0); + expect(forbidden.deletes).toEqual([]); - const unavailable = await runSweep({ lists: [503, 503, 503], deletes: {} }); - expect(unavailable.exitCode).not.toBe(0); - expect(unavailable.deletes).toEqual([]); + const unavailable = await runSweep({ lists: [503, 503, 503], deletes: {} }); + expect(unavailable.exitCode).not.toBe(0); + expect(unavailable.deletes).toEqual([]); - const malformedReconciliation = await runSweep({ - lists: [[active("bad-evidence")], { projects: [] }], - deletes: { "bad-evidence": { status: 403 } }, - }); - expect(malformedReconciliation.exitCode).not.toBe(0); - }); + const malformedReconciliation = await runSweep({ + lists: [[active("bad-evidence")], { projects: [] }], + deletes: { "bad-evidence": { status: 403 } }, + }); + expect(malformedReconciliation.exitCode).not.toBe(0); + }, + sweepBudget(3), + ); - test("attempts every owned project and fails if one remains active", async () => { - const result = await runSweep({ - lists: [ - [active("stuck"), active("other"), { ref: "foreign", name: "unrelated", status: "ACTIVE" }], - [active("stuck"), { ref: "foreign", name: "unrelated", status: "ACTIVE" }], - ], - deletes: { stuck: { status: 403 }, other: { status: 204 } }, + test("attempts every owned project and fails if one remains active", async () => { + const result = await runSweep({ + lists: [ + [ + active("stuck"), + active("other"), + { ref: "foreign", name: "unrelated", status: "ACTIVE" }, + ], + [active("stuck"), { ref: "foreign", name: "unrelated", status: "ACTIVE" }], + ], + deletes: { stuck: { status: 403 }, other: { status: 204 } }, + }); + expect(result.exitCode).not.toBe(0); + expect(result.deletes).toEqual(["stuck", "other"]); }); - expect(result.exitCode).not.toBe(0); - expect(result.deletes).toEqual(["stuck", "other"]); - }); - test("retries a transient deletion after fresh evidence still shows the project", async () => { - const result = await runSweep({ - lists: (deletes) => (deletes.length >= 2 ? [] : [active("flaky")]), - deletes: {}, - deleteStatuses: { flaky: [503, 204] }, + test("retries a transient deletion after fresh evidence still shows the project", async () => { + const result = await runSweep({ + lists: (deletes) => (deletes.length >= 2 ? [] : [active("flaky")]), + deletes: {}, + deleteStatuses: { flaky: [503, 204] }, + }); + expect(result.exitCode).toBe(0); + expect(result.deletes).toEqual(["flaky", "flaky"]); + expect(result.stderr).toContain( + "delete retry completed for project flaky (HTTP 204 request delete-flaky)", + ); }); - expect(result.exitCode).toBe(0); - expect(result.deletes).toEqual(["flaky", "flaky"]); - expect(result.stderr).toContain( - "delete retry completed for project flaky (HTTP 204 request delete-flaky)", - ); - }); -}); + }, +); diff --git a/apps/cli/scripts/upload-release-assets.integration.test.ts b/apps/cli/scripts/upload-release-assets.integration.test.ts index 1866370852..699cae4bf1 100644 --- a/apps/cli/scripts/upload-release-assets.integration.test.ts +++ b/apps/cli/scripts/upload-release-assets.integration.test.ts @@ -91,20 +91,22 @@ describe("upload-release-assets against a fake gh", () => { await uploadAssets("v1.0.0", [asset], io, { maxAttempts: 2, - timeoutMs: 500, + timeoutMs: 5_000, backoffMs: () => 0, }); - expect(Date.now() - startedAt).toBeLessThan(10_000); + const elapsed = Date.now() - startedAt; + expect(elapsed).toBeGreaterThanOrEqual(5_000); + expect(elapsed).toBeLessThan(25_000); expect(await calls()).toEqual([ `release upload v1.0.0 ${asset.path} --clobber`, `release upload v1.0.0 ${asset.path} --clobber`, ]); expect(logs).toEqual([ - `Upload of ${asset.name} failed on attempt 1 (timed out after 0.5s); retrying in 0s.`, + `Upload of ${asset.name} failed on attempt 1 (timed out after 5s); retrying in 0s.`, `Uploaded ${asset.name} (attempt 2).`, ]); - }, 20_000); + }, 40_000); test("kills an upload that ignores SIGTERM once the grace period passes", async () => { const { directory, env, calls } = await fakeGhOnPath(); @@ -114,18 +116,18 @@ describe("upload-release-assets against a fake gh", () => { await uploadAssets("v1.0.0", [asset], io, { maxAttempts: 2, - timeoutMs: 500, + timeoutMs: 5_000, backoffMs: () => 0, }); const elapsed = Date.now() - startedAt; - expect(elapsed).toBeGreaterThanOrEqual(500 + KILL_GRACE_MS); - expect(elapsed).toBeLessThan(20_000); + expect(elapsed).toBeGreaterThanOrEqual(5_000 + KILL_GRACE_MS); + expect(elapsed).toBeLessThan(25_000); expect(await calls()).toHaveLength(2); expect(logs[0]).toBe( - `Upload of ${asset.name} failed on attempt 1 (timed out after 0.5s); retrying in 0s.`, + `Upload of ${asset.name} failed on attempt 1 (timed out after 5s); retrying in 0s.`, ); - }, 30_000); + }, 40_000); test("surfaces gh's stderr for a failed attempt and retries it", async () => { const { directory, env, calls } = await fakeGhOnPath(); diff --git a/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts b/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts index bb03844234..006cbf4ae6 100644 --- a/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts +++ b/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from "vitest"; import { dockerfileServiceImageRaw } from "../../shared/services/dockerfile-images.ts"; -import { toSlimImage } from "../../shared/services/slim-images.ts"; +import { + expectedPinnedImage, + GHCR_SLIM_IMAGE_PATTERN, +} from "../../../tests/helpers/slim-images.ts"; import { resolvePinnedImage } from "./pinned-image.ts"; const currentTag = (alias: string) => dockerfileServiceImageRaw(alias).split(":")[1] ?? ""; @@ -24,10 +27,10 @@ describe("resolvePinnedImage", () => { }); it("resolves slim images when the flag is on and the pin is current", () => { - expect(resolvePinnedImage("gotrue", "auth", {}, true)).toBe(toSlimImage("gotrue", currentAuth)); - expect(resolvePinnedImage("gotrue", "auth", { auth: currentAuthTag }, true)).toBe( - toSlimImage("gotrue", currentAuth), - ); + const pinned = expectedPinnedImage("gotrue", currentAuth); + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); + expect(resolvePinnedImage("gotrue", "auth", {}, true)).toBe(pinned); + expect(resolvePinnedImage("gotrue", "auth", { auth: currentAuthTag }, true)).toBe(pinned); }); it("keeps a historical pin on docker.io", () => { @@ -43,21 +46,18 @@ describe("resolvePinnedImage", () => { }); it("normalizes a current pooler pin onto the slim tag scheme", () => { + const pinned = expectedPinnedImage("supavisor", currentPooler); + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); expect(resolvePinnedImage("supavisor", "pooler", { pooler: currentPoolerTag }, true)).toBe( - toSlimImage("supavisor", currentPooler), + pinned, ); - expect( - resolvePinnedImage( - "supavisor", - "pooler", - { - pooler: currentPoolerTag.startsWith("v") - ? currentPoolerTag.slice(1) - : `v${currentPoolerTag}`, - }, - true, - ), - ).toBe(toSlimImage("supavisor", currentPooler)); + // The opposite `v`-prefix variant of the same pin still counts as current + // (`pinMatchesCurrentImage` normalizes both before comparing), so it resolves to the very same + // catalog-pinned slim image. + const altPoolerTag = currentPoolerTag.startsWith("v") + ? currentPoolerTag.slice(1) + : `v${currentPoolerTag}`; + expect(resolvePinnedImage("supavisor", "pooler", { pooler: altPoolerTag }, true)).toBe(pinned); }); it("keeps a historical postgres pin on docker.io", () => { @@ -68,7 +68,7 @@ describe("resolvePinnedImage", () => { "supabase/postgres:17.4.1.1", ); expect(resolvePinnedImage("pg", "postgres", { postgres: currentPostgresTag }, true)).toBe( - toSlimImage("pg", currentPostgres), + expectedPinnedImage("pg", currentPostgres), ); }); }); diff --git a/apps/cli/src/command-internal/db-image.ts b/apps/cli/src/command-internal/db-image.ts index cee9e8b932..a154f4aab2 100644 --- a/apps/cli/src/command-internal/db-image.ts +++ b/apps/cli/src/command-internal/db-image.ts @@ -74,7 +74,7 @@ export const resolveDbImage = Effect.fnUntraced(function* ( return { image, configImage: image }; } const slim = yield* slimImagesEnabled; - const currentRaw = postgresImageForDbMajorVersion(majorVersion, slim) ?? pgImageRaw(); + const currentRaw = postgresImageForDbMajorVersion(majorVersion) ?? pgImageRaw(); let appliedPin: string | undefined; if (majorVersion > 14) { const versionPath = path.join(workdir, "supabase", ".temp", "postgres-version"); diff --git a/apps/cli/src/command-internal/db-image.unit.test.ts b/apps/cli/src/command-internal/db-image.unit.test.ts index 6819ae401b..d66aabd1b2 100644 --- a/apps/cli/src/command-internal/db-image.unit.test.ts +++ b/apps/cli/src/command-internal/db-image.unit.test.ts @@ -10,17 +10,17 @@ import { dockerfileServiceImage, dockerfileServiceImageRaw, } from "../shared/services/dockerfile-images.ts"; -import { - POSTGRES_FALLBACK_IMAGE_PG14, - POSTGRES_FALLBACK_IMAGE_PG15, - POSTGRES_FALLBACK_IMAGE_PG15_SLIM, -} from "../shared/services/services.shared.ts"; -import { imageTag, toSlimImage } from "../shared/services/slim-images.ts"; +import { imageTag } from "../shared/services/slim-images.ts"; +import { expectedPinnedImage, GHCR_SLIM_IMAGE_PATTERN } from "../../tests/helpers/slim-images.ts"; import { resolveDbImage } from "./db-image.ts"; const currentPostgres = dockerfileServiceImageRaw("pg"); const currentPostgresTag = imageTag(currentPostgres) ?? ""; -const pg15SlimTag = imageTag(POSTGRES_FALLBACK_IMAGE_PG15_SLIM) ?? ""; +// PG13/15 and PG14 now come from the Dockerfile's generated `pg15`/hand-pinned `pg14` stages +// (the single version table), not hardcoded fallback constants. +const pg15Image = dockerfileServiceImageRaw("pg15"); +const pg15Tag = imageTag(pg15Image) ?? ""; +const pg14Image = dockerfileServiceImageRaw("pg14"); const withTemp = () => mkdtempSync(join(tmpdir(), "db-image-")); @@ -49,16 +49,16 @@ describe("resolveDbImage", () => { const dir = withTemp(); return Effect.gen(function* () { expect(yield* resolve(dir, 13)).toEqual({ - image: POSTGRES_FALLBACK_IMAGE_PG15, - configImage: POSTGRES_FALLBACK_IMAGE_PG15, + image: pg15Image, + configImage: pg15Image, }); expect(yield* resolve(dir, 14)).toEqual({ - image: POSTGRES_FALLBACK_IMAGE_PG14, - configImage: POSTGRES_FALLBACK_IMAGE_PG14, + image: pg14Image, + configImage: pg14Image, }); expect(yield* resolve(dir, 15)).toEqual({ - image: POSTGRES_FALLBACK_IMAGE_PG15, - configImage: POSTGRES_FALLBACK_IMAGE_PG15, + image: pg15Image, + configImage: pg15Image, }); expect(yield* resolve(dir, 17)).toEqual({ image: dockerfileServiceImage("pg", false), @@ -93,8 +93,8 @@ describe("resolveDbImage", () => { const dir = withTemp(); return Effect.gen(function* () { expect(yield* resolve(dir, 14, "16.0.0.1")).toEqual({ - image: POSTGRES_FALLBACK_IMAGE_PG14, - configImage: POSTGRES_FALLBACK_IMAGE_PG14, + image: pg14Image, + configImage: pg14Image, }); rmSync(dir, { recursive: true, force: true }); }); @@ -106,24 +106,26 @@ describe("resolveDbImage", () => { const dir = withTemp(); return Effect.gen(function* () { expect(yield* resolve(dir, 14)).toEqual({ - image: POSTGRES_FALLBACK_IMAGE_PG14, - configImage: POSTGRES_FALLBACK_IMAGE_PG14, + image: pg14Image, + configImage: pg14Image, }); rmSync(dir, { recursive: true, force: true }); }); }); - it.effect("rewrites the current PG15 fallback to the slim registry", () => { + it.effect("rewrites the current PG15 default tag to its catalog-pinned slim image", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const dir = withTemp(); + const pinned = expectedPinnedImage("pg", pg15Image); + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); return Effect.gen(function* () { expect(yield* resolve(dir, 15)).toEqual({ - image: toSlimImage("pg", POSTGRES_FALLBACK_IMAGE_PG15_SLIM), - configImage: POSTGRES_FALLBACK_IMAGE_PG15_SLIM, + image: pinned, + configImage: pg15Image, }); expect(yield* resolve(dir, 13)).toEqual({ - image: toSlimImage("pg", POSTGRES_FALLBACK_IMAGE_PG15_SLIM), - configImage: POSTGRES_FALLBACK_IMAGE_PG15_SLIM, + image: pinned, + configImage: pg15Image, }); rmSync(dir, { recursive: true, force: true }); }); @@ -142,14 +144,16 @@ describe("resolveDbImage", () => { }); }); - it.effect("rewrites a current PG15 pin to the slim registry", () => { + it.effect("rewrites a current PG15 pin to its catalog-pinned slim image", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const dir = withTemp(); - writePin(dir, pg15SlimTag); + writePin(dir, pg15Tag); + const pinned = expectedPinnedImage("pg", pg15Image); + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); return Effect.gen(function* () { expect(yield* resolve(dir, 15)).toEqual({ - image: toSlimImage("pg", POSTGRES_FALLBACK_IMAGE_PG15_SLIM), - configImage: POSTGRES_FALLBACK_IMAGE_PG15_SLIM, + image: pinned, + configImage: pg15Image, }); rmSync(dir, { recursive: true, force: true }); }); @@ -168,13 +172,15 @@ describe("resolveDbImage", () => { }); }); - it.effect("rewrites the current Dockerfile pin to the slim registry", () => { + it.effect("rewrites the current Dockerfile pin to its catalog-pinned slim image", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const dir = withTemp(); writePin(dir, currentPostgresTag); + const pinned = expectedPinnedImage("pg", currentPostgres); + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); return Effect.gen(function* () { expect(yield* resolve(dir, 17)).toEqual({ - image: toSlimImage("pg", currentPostgres), + image: pinned, configImage: currentPostgres, }); rmSync(dir, { recursive: true, force: true }); diff --git a/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts b/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts index e791bb63e7..dcb6df4838 100644 --- a/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts +++ b/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts @@ -10,7 +10,8 @@ import { dockerfileServiceImage, dockerfileServiceImageRaw, } from "../shared/services/dockerfile-images.ts"; -import { slimImagesEnabled, toSlimImage } from "../shared/services/slim-images.ts"; +import { expectedPinnedImage, GHCR_SLIM_IMAGE_PATTERN } from "../../tests/helpers/slim-images.ts"; +import { slimImagesEnabled } from "../shared/services/slim-images.ts"; import { resolveEdgeRuntimeImage } from "./edge-runtime-image.ts"; const currentEdgeRuntime = dockerfileServiceImageRaw("edgeruntime"); @@ -82,7 +83,7 @@ describe("resolveEdgeRuntimeImage", () => { ); }); - it.effect("rewrites the current Dockerfile pin onto the slim base", () => { + it.effect("rewrites the current Dockerfile pin to its catalog-pinned slim image", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "1"); const dir = mkdtempSync(join(tmpdir(), "edge-img-")); mkdirSync(join(dir, "supabase", ".temp"), { recursive: true }); @@ -90,10 +91,12 @@ describe("resolveEdgeRuntimeImage", () => { join(dir, "supabase", ".temp", "edge-runtime-version"), `${currentEdgeRuntimeTag}\n`, ); + const pinned = expectedPinnedImage("edgeruntime", currentEdgeRuntime); + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); return resolve(dir, 2).pipe( Effect.tap((image) => Effect.sync(() => { - expect(image).toBe(toSlimImage("edgeruntime", currentEdgeRuntime)); + expect(image).toBe(pinned); rmSync(dir, { recursive: true, force: true }); }), ), diff --git a/apps/cli/src/command-internal/stack-shadow.integration.test.ts b/apps/cli/src/command-internal/stack-shadow.integration.test.ts index 8fe9849ca3..ca7dae1a4b 100644 --- a/apps/cli/src/command-internal/stack-shadow.integration.test.ts +++ b/apps/cli/src/command-internal/stack-shadow.integration.test.ts @@ -457,59 +457,62 @@ describe("stack shadow databases", () => { 120_000, ); - it.live("prints the cleanup commands when a shadow's destroy skips its engine", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-skipped-" }); - const output = mockOutput(); - const cleanupCommand = "docker rm --force $(docker ps --all --quiet)"; - // A real registration whose database creation fails fast and whose destroy reports skipped cleanup. - const api = Layer.effect( - StackApi, - Effect.gen(function* () { - const real = yield* StackApi; - return StackApi.of({ - ...real, - create: (options) => - real.create(options).pipe( - Effect.map((stack) => ({ - ...stack, - services: { - ...stack.services, - create: () => - Effect.fail(new StackError({ operation: "create", message: "injected" })), - }, - destroy: Effect.succeed({ - runtimeCleanup: "skipped", - engine: "docker", - cleanupCommands: [cleanupCommand], - } as const), - })), - ), - }); - }), - ).pipe(Layer.provide(stackApiLayer), Layer.provide(BunServices.layer)); + it.live( + "prints the cleanup commands when a shadow's destroy skips its engine", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-skipped-" }); + const output = mockOutput(); + const cleanupCommand = "docker rm --force $(docker ps --all --quiet)"; + // A real registration whose database creation fails fast and whose destroy reports skipped cleanup. + const api = Layer.effect( + StackApi, + Effect.gen(function* () { + const real = yield* StackApi; + return StackApi.of({ + ...real, + create: (options) => + real.create(options).pipe( + Effect.map((stack) => ({ + ...stack, + services: { + ...stack.services, + create: () => + Effect.fail(new StackError({ operation: "create", message: "injected" })), + }, + destroy: Effect.succeed({ + runtimeCleanup: "skipped", + engine: "docker", + cleanupCommands: [cleanupCommand], + } as const), + })), + ), + }); + }), + ).pipe(Layer.provide(stackApiLayer), Layer.provide(BunServices.layer)); - yield* stackWithShadowDatabase(input(fs, path, root), () => Effect.void, { - runtime: "native", - }).pipe( - Effect.provide( - Layer.mergeAll( - api, - stackCatalogSetupLayer, - dbConnectionLayer, - runtimeInfoLayer, - mockCommandSettings({ workdir: root, supabaseHome: root }), - output.layer, + yield* stackWithShadowDatabase(input(fs, path, root), () => Effect.void, { + runtime: "native", + }).pipe( + Effect.provide( + Layer.mergeAll( + api, + stackCatalogSetupLayer, + dbConnectionLayer, + runtimeInfoLayer, + mockCommandSettings({ workdir: root, supabaseHome: root }), + output.layer, + ), ), - ), - Effect.flip, - ); + Effect.flip, + ); - expect(output.stderrText).toMatch( - /Warning: Docker was unavailable, so Docker resources for shadow stack [0-9a-f]{64} were not removed\. Once it is running, remove them with:\n {2}docker rm --force \$\(docker ps --all --quiet\)\n/u, - ); - }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + expect(output.stderrText).toMatch( + /Warning: Docker was unavailable, so Docker resources for shadow stack [0-9a-f]{64} were not removed\. Once it is running, remove them with:\n {2}docker rm --force \$\(docker ps --all --quiet\)\n/u, + ); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + 120_000, ); }); diff --git a/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts b/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts index 7d2a9ab96e..3cdd90f8a7 100644 --- a/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts +++ b/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts @@ -33,6 +33,48 @@ import { DeclarativeShadowDbError } from "./pgdelta.errors.ts"; import { declarativeSeamLayer } from "./pgdelta.seam.layer.ts"; import { DeclarativeSeam } from "./pgdelta.seam.service.ts"; +// This fixture catalog's pin must be keyed to the Dockerfile's own `pg` tag, or `toSlimImage` +// would find no match and fall back to the upstream (non-slim) image regardless of +// `SUPABASE_USE_SLIM_IMAGES` — masking the family-mismatch check below. `vi.hoisted` runs before +// every top-level `import` (including this file's own), so `dockerfileServiceImageRaw` isn't +// bound yet when this runs; `require` (CJS, unaffected by that ESM hoisting order) reads the +// Dockerfile directly instead, keeping this correct across every future Dockerfile bump. +const pgTag = vi.hoisted(() => { + const fs: typeof import("node:fs") = require("node:fs"); + const url: typeof import("node:url") = require("node:url"); + const dockerfilePath = url.fileURLToPath( + new URL("../../../shared/services/Dockerfile", import.meta.url), + ); + const match = /^FROM\s+supabase\/postgres:(\S+)\s+AS\s+pg$/m.exec( + fs.readFileSync(dockerfilePath, "utf8"), + ); + if (match?.[1] === undefined) throw new Error("pg tag not found in the Dockerfile"); + return match[1]; +}); + +vi.mock("@supabase/stack/internal/artifacts", () => { + const digest = "d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c"; + const nativePin = { archive: digest, manifest: digest }; + return { + catalogPins: () => [ + { + service: "database", + sourceService: "postgres", + pin: { + upstreamVersion: pgTag, + revision: 0, + image: `ghcr.io/supabase/cli/postgres:${pgTag}-r0@sha256:${digest}`, + natives: { + "darwin-arm64": nativePin, + "linux-amd64": nativePin, + "linux-arm64": nativePin, + }, + }, + }, + ], + }; +}); + /** * Integration coverage for the fully-native `declarativeSeamLayer`: `generate`/`sync`'s own * tests stub `DeclarativeSeam` entirely, so this file is the only place the real local-database @@ -207,6 +249,46 @@ describe("declarativeSeamLayer.ensureLocalPostgresImageCurrent", () => { }, ); + it.effect("flags a stale slim container when only its revision has drifted from a hotfix", () => { + vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); + const dir = tmp.current; + // Same upstream version and family as the fixture catalog's pin, but at a different + // revision (r1, a different digest) — the hotfix-drift case this model exists to catch, + // and the one a bare upstream-version comparison would mask. + const { layer } = setup(dir, { + dbInspectImage: `ghcr.io/supabase/cli/postgres:${pgTag}-r1@sha256:${"a".repeat(64)}`, + }); + return Effect.gen(function* () { + const seam = yield* DeclarativeSeam; + const exit = yield* seam.ensureLocalPostgresImageCurrent.pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + const error = failError(exit); + expect(error).toBeInstanceOf(DeclarativeShadowDbError); + expect((error as DeclarativeShadowDbError).message).toContain( + "local Postgres container image is stale", + ); + // Same family (slim vs slim): the generic remediation, not the family-mismatch wording. + expect((error as DeclarativeShadowDbError).message).not.toContain( + "same SUPABASE_USE_SLIM_IMAGES setting", + ); + expect((error as DeclarativeShadowDbError).message).toContain("--no-backup"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("passes when a slim container matches the expected image's release and digest", () => { + vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); + const dir = tmp.current; + // The exact image (release version and digest) the fixture catalog pins at r0. + const { layer } = setup(dir, { + dbInspectImage: `ghcr.io/supabase/cli/postgres:${pgTag}-r0@sha256:d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c`, + }); + return Effect.gen(function* () { + const seam = yield* DeclarativeSeam; + const exit = yield* seam.ensureLocalPostgresImageCurrent.pipe(Effect.exit); + expect(Exit.isSuccess(exit)).toBe(true); + }).pipe(Effect.provide(layer)); + }); + it.effect("bails out when inspect succeeds but the image name is unparseable", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const dir = tmp.current; diff --git a/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts b/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts index 514fba89f0..1223142205 100644 --- a/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts +++ b/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts @@ -9,7 +9,8 @@ import { resolveDbImage } from "../../../command-internal/db-image.ts"; import { readDbToml } from "../../../command-internal/db-config.toml-read.ts"; import { getRegistryImageUrl } from "../../../command-internal/docker-registry.ts"; import { isDockerDaemonUnreachable } from "../../../command-internal/docker-suggest.ts"; -import { isSlimImageRef } from "../../../shared/services/slim-images.ts"; +import { imageDigest, imageTag, isSlimImageRef } from "../../../shared/services/slim-images.ts"; +import { upstreamVersionFromTag } from "../../../shared/services/services.shared.ts"; import { isLocalDbRunning } from "../../../command-internal/db-bootstrap/local-db-running.ts"; import { startLocalDatabase } from "../../../command-internal/db-bootstrap/start-local-database.ts"; import { resolveLocalProjectId, localDbContainerId } from "../../../command-internal/docker-ids.ts"; @@ -250,20 +251,34 @@ export const declarativeSeamLayer = Layer.effect( ), Effect.map((value) => value.trim()), ); - const actualTag = dockerImageTag(actual); - const expectedTag = dockerImageTag(expected); - if (actual.length === 0 || actualTag.length === 0 || expectedTag.length === 0) { + const actualTag = imageTag(actual); + const expectedTag = imageTag(expected); + if (actual.length === 0 || actualTag === undefined || expectedTag === undefined) { return; } // Slim refs never go through a registry mirror, so a family mismatch // (e.g. a docker.io container satisfying a ghcr.io/supabase/cli - // expectation) is stale even when the tags happen to match. + // expectation) is stale even when the upstream versions happen to match. const familyMismatch = isSlimImageRef(expected) !== isSlimImageRef(actual); - if (!familyMismatch && actualTag === expectedTag) { - return; + if (!familyMismatch) { + // Same family: within slim, a `-r` hotfix bump must still be caught, so compare + // the digest when both refs carry one (most precise), or the full tag otherwise — + // never the bare upstream version, which would mask a same-upstream revision drift. + const actualDigest = imageDigest(actual); + const expectedDigest = imageDigest(expected); + const current = + actualDigest !== undefined && expectedDigest !== undefined + ? actualDigest === expectedDigest + : actualTag === expectedTag; + if (current) return; } + // Across families, only the upstream version is comparable (a slim tag's `-r` + // has no docker.io equivalent) — used to pick the remediation wording, not staleness: + // a family mismatch is always stale. + const upstreamMatches = + upstreamVersionFromTag(actualTag) === upstreamVersionFromTag(expectedTag); const remediation = - familyMismatch && actualTag === expectedTag + familyMismatch && upstreamMatches ? "The tags match but the image family does not (slim vs docker.io). Run supabase stop, then supabase start with the same SUPABASE_USE_SLIM_IMAGES setting before syncing declarative schemas." : "Run supabase stop --all --no-backup, then supabase start before syncing declarative schemas."; return yield* new DeclarativeShadowDbError({ @@ -281,13 +296,6 @@ type StartLocalDatabaseDeps = ? R : never; -function dockerImageTag(image: string): string { - const trimmed = image.trim(); - const index = trimmed.lastIndexOf(":"); - if (index < 0 || index === trimmed.length - 1) return ""; - return trimmed.slice(index + 1); -} - export function isMissingContainerInspectError(stderr: string): boolean { return stderr.toLowerCase().includes("no such container"); } diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index 2ede82a5ec..debb4eae72 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -1577,7 +1577,10 @@ describe("db start stack backend", () => { const databaseCreation: Extract = { service: "database", config: { - version: "17.6.1.173", + // Derived from the real catalog's major-17 pin, not hardcoded: `dbStart`'s stack-backend + // path resolves the desired version through the same catalog, so a bump would otherwise + // make the "resumes the existing database" test below see a spurious version mismatch. + version: postgresVersion("17"), databasePassword: Redacted.make("secret"), jwtSecret: Redacted.make("secret"), jwtExpiry: 3600, diff --git a/apps/cli/src/commands/services/services-local-stack.ts b/apps/cli/src/commands/services/services-local-stack.ts index 2bfa8f6d78..7a37515ff9 100644 --- a/apps/cli/src/commands/services/services-local-stack.ts +++ b/apps/cli/src/commands/services/services-local-stack.ts @@ -6,6 +6,7 @@ import { import { Effect, Result } from "effect"; import { loadLocalProjectContext } from "../../command-internal/local-project-context.ts"; import { envOverrideMajorVersion } from "../../command-internal/local-config-values.ts"; +import { upstreamVersionFromTag } from "../../shared/services/services.shared.ts"; import type { ServiceVersionRow } from "../../shared/services/services.shared.ts"; import type { RemoteServiceName } from "../../shared/services/services.shared.ts"; @@ -53,7 +54,7 @@ export const stackServiceVersions = Effect.fn("services.stackServiceVersions")(f return { name, // A slim revision suffix (`-rN`) repackages the same upstream release. - local: artifact.version.replace(/-r\d+$/u, ""), + local: upstreamVersionFromTag(artifact.version), remote: remoteName === undefined ? "" : (remote[remoteName] ?? ""), } satisfies ServiceVersionRow; }), diff --git a/apps/cli/src/commands/services/services.integration.test.ts b/apps/cli/src/commands/services/services.integration.test.ts index b468afc5e6..839a972d36 100644 --- a/apps/cli/src/commands/services/services.integration.test.ts +++ b/apps/cli/src/commands/services/services.integration.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "@effect/vitest"; +import { catalogPins } from "@supabase/stack/internal/artifacts"; import { BunServices } from "@effect/platform-bun"; import { CliOutput, Command } from "effect/unstable/cli"; import { @@ -33,7 +34,6 @@ import { import { mockTelemetryStateTracked, useTempWorkdir } from "../../../tests/helpers/command-mocks.ts"; import { dockerfileServiceImageRaw } from "../../shared/services/dockerfile-images.ts"; import { postgresImageForDbMajorVersion } from "../../shared/services/services.shared.ts"; -import { slimImagesEnabled } from "../../shared/services/slim-images.ts"; import { textCliOutputFormatter } from "../../shared/output/text-formatter.ts"; import { processControlLayer } from "../../shared/runtime/process-control.layer.ts"; import { TelemetryRuntime } from "../../shared/telemetry/runtime.service.ts"; @@ -43,6 +43,25 @@ import { services } from "./services.handler.ts"; const LOCAL_POSTGRES_VERSION = dockerfileServiceImageRaw("pg").split(":")[1] ?? ""; +/** + * The real stack catalog's pinned upstream version for `sourceService` — the default pin, or + * (for postgres) the additional 15.x line when `additional` is set. Derived rather than + * hardcoded, so a catalog bump never makes these "stack backend" assertions go stale. + */ +function catalogUpstreamVersion( + sourceService: string, + options: { readonly additional?: boolean } = {}, +): string { + const wantDefault = !(options.additional ?? false); + const entry = catalogPins().find( + (candidate) => candidate.sourceService === sourceService && candidate.isDefault === wantDefault, + ); + if (entry === undefined) { + throw new Error(`No catalog pin for '${sourceService}' (default=${wantDefault}).`); + } + return entry.pin.upstreamVersion; +} + /** Shape of one row in the `--output json` services array. */ const ServiceRows = Schema.Array( Schema.Struct({ @@ -189,8 +208,12 @@ const writeTempFile = Effect.fnUntraced(function* (workdir: string, name: string yield* fs.writeFileString(path.join(tempDir, name), content); }); -function postgresVersionForDbMajorVersion(majorVersion: number, slim: boolean): string { - const image = postgresImageForDbMajorVersion(majorVersion, slim); +// `postgresImageForDbMajorVersion` always returns the raw docker.io reference (slim translation +// is a downstream concern); its tag is the Dockerfile-generated one, which always matches the +// catalog's pinned upstream version, so this is the same version the CLI reports whether or not +// the slim flag is on. +function postgresVersionForDbMajorVersion(majorVersion: number): string { + const image = postgresImageForDbMajorVersion(majorVersion); if (image === undefined) { throw new Error(`Missing Postgres image for db major ${majorVersion}.`); } @@ -317,7 +340,7 @@ describe("services", () => { expect(rows).toContainEqual( expect.objectContaining({ name: "supabase/postgres", - local: postgresVersionForDbMajorVersion(15, yield* slimImagesEnabled), + local: postgresVersionForDbMajorVersion(15), }), ); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), @@ -334,13 +357,22 @@ describe("services", () => { expect(out.stderrText).toBe(""); expect(rows).toHaveLength(13); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "15.14.1.173" }), + expect.objectContaining({ + name: "ghcr.io/supabase/cli/postgres", + local: catalogUpstreamVersion("postgres", { additional: true }), + }), ); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/mailpit", local: "v1.30.2" }), + expect.objectContaining({ + name: "ghcr.io/supabase/cli/mailpit", + local: catalogUpstreamVersion("mailpit"), + }), ); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/vector", local: "0.53.0" }), + expect.objectContaining({ + name: "ghcr.io/supabase/cli/vector", + local: catalogUpstreamVersion("vector"), + }), ); expect(rows).toContainEqual( expect.objectContaining({ name: "ghcr.io/supabase/cli/storage", local: "v1.79.28" }), @@ -363,7 +395,10 @@ describe("services", () => { const rows = yield* decodeServiceRows(out.stdoutText); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "15.14.1.173" }), + expect.objectContaining({ + name: "ghcr.io/supabase/cli/postgres", + local: catalogUpstreamVersion("postgres", { additional: true }), + }), ); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); @@ -381,10 +416,16 @@ describe("services", () => { const rows = yield* decodeServiceRows(out.stdoutText); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "17.6.1.173" }), + expect.objectContaining({ + name: "ghcr.io/supabase/cli/postgres", + local: catalogUpstreamVersion("postgres"), + }), ); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/auth", local: "v2.196.0" }), + expect.objectContaining({ + name: "ghcr.io/supabase/cli/auth", + local: catalogUpstreamVersion("auth"), + }), ); expect(out.stderrText).toContain("unsupported PostgreSQL major version: 16"); expect(out.stderrText).toContain("using default stack catalog versions"); @@ -400,7 +441,10 @@ describe("services", () => { const rows = yield* decodeServiceRows(out.stdoutText); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "17.6.1.173" }), + expect.objectContaining({ + name: "ghcr.io/supabase/cli/postgres", + local: catalogUpstreamVersion("postgres"), + }), ); expect(out.stderrText).toMatch(/^failed to read config:/); expect(out.stderrText).toContain("using default stack catalog versions"); @@ -421,7 +465,7 @@ describe("services", () => { expect(rows).toContainEqual( expect.objectContaining({ name: "supabase/postgres", - local: postgresVersionForDbMajorVersion(15, yield* slimImagesEnabled), + local: postgresVersionForDbMajorVersion(15), }), ); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), @@ -448,7 +492,7 @@ major_version = 15 expect(rows).toContainEqual( expect.objectContaining({ name: "supabase/postgres", - local: postgresVersionForDbMajorVersion(15, yield* slimImagesEnabled), + local: postgresVersionForDbMajorVersion(15), }), ); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), @@ -566,7 +610,7 @@ major_version = 15 expect(rows).toContainEqual( expect.objectContaining({ name: "supabase/postgres", - local: postgresVersionForDbMajorVersion(15, yield* slimImagesEnabled), + local: postgresVersionForDbMajorVersion(15), remote: "17.6.1.200", }), ); @@ -584,7 +628,7 @@ major_version = 15 expect(stackRows).toContainEqual( expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", - local: "17.6.1.173", + local: catalogUpstreamVersion("postgres"), remote: "17.6.1.200", }), ); diff --git a/apps/cli/src/commands/start/services/vector.service.ts b/apps/cli/src/commands/start/services/vector.service.ts index fac0f51757..ec49dab78b 100644 --- a/apps/cli/src/commands/start/services/vector.service.ts +++ b/apps/cli/src/commands/start/services/vector.service.ts @@ -222,11 +222,14 @@ const VECTOR_HEALTHCHECK = { } as const; /** - * Writes the rendered `vector.yaml` via a `cat <<'EOF'` heredoc, waits on Logflare's `/health` - * (sinks would otherwise start too early), then `exec`s Vector so it stays PID 1. A TERM trap - * covers the wait so `docker stop` does not burn 10s if Logflare is still down; `-T 2` bounds each - * probe so a hung health endpoint can't defer the trap. Slim Vector ships BusyBox wget, so the - * wait uses `-q --spider` instead of GNU's `--no-verbose --tries`. + * Creates `/etc/vector` (absent from Vector 0.58's images, both slim and upstream), writes the + * rendered `vector.yaml` via a `cat <<'EOF'` heredoc, waits on Logflare's `/health` (sinks would + * otherwise start too early), then `exec`s Vector so it stays PID 1. A TERM trap covers the wait + * so `docker stop` does not burn 10s if Logflare is still down; `-T 2` bounds each probe so a hung + * health endpoint can't defer the trap. Slim Vector ships BusyBox wget, so the wait uses + * `-q --spider` instead of GNU's `--no-verbose --tries`. Both images run as root, so `mkdir -p` + * needs no separate ownership handling, and `/var/lib/vector` (the `docker_logs` source's + * checkpoint `data_dir`) already exists in both. */ export function buildVectorEntrypointScript( vectorYaml: string, @@ -237,7 +240,7 @@ export function buildVectorEntrypointScript( ? slimWgetWaitCommand(`http://${logflareId}:4000/health`) : `wget --no-verbose --tries=1 -T 2 --spider http://${logflareId}:4000/health`; return ( - "cat <<'EOF' > /etc/vector/vector.yaml\n" + + "mkdir -p /etc/vector\ncat <<'EOF' > /etc/vector/vector.yaml\n" + vectorYaml + "\nEOF\ntrap 'exit 143' TERM\nuntil " + wget + diff --git a/apps/cli/src/commands/start/services/vector.service.unit.test.ts b/apps/cli/src/commands/start/services/vector.service.unit.test.ts index c82f2bae8b..10c149fc10 100644 --- a/apps/cli/src/commands/start/services/vector.service.unit.test.ts +++ b/apps/cli/src/commands/start/services/vector.service.unit.test.ts @@ -208,7 +208,7 @@ describe("resolveVectorDockerSocketPlan", () => { describe("buildVectorEntrypointScript", () => { test("writes vector.yaml then waits on Logflare's health endpoint before exec'ing vector (start.go:449-454)", () => { expect(buildVectorEntrypointScript("VECTOR_YAML", "supabase_analytics_proj")).toBe( - "cat <<'EOF' > /etc/vector/vector.yaml\n" + + "mkdir -p /etc/vector\ncat <<'EOF' > /etc/vector/vector.yaml\n" + "VECTOR_YAML" + "\nEOF\ntrap 'exit 143' TERM\nuntil wget --no-verbose --tries=1 -T 2 --spider http://" + "supabase_analytics_proj" + diff --git a/apps/cli/src/commands/start/start.lifecycle.e2e.test.ts b/apps/cli/src/commands/start/start.lifecycle.e2e.test.ts index c7429d744b..59b3468509 100644 --- a/apps/cli/src/commands/start/start.lifecycle.e2e.test.ts +++ b/apps/cli/src/commands/start/start.lifecycle.e2e.test.ts @@ -338,4 +338,55 @@ describe("supabase start (e2e)", () => { }).pipe(Effect.provide(BunServices.layer)), START_TIMEOUT_MS + LIFECYCLE_OVERHEAD_MS + CLEANUP_TIMEOUT_MS, ); + + // Only Postgres, Logflare, and Vector run. Every other scenario here excludes Logflare and Vector, + // so this is the one that exercises the Vector image's entrypoint end to end. + it.live( + "starts Vector against a real Logflare and reaches healthy", + () => + Effect.gen(function* () { + const projectDir = yield* makeProject("sb-start-e2e-vector-"); + const path = yield* Path.Path; + const projectId = sanitizeProjectId(path.basename(projectDir)); + const vectorContainer = serviceContainerName("vector", projectId); + + const init = yield* runSupabaseEffect(["init"], { + cwd: projectDir, + exitTimeoutMs: SHORT_E2E_TIMEOUT_MS, + }); + requireCliSuccess(init, "init setup"); + yield* overridePorts(projectDir); + + const excludeArgs = SERVICE_CATALOG.flatMap((entry) => + entry.excludeKey === undefined || + entry.excludeKey === "logflare" || + entry.excludeKey === "vector" + ? [] + : ["--exclude", entry.excludeKey], + ); + const start = yield* runSupabaseEffect(["start", ...excludeArgs], { + cwd: projectDir, + exitTimeoutMs: START_TIMEOUT_MS, + }); + + if (start.exitCode !== 0) { + const logs = yield* runDockerEffect(["logs", vectorContainer]).pipe( + Effect.map(({ stdout, stderr }) => `${stdout}${stderr}`.trim() || ""), + Effect.catch((error) => Effect.succeed(``)), + ); + throw new Error( + `start failed (exit ${start.exitCode})\nstdout:\n${start.stdout}\nstderr:\n${start.stderr}\n${vectorContainer} logs:\n${logs}`, + ); + } + + const health = yield* runDockerEffect([ + "inspect", + vectorContainer, + "--format", + "{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}", + ]); + expect(health.stdout.trim()).toBe("healthy"); + }).pipe(Effect.provide(BunServices.layer)), + START_TIMEOUT_MS + LIFECYCLE_OVERHEAD_MS + CLEANUP_TIMEOUT_MS, + ); }); diff --git a/apps/cli/src/commands/start/start.services.unit.test.ts b/apps/cli/src/commands/start/start.services.unit.test.ts index 8443a5eecd..b34feac5c3 100644 --- a/apps/cli/src/commands/start/start.services.unit.test.ts +++ b/apps/cli/src/commands/start/start.services.unit.test.ts @@ -4,7 +4,10 @@ import { describe, expect, it } from "vitest"; import { dockerfileServiceImageRaw } from "../../shared/services/dockerfile-images.ts"; import type { LocalServiceVersionOverrides } from "../../shared/services/services.shared.ts"; -import { toSlimImage } from "../../shared/services/slim-images.ts"; +import { + expectedPinnedImage, + GHCR_SLIM_IMAGE_PATTERN, +} from "../../../tests/helpers/slim-images.ts"; import { serviceContainerIds, localDbContainerId } from "../../command-internal/docker-ids.ts"; import { SERVICE_CATALOG } from "../../command-internal/service-catalog.ts"; import { resolveStartGates, resolveStartImagePlan, type StartGates } from "./start.gates.ts"; @@ -248,13 +251,20 @@ describe("resolveStartImagePlan under SUPABASE_USE_SLIM_IMAGES", () => { }); it("plans slim images when the flag is on, keeping unmapped services on docker.io", () => { - expect(imageFor("gotrue", true)).toBe(toSlimImage("gotrue", currentGotrue)); - expect(imageFor("logflare", true)).toBe(toSlimImage("logflare", currentLogflare)); - expect(imageFor("vector", true)).toBe(toSlimImage("vector", currentVector)); - expect(imageFor("supavisor", true, { pooler: currentPoolerTag })).toBe( - toSlimImage("supavisor", currentPooler), - ); + const gotruePinned = expectedPinnedImage("gotrue", currentGotrue); + const logflarePinned = expectedPinnedImage("logflare", currentLogflare); + const vectorPinned = expectedPinnedImage("vector", currentVector); + const poolerPinned = expectedPinnedImage("supavisor", currentPooler); + for (const pinned of [gotruePinned, logflarePinned, vectorPinned, poolerPinned]) { + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); + } + expect(imageFor("gotrue", true)).toBe(gotruePinned); + expect(imageFor("logflare", true)).toBe(logflarePinned); + expect(imageFor("vector", true)).toBe(vectorPinned); + expect(imageFor("supavisor", true, { pooler: currentPoolerTag })).toBe(poolerPinned); + // Deliberate fallback: a historical pin the catalog doesn't carry stays on docker.io. expect(imageFor("supavisor", true, { pooler: "2.0.0" })).toBe("supabase/supavisor:2.0.0"); + // Deliberate fallback: kong has no slim build at all. expect(imageFor("kong", true)).toBe("library/kong:2.8.1"); }); }); diff --git a/apps/cli/src/commands/start/start.slim-images.e2e.test.ts b/apps/cli/src/commands/start/start.slim-images.e2e.test.ts index 65dae6a808..2a44aacfb3 100644 --- a/apps/cli/src/commands/start/start.slim-images.e2e.test.ts +++ b/apps/cli/src/commands/start/start.slim-images.e2e.test.ts @@ -2,9 +2,14 @@ import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/ import { BunServices } from "@effect/platform-bun"; import { Clock, Data, Effect, FileSystem, Layer, Path, Schema } from "effect"; import { beforeAll, describe, expect, it } from "@effect/vitest"; +import { catalogPins, type ServiceKind } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImageRaw } from "../../shared/services/dockerfile-images.ts"; -import { toSlimImage } from "../../shared/services/slim-images.ts"; +import { isSlimImageRef, toSlimImage } from "../../shared/services/slim-images.ts"; +import { + expectedPinnedImage, + GHCR_SLIM_IMAGE_PATTERN, +} from "../../../tests/helpers/slim-images.ts"; import { buildHealthCmdArg } from "../../command-internal/db-bootstrap/docker-create-args.ts"; import { slimWgetHealthcheck, @@ -63,20 +68,29 @@ const PULL_ALIASES = [ "mailpit", "kong", ] as const; -/** Slim images whose in-container probe is BusyBox wget. */ -const WGET_PROBE_ALIASES = [ - "gotrue", +/** + * Catalog services whose slim image ships BusyBox wget as its in-container probe (not the + * Dockerfile-derived image: whether the Dockerfile's tag currently matches that catalog pin is + * unrelated to whether the pinned slim image itself accepts the BusyBox argv). + */ +const WGET_PROBE_SERVICES: ReadonlyArray = [ + "auth", "realtime", "storage", - "logflare", - "supavisor", + "analytics", "vector", -] as const; + "pooler", +]; + +function wgetProbeCatalogImages(): ReadonlyArray { + return catalogPins() + .filter((entry) => WGET_PROBE_SERVICES.includes(entry.service)) + .map((entry) => entry.pin.image); +} function latestImagesToPull(): ReadonlyArray { - return [...new Set([...PULL_ALIASES, ...WGET_PROBE_ALIASES])].map((alias) => - toSlimImage(alias, dockerfileServiceImageRaw(alias)), - ); + const dockerfileImages = PULL_ALIASES.map((alias) => expectedSlimImage(alias)); + return [...new Set([...dockerfileImages, ...wgetProbeCatalogImages()])]; } function readSectionPort(config: string, section: string): number { @@ -92,8 +106,73 @@ const containerImage = Effect.fnUntraced(function* (name: string) { return stdout.trim(); }); +/** `kong` has no slim build at all, so this stays fallback-tolerant for the pull-ahead list. */ function expectedSlimImage(alias: string): string { - return toSlimImage(alias, dockerfileServiceImageRaw(alias)); + const raw = dockerfileServiceImageRaw(alias); + return toSlimImage(alias, raw) ?? raw; +} + +/** + * The Dockerfile aliases whose spec builder switches its healthcheck on `usesSlimImageRuntime` + * (`*.service.ts`). Every one of them is slim today (`expectedPinnedImage`), but the assertions + * below still derive the expected healthcheck from the image actually resolved, not from that + * assumption directly. + */ +type HealthcheckedAlias = "gotrue" | "storage" | "realtime"; + +/** Mirrors each service's own upstream (non-slim) `healthcheck.test`, `CMD` prefix included. */ +function upstreamHealthcheckTest(alias: HealthcheckedAlias): ReadonlyArray { + switch (alias) { + case "gotrue": + return [ + "CMD", + "wget", + "--no-verbose", + "--tries=1", + "--spider", + "http://127.0.0.1:9999/health", + ]; + case "storage": + return [ + "CMD", + "wget", + "--no-verbose", + "--tries=1", + "--spider", + "http://127.0.0.1:5000/status", + ]; + case "realtime": + return [ + "CMD", + "curl", + "-sSfL", + "--head", + "-o", + "/dev/null", + "-H", + `Host:${REALTIME_TENANT_ID}`, + "http://127.0.0.1:4000/api/ping", + ]; + } +} + +/** Mirrors each service's own slim (BusyBox wget) `healthcheck.test`, `CMD` prefix included. */ +function slimHealthcheckTest(alias: HealthcheckedAlias): ReadonlyArray { + switch (alias) { + case "gotrue": + return slimWgetHealthcheck("http://127.0.0.1:9999/health").test; + case "storage": + return slimWgetHealthcheck("http://127.0.0.1:5000/status").test; + case "realtime": + return slimWgetHealthcheck("http://127.0.0.1:4000/api/ping", { + header: `Host:${REALTIME_TENANT_ID}`, + }).test; + } +} + +/** The `healthcheck.test` a container running `image` must have — matched to its family. */ +function expectedHealthcheckTest(alias: HealthcheckedAlias, image: string): ReadonlyArray { + return isSlimImageRef(image) ? slimHealthcheckTest(alias) : upstreamHealthcheckTest(alias); } const containerHealthcheckTest = Effect.fnUntraced(function* (name: string) { @@ -185,19 +264,20 @@ describe("supabase start slim images (e2e)", () => { ); it.live( - "every slim wget image accepts the BusyBox healthcheck argv", + "every pinned slim wget image accepts the BusyBox healthcheck argv", () => Effect.gen(function* () { - for (const alias of WGET_PROBE_ALIASES) { - const image = expectedSlimImage(alias); + for (const entry of catalogPins()) { + if (!WGET_PROBE_SERVICES.includes(entry.service)) continue; + const image = entry.pin.image; const probe = - alias === "realtime" + entry.service === "realtime" ? slimWgetHealthcheck("http://127.0.0.1:9/", { header: `Host:${REALTIME_TENANT_ID}`, }) : slimWgetHealthcheck("http://127.0.0.1:9/"); expectBusyBoxAccepted(yield* runWgetInImage(image, probe.test.slice(2)), image); - if (alias === "vector") { + if (entry.service === "vector") { const waitArgs = slimWgetWaitCommand("http://127.0.0.1:9/").split(" ").slice(1); expectBusyBoxAccepted(yield* runWgetInImage(image, waitArgs), `${image} wait`); } @@ -207,7 +287,7 @@ describe("supabase start slim images (e2e)", () => { ); it.live( - "starts the latest slim images, serves a function without a version pin, and keeps the Dockerfile tag", + "starts each service on its resolved image (slim or upstream), matching healthchecks to family, and serves a function without a version pin", () => Effect.gen(function* () { const projectDir = yield* makeProject("sb-slim-start-e2e-"); @@ -244,25 +324,40 @@ describe("supabase start slim images (e2e)", () => { }); expect(start.exitCode, `stdout:\n${start.stdout}\nstderr:\n${start.stderr}`).toBe(0); - expect(yield* containerImage(dbContainer)).toBe(expectedSlimImage("pg")); - expect(yield* containerImage(storageContainer)).toBe(expectedSlimImage("storage")); - expect(yield* containerImage(edgeRuntimeContainer)).toBe(expectedSlimImage("edgeruntime")); + // Every alias here is slim-capable, and its default Dockerfile tag is generated from the + // catalog, so it matches a catalog pin — so each expected image is read straight from the catalog + // (`expectedPinnedImage`), independent of `toSlimImage`. + const authImage = expectedPinnedImage("gotrue", dockerfileServiceImageRaw("gotrue")); + const realtimeImage = expectedPinnedImage( + "realtime", + dockerfileServiceImageRaw("realtime"), + ); + const storageImage = expectedPinnedImage("storage", dockerfileServiceImageRaw("storage")); + for (const image of [authImage, realtimeImage, storageImage]) { + expect(image).toMatch(GHCR_SLIM_IMAGE_PATTERN); + } + + expect(yield* containerImage(dbContainer)).toBe( + expectedPinnedImage("pg", dockerfileServiceImageRaw("pg")), + ); + expect(yield* containerImage(storageContainer)).toBe(storageImage); + expect(yield* containerImage(edgeRuntimeContainer)).toBe( + expectedPinnedImage("edgeruntime", dockerfileServiceImageRaw("edgeruntime")), + ); + expect(yield* containerImage(authContainer)).toBe(authImage); + expect(yield* containerImage(realtimeContainer)).toBe(realtimeImage); expect(yield* containerHealthcheckTest(authContainer)).toEqual([ "CMD-SHELL", - buildHealthCmdArg(slimWgetHealthcheck("http://127.0.0.1:9999/health").test), + buildHealthCmdArg(expectedHealthcheckTest("gotrue", authImage)), ]); expect(yield* containerHealthcheckTest(realtimeContainer)).toEqual([ "CMD-SHELL", - buildHealthCmdArg( - slimWgetHealthcheck("http://127.0.0.1:4000/api/ping", { - header: `Host:${REALTIME_TENANT_ID}`, - }).test, - ), + buildHealthCmdArg(expectedHealthcheckTest("realtime", realtimeImage)), ]); expect(yield* containerHealthcheckTest(storageContainer)).toEqual([ "CMD-SHELL", - buildHealthCmdArg(slimWgetHealthcheck("http://127.0.0.1:5000/status").test), + buildHealthCmdArg(expectedHealthcheckTest("storage", storageImage)), ]); expect(yield* containerHealthStatus(authContainer)).toBe("healthy"); expect(yield* containerHealthStatus(realtimeContainer)).toBe("healthy"); diff --git a/apps/cli/src/shared/functions/functions.shared.unit.test.ts b/apps/cli/src/shared/functions/functions.shared.unit.test.ts index 64e5a3af49..57e310bf51 100644 --- a/apps/cli/src/shared/functions/functions.shared.unit.test.ts +++ b/apps/cli/src/shared/functions/functions.shared.unit.test.ts @@ -2,6 +2,10 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { Effect } from "effect"; import { dockerfileServiceImageRaw } from "../services/dockerfile-images.ts"; +import { + expectedPinnedImage, + GHCR_SLIM_IMAGE_PATTERN, +} from "../../../tests/helpers/slim-images.ts"; import { DENO1_EDGE_RUNTIME_VERSION, edgeRuntimeImage, @@ -22,10 +26,10 @@ describe("edgeRuntimeImage", () => { ); }); - it("rewrites the current Dockerfile tag onto the slim ghcr.io image when the flag is on", () => { - expect(edgeRuntimeImage(currentEdgeRuntimeTag, true)).toBe( - `ghcr.io/supabase/cli/edge-runtime:${currentEdgeRuntimeTag}`, - ); + it("rewrites the current Dockerfile tag onto the catalog-pinned slim ghcr.io image when the flag is on", () => { + const pinned = expectedPinnedImage("edgeruntime", rawEdgeRuntimeImage); + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); + expect(edgeRuntimeImage(currentEdgeRuntimeTag, true)).toBe(pinned); }); it("keeps a historical pin on docker.io when the flag is on", () => { diff --git a/apps/cli/src/shared/services/Dockerfile b/apps/cli/src/shared/services/Dockerfile index a9c85ee0ca..aee8c87f83 100644 --- a/apps/cli/src/shared/services/Dockerfile +++ b/apps/cli/src/shared/services/Dockerfile @@ -1,21 +1,29 @@ -# Exposed for updates by .github/dependabot.yml -FROM supabase/postgres:17.6.1.171 AS pg +# The tag pinned below is generated from packages/stack/src/Artifacts.ts by +# apps/cli/scripts/render-service-dockerfile.ts, for these aliases only: pg, pg15, mailpit, +# postgrest, pgmeta, studio, imgproxy, edgeruntime, vector, supavisor, gotrue, realtime, storage, +# logflare. Do not hand-edit those tags; a CI drift check enforces this. Run the generator after +# a catalog update. Everything else in this file (this comment, kong, pg14, the one-shot job +# images) is hand- or Dependabot-managed. +FROM supabase/postgres:17.11.0.002 AS pg +FROM supabase/postgres:15.19.0.002 AS pg15 +# Postgres 14 has no slim build; bumped by hand (Dependabot ignores supabase/postgres). +FROM supabase/postgres:14.1.0.89 AS pg14 # New always-on service alias: extend SERVICE_IMAGE_ALIASES in # shared/services/services.shared.ts and DOCKERFILE_ALIAS_BY_SERVICE in # commands/start/start.gates.ts. FROM library/kong:2.8.1 AS kong -FROM axllent/mailpit:v1.30.2 AS mailpit -FROM postgrest/postgrest:v16.3 AS postgrest +FROM axllent/mailpit:v1.31.3 AS mailpit +FROM postgrest/postgrest:v16.4 AS postgrest FROM supabase/postgres-meta:v0.99.0 AS pgmeta -FROM supabase/studio:2026.09.14-sha-4dd8a95 AS studio -FROM darthsim/imgproxy:v3.8.0 AS imgproxy +FROM supabase/studio:2026.09.28-sha-5e59b60 AS studio +FROM darthsim/imgproxy:v3.26.0 AS imgproxy FROM supabase/edge-runtime:v1.77.1 AS edgeruntime -FROM timberio/vector:0.53.0-alpine AS vector +FROM timberio/vector:0.58.0-alpine AS vector FROM supabase/supavisor:2.9.13 AS supavisor FROM supabase/gotrue:v2.197.0 AS gotrue -FROM supabase/realtime:v2.135.3 AS realtime -FROM supabase/storage-api:v1.77.0 AS storage -FROM supabase/logflare:1.50.12 AS logflare +FROM supabase/realtime:v2.140.3 AS realtime +FROM supabase/storage-api:v1.79.28 AS storage +FROM supabase/logflare:1.50.15 AS logflare # One-shot job images (not started as long-running containers); differ's alias # is resolved via dockerfileServiceImage("differ") in commands/db/diff/pgadmin-diff.ts. FROM supabase/pgadmin-schema-diff:cli-0.0.5 AS differ diff --git a/apps/cli/src/shared/services/dockerfile-go-sync.unit.test.ts b/apps/cli/src/shared/services/dockerfile-go-sync.unit.test.ts deleted file mode 100644 index 5471886e14..0000000000 --- a/apps/cli/src/shared/services/dockerfile-go-sync.unit.test.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { BunServices } from "@effect/platform-bun"; -import { describe, expect, it } from "@effect/vitest"; -import { Effect, FileSystem, Path } from "effect"; -import serviceImagesDockerfile from "./Dockerfile" with { type: "text" }; - -// The Go tree still `go:embed`s its own copy for a dependency that hasn't been removed -// yet; this keeps the two copies in sync until apps/cli-go is deleted, at which point -// this test should be deleted alongside it. -describe("Go Dockerfile sync guard", () => { - it.effect("keeps the Go tree's embedded Dockerfile byte-identical to the TS-owned copy", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const goDockerfile = yield* fs.readFileString( - path.resolve(import.meta.dirname, "../../../../cli-go/pkg/config/templates/Dockerfile"), - ); - expect(goDockerfile).toBe(serviceImagesDockerfile); - }).pipe(Effect.provide(BunServices.layer)), - ); -}); diff --git a/apps/cli/src/shared/services/services.shared.ts b/apps/cli/src/shared/services/services.shared.ts index fb86318605..3f0f553586 100644 --- a/apps/cli/src/shared/services/services.shared.ts +++ b/apps/cli/src/shared/services/services.shared.ts @@ -10,11 +10,18 @@ import { ErrorActionabilityId, } from "../telemetry/error-actionability.ts"; import { + dockerfileServiceImageRaw, dockerfileServiceImages, parseDockerfileServiceImages, type DockerfileImageSpec, } from "./dockerfile-images.ts"; -import { slimImageForAlias, slimImageForCurrentPin } from "./slim-images.ts"; +import { + imageRepository, + imageTag, + replaceImageTag, + slimImageForAlias, + slimImageForCurrentPin, +} from "./slim-images.ts"; export { parseDockerfileServiceImages } from "./dockerfile-images.ts"; @@ -116,35 +123,25 @@ export function localServiceImagesFromDockerfile( const LOCAL_SERVICE_IMAGES = localServiceImagesFromSpecs(dockerfileServiceImages); -export const POSTGRES_FALLBACK_IMAGE_PG14 = "supabase/postgres:14.1.0.89"; -/** Flag-off PG13/15 docker.io pin. */ -export const POSTGRES_FALLBACK_IMAGE_PG15 = "supabase/postgres:15.8.1.085"; -/** Published slim PG15 pin; flag-on majors 13/15 slim-translate this, not 15.8. */ -export const POSTGRES_FALLBACK_IMAGE_PG15_SLIM = "supabase/postgres:15.14.1.167"; - -export function postgresImageForDbMajorVersion( - majorVersion: number, - slim: boolean, -): string | undefined { +/** + * Resolves PG13/14/15 against the Dockerfile's `pg15`/`pg14` stages — the single version table, + * generated (`pg15`) or hand-pinned (`pg14`, no slim build) from the stack catalog. Always the + * raw docker.io reference; slim translation happens downstream via the same `toSlimImage("pg", + * …)` path every other slim-capable service uses, since a slim-capable service's Dockerfile tag + * always matches a catalog upstream version by construction. + */ +export function postgresImageForDbMajorVersion(majorVersion: number): string | undefined { switch (majorVersion) { case 13: case 15: - return slim ? POSTGRES_FALLBACK_IMAGE_PG15_SLIM : POSTGRES_FALLBACK_IMAGE_PG15; + return dockerfileServiceImageRaw("pg15"); case 14: - return POSTGRES_FALLBACK_IMAGE_PG14; + return dockerfileServiceImageRaw("pg14"); default: return undefined; } } -function replaceImageTag(image: string, tag: string): string { - const index = image.lastIndexOf(":"); - if (index === -1) { - return image; - } - return `${image.slice(0, index + 1)}${tag.trim()}`; -} - /** Applies that service's image-tag prefix when the version does not already start with it. */ export function tagForServiceVersion(service: LocalServiceVersionName, version: string): string { const trimmed = version.trim(); @@ -183,9 +180,11 @@ function localServiceImagesForOptions( if (version === undefined || version.trim().length === 0) { return baseImage === service.image ? service : { ...service, image: baseImage }; } + // `slim-images.ts`'s `replaceImageTag` doesn't trim (its own callers already do), so this + // path — the only one that skips `tagForServiceVersion`'s trim — trims here. const pin = normalizeVersionTags ? tagForServiceVersion(service.localService, version) - : version; + : version.trim(); if (override === undefined && slim) { return { ...service, @@ -216,17 +215,26 @@ export interface ServiceVersionRow { readonly remote: string; } +/** A release tag's `-r` suffix, matching the slim-services revision grammar. */ +const RELEASE_REVISION_SUFFIX = /^(?.+)-r(?:0|[1-9][0-9]*)$/; + +/** Strips a slim release tag's `-r` suffix, if any, back to its upstream version. */ +export function upstreamVersionFromTag(tag: string): string { + return RELEASE_REVISION_SUFFIX.exec(tag)?.groups?.upstream ?? tag; +} + function toServiceVersionRow( service: ServiceImageSpec, remote: Partial> = {}, ): ServiceVersionRow { - const tagSeparator = service.image.lastIndexOf(":"); - if (tagSeparator === -1) { + // `@`-aware (via `imageTag`/`imageRepository`): a slim catalog pin's image carries a + // `@sha256:…` digest after the tag. + const name = imageRepository(service.image); + const tag = imageTag(service.image); + if (name === undefined || tag === undefined) { throw new Error(`Invalid service image entry: ${service.image}`); } - - const name = service.image.slice(0, tagSeparator); - const local = service.image.slice(tagSeparator + 1); + const local = upstreamVersionFromTag(tag); return { name, diff --git a/apps/cli/src/shared/services/services.shared.unit.test.ts b/apps/cli/src/shared/services/services.shared.unit.test.ts index 2471e2423c..ff9e180f5c 100644 --- a/apps/cli/src/shared/services/services.shared.unit.test.ts +++ b/apps/cli/src/shared/services/services.shared.unit.test.ts @@ -1,17 +1,51 @@ import { describe, expect, it, test } from "@effect/vitest"; import { Effect, Redacted } from "effect"; import { FetchHttpClient } from "effect/unstable/http"; +import { vi } from "vitest"; import serviceImagesDockerfile from "./Dockerfile" with { type: "text" }; +import { dockerfileServiceImageRaw } from "./dockerfile-images.ts"; import { fetchLinkedServiceVersions, listLocalServiceVersions, localServiceImagesFromDockerfile, + mergeRemoteServiceVersions, parseDockerfileServiceImages, postgresImageForDbMajorVersion, renderServicesTable, renderServicesWarning, } from "./services.shared.ts"; +// Only `auth` is pinned in this fixture catalog, at the current Dockerfile-independent version +// `v2.197.0-r0`, with a realistic (non-placeholder) fixture digest built to the real +// `ArtifactPin`/`NativePin` shape from `@supabase/stack/internal/artifacts`. Every other service +// is deliberately absent, so `toSlimImage` falls through to the upstream image for them — the +// permanent state for a non-slim-capable alias (kong, `pg14`, the job images): the Dockerfile's +// slim-capable lines are generated from the catalog now, so they never disagree with it. +// `vi.mock` factories are hoisted above every other top-level statement, so the fixture is +// inlined rather than referencing an outer const. +vi.mock("@supabase/stack/internal/artifacts", () => { + const digest = "260e94edb8d402555791146fcf70b8e90efdc6a81877a04e5aa26f0f416a5dd7"; + const nativePin = { archive: digest, manifest: digest }; + return { + catalogPins: () => [ + { + service: "auth", + sourceService: "auth", + pin: { + upstreamVersion: "v2.197.0", + revision: 0, + image: `ghcr.io/supabase/cli/auth:v2.197.0-r0@sha256:${digest}`, + natives: { + "darwin-arm64": nativePin, + "linux-amd64": nativePin, + "linux-arm64": nativePin, + }, + }, + }, + ], + }; +}); + const ACCESS_TOKEN = Redacted.make(`sbp_${"a".repeat(40)}`); const PROJECT_REF = "abcdefghijklmnopqrst"; @@ -75,26 +109,39 @@ describe("services shared", () => { ]); }); - test("keeps the established PG13/15 fallback unless the slim flag is on", () => { - expect(postgresImageForDbMajorVersion(13, false)).toBe("supabase/postgres:15.8.1.085"); - expect(postgresImageForDbMajorVersion(15, false)).toBe("supabase/postgres:15.8.1.085"); - expect(postgresImageForDbMajorVersion(13, true)).toBe("supabase/postgres:15.14.1.167"); - expect(postgresImageForDbMajorVersion(15, true)).toBe("supabase/postgres:15.14.1.167"); + test("resolves PG13/14/15 from the Dockerfile's pg15/pg14 stages, regardless of the slim flag", () => { + const pg15 = dockerfileServiceImageRaw("pg15"); + const pg14 = dockerfileServiceImageRaw("pg14"); + expect(postgresImageForDbMajorVersion(13)).toBe(pg15); + expect(postgresImageForDbMajorVersion(15)).toBe(pg15); + expect(postgresImageForDbMajorVersion(14)).toBe(pg14); + // Always the raw docker.io reference; slim translation is a separate, downstream concern + // (`toSlimImage`/`slimImageForCurrentPin`), so this function itself doesn't read the flag. + expect(postgresImageForDbMajorVersion(13)).toBe(pg15); + expect(postgresImageForDbMajorVersion(15)).toBe(pg15); + expect(postgresImageForDbMajorVersion(14)).toBe(pg14); }); - test("lists slim images when SUPABASE_USE_SLIM_IMAGES is set", () => { - expect(listLocalServiceVersions({ slim: true }).map((row) => row.name)).toEqual([ - "ghcr.io/supabase/cli/postgres", - "ghcr.io/supabase/cli/auth", - "ghcr.io/supabase/cli/postgrest", - "ghcr.io/supabase/cli/realtime", - "ghcr.io/supabase/cli/storage", - "ghcr.io/supabase/cli/edge-runtime", - "ghcr.io/supabase/cli/studio", - "ghcr.io/supabase/cli/pgmeta", - "ghcr.io/supabase/cli/analytics", - "ghcr.io/supabase/cli/pooler", - ]); + test("slim-translates a version override that matches a catalog pin", () => { + expect( + listLocalServiceVersions({ slim: true, serviceVersions: { auth: "v2.197.0" } }), + ).toContainEqual({ + name: "ghcr.io/supabase/cli/auth", + // The catalog's release version (`v2.197.0-r0`) is a Dockerfile/manifest tag; the row shows + // the upstream version so a `supabase services` mismatch check compares upstream to upstream. + local: "v2.197.0", + remote: "", + }); + }); + + test("keeps a version override that isn't in the catalog on docker.io", () => { + expect( + listLocalServiceVersions({ slim: true, serviceVersions: { storage: "v1.70.3" } }), + ).toContainEqual({ + name: "supabase/storage-api", + local: "v1.70.3", + remote: "", + }); }); test("keeps historical pins on docker.io when slimCurrentPinOnly is set", () => { @@ -122,16 +169,6 @@ describe("services shared", () => { ).toContainEqual({ name: "supabase/gotrue", local: "v2.151.0", remote: "" }); }); - test("slim-translates catalog version overrides that are not the Dockerfile pin", () => { - expect( - listLocalServiceVersions({ slim: true, serviceVersions: { storage: "v1.70.3" } }), - ).toContainEqual({ - name: "ghcr.io/supabase/cli/storage", - local: "v1.70.3", - remote: "", - }); - }); - // Explicit overrides keep their registry; a serviceVersions pin still rewrites the tag. test("leaves explicit image overrides on docker.io when SUPABASE_USE_SLIM_IMAGES is set", () => { const rows = listLocalServiceVersions({ @@ -152,6 +189,27 @@ describe("services shared", () => { ); }); + // A digest-carrying override paired with a serviceVersions pin exercises the same + // `replaceImageTag` used for the plain-tag case above; it must drop the stale digest + // rather than splice the new tag into it (`…-r0@sha256:`). + test("rewrites the tag on a digest-carrying image override, dropping the stale digest", () => { + const rows = listLocalServiceVersions({ + slim: true, + imageOverrides: { + postgres: + "ghcr.io/supabase/cli/postgres:17.6.1.173-r0@sha256:24e96b8d5daf90f67a62b5593d3008446744007e0a57e302d269c02a4e459e8f", + }, + normalizeVersionTags: false, + serviceVersions: { postgres: "17.6.1.200" }, + }); + + expect(rows).toContainEqual({ + name: "ghcr.io/supabase/cli/postgres", + local: "17.6.1.200", + remote: "", + }); + }); + test("can preserve raw local service version overrides", () => { expect( listLocalServiceVersions({ @@ -501,4 +559,20 @@ describe("services shared", () => { ]), ).toContain("supabase/postgres:17.6.1.132 => 17.6.1.200"); }); + + test("compares upstream versions for a slim catalog pin, not the release tag", () => { + const rows = mergeRemoteServiceVersions( + { auth: "v2.197.0" }, + { slim: true, serviceVersions: { auth: "v2.197.0" } }, + ); + + expect(rows).toContainEqual({ + name: "ghcr.io/supabase/cli/auth", + local: "v2.197.0", + remote: "v2.197.0", + }); + // The pinned image's release tag (`v2.197.0-r0@sha256:…`) never surfaces as a mismatch + // against the upstream-only remote version. + expect(renderServicesWarning(rows)).toBeUndefined(); + }); }); diff --git a/apps/cli/src/shared/services/slim-images.catalog-alignment.unit.test.ts b/apps/cli/src/shared/services/slim-images.catalog-alignment.unit.test.ts new file mode 100644 index 0000000000..ea5fd0dae4 --- /dev/null +++ b/apps/cli/src/shared/services/slim-images.catalog-alignment.unit.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { dockerfileServiceImageRaw } from "./dockerfile-images.ts"; +import { slimCatalogPin, toSlimImage } from "./slim-images.ts"; + +/** + * Every slim-capable Dockerfile alias, against the real (unmocked) catalog. The Dockerfile is + * generated from that same catalog (`render-service-dockerfile.ts`), so each alias's raw tag is + * always one of the catalog's own upstream versions by construction — no fixture, no + * `vi.mock`, exercising the Dockerfile/catalog alignment end to end. + */ +const SLIM_CAPABLE_ALIASES = [ + "pg", + "pg15", + "gotrue", + "postgrest", + "realtime", + "storage", + "edgeruntime", + "studio", + "pgmeta", + "logflare", + "supavisor", + "vector", + "imgproxy", + "mailpit", +] as const; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("slim mode against the real catalog", () => { + it.each(SLIM_CAPABLE_ALIASES)("resolves the %s alias to its pinned catalog image", (alias) => { + vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); + const raw = dockerfileServiceImageRaw(alias); + const pin = slimCatalogPin(alias, raw); + expect(pin).toBeDefined(); + const resolved = toSlimImage(alias, raw); + expect(resolved).toBeDefined(); + expect(resolved).toMatch(/^ghcr\.io\/supabase\/cli\//); + }); + + it("has no slim entry for aliases with no slim build", () => { + for (const alias of ["kong", "pg14", "differ", "migra", "pgprove"]) { + const raw = dockerfileServiceImageRaw(alias); + expect(toSlimImage(alias, raw)).toBeUndefined(); + } + }); +}); diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index e5fb6b0521..440e3a5284 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -1,3 +1,4 @@ +import { catalogPins } from "@supabase/stack/internal/artifacts"; import { Config, ConfigProvider, Effect, Option } from "effect"; const SLIM_IMAGES_ENV = "SUPABASE_USE_SLIM_IMAGES"; @@ -5,11 +6,13 @@ const SLIM_IMAGE_PREFIX = "ghcr.io/supabase/cli/"; /** * Maps embedded-Dockerfile aliases onto the slim service catalog. Aliases with - * no slim build (kong, the `differ`/`migra`/`pgprove` job images) are absent and - * keep their docker.io reference. OrioleDB tags are excluded in `slimCatalogPin`. + * no slim build (kong, `pg14`, the `differ`/`migra`/`pgprove` job images) are + * absent and keep their docker.io reference. OrioleDB tags are excluded in + * `slimCatalogPin`. */ const SLIM_SERVICE_BY_ALIAS = { pg: "postgres", + pg15: "postgres", gotrue: "auth", postgrest: "postgrest", realtime: "realtime", @@ -70,7 +73,7 @@ export interface SlimCatalogPin { } /** OrioleDB tags are docker.io-only; slim-services does not publish them. */ -export function isOrioleImage(image: string): boolean { +function isOrioleImage(image: string): boolean { const tag = imageTag(image); return tag !== undefined && tag.toLowerCase().includes("orioledb"); } @@ -98,35 +101,71 @@ export function slimCatalogPin(alias: string, image: string): SlimCatalogPin | u } /** - * Rewrites a docker.io image reference to its `ghcr.io/supabase/cli` slim - * equivalent, keeping the pin's version. This helper owns tag normalization - * (`v`-prefixing, `tagPrefix`), so pins that differ only in prefix between the - * two registries (`supavisor`, `logflare`) land on the right slim tag. Vector's - * docker.io tags carry an `-alpine` variant suffix that the slim build does - * not publish, so the strip is scoped to `vector` only — an `-alpine`-suffixed - * pin on any other service is a real tag, not a variant marker. + * Looks up the pinned catalog image (with its published `@sha256` digest) for + * `service` whose `upstreamVersion` equals `version`. Reads the same catalog + * `apps/cli`'s stack-independent clients use, keyed by the slim-services + * `sourceService` name (which matches this module's `SlimServiceName`). */ -export function toSlimImage(alias: string, image: string): string { +function catalogImageFor(service: SlimServiceName, upstreamVersion: string): string | undefined { + for (const entry of catalogPins()) { + if (entry.sourceService === service && entry.pin.upstreamVersion === upstreamVersion) { + return entry.pin.image; + } + } + return undefined; +} + +/** + * Resolves the catalog's pinned slim image (repository, release version and + * digest) whose `upstreamVersion` normalizes to `image`'s tag for `alias`. + * This owns tag normalization (`v`-prefixing, `tagPrefix`, vector's `-alpine` + * strip) via {@link slimCatalogPin}, so pins that differ only in prefix + * between the two registries (`supavisor`, `logflare`) still match. Returns `undefined` whenever + * no catalog pin matches `alias` and `image`'s tag — callers then keep the upstream (non-slim) + * image instead of guessing a slim tag. That covers more than "no slim build": an alias with no + * slim build at all (kong, `pg14`, the one-shot job images); an excluded tag on an alias that + * does have one (an OrioleDB `pg` tag, which {@link slimCatalogPin} always excludes); and a tag + * the catalog simply doesn't pin (an upstream version the catalog hasn't caught up to yet, or a + * hosted-project override from `supabase link` that doesn't match the pinned upstream version). + */ +export function toSlimImage(alias: string, image: string): string | undefined { const pin = slimCatalogPin(alias, image); if (pin === undefined) { - return image; + return undefined; } - return `${SLIM_IMAGE_PREFIX}${pin.service}:${pin.version}`; + return catalogImageFor(pin.service, pin.version); } /** `toSlimImage` behind the feature flag; a no-op while the flag is off. */ export function slimImageForAlias(alias: string, image: string, enabled: boolean): string { - return enabled ? toSlimImage(alias, image) : image; + return enabled ? (toSlimImage(alias, image) ?? image) : image; } +/** The tag portion of `image`, ignoring any `@sha256:…` digest suffix. */ export function imageTag(image: string): string | undefined { - const tagSeparator = image.lastIndexOf(":"); - return tagSeparator === -1 ? undefined : image.slice(tagSeparator + 1); + const withoutDigest = image.split("@")[0] ?? image; + const tagSeparator = withoutDigest.lastIndexOf(":"); + return tagSeparator === -1 ? undefined : withoutDigest.slice(tagSeparator + 1); +} + +/** The repository portion of `image` (before the tag), the other half of `imageTag`'s split. */ +export function imageRepository(image: string): string | undefined { + const withoutDigest = image.split("@")[0] ?? image; + const tagSeparator = withoutDigest.lastIndexOf(":"); + return tagSeparator === -1 ? undefined : withoutDigest.slice(0, tagSeparator); +} + +/** The `@sha256:…` digest suffix of `image`, if it carries one. */ +export function imageDigest(image: string): string | undefined { + const at = image.indexOf("@"); + return at === -1 ? undefined : image.slice(at + 1); } -function replaceImageTag(image: string, tag: string): string { - const tagSeparator = image.lastIndexOf(":"); - return tagSeparator === -1 ? image : `${image.slice(0, tagSeparator + 1)}${tag}`; +/** Replaces `image`'s tag with `tag`, dropping any `@sha256:…` digest — a new tag invalidates it. */ +export function replaceImageTag(image: string, tag: string): string { + const withoutDigest = image.split("@")[0] ?? image; + const tagSeparator = withoutDigest.lastIndexOf(":"); + return tagSeparator === -1 ? image : `${withoutDigest.slice(0, tagSeparator + 1)}${tag}`; } /** @@ -167,7 +206,7 @@ export function slimImageForCurrentPin( if (trimmed.length > 0 && !pinMatchesCurrentImage(alias, trimmed, currentRawImage)) { return tagged; } - return toSlimImage(alias, tagged); + return toSlimImage(alias, tagged) ?? tagged; } /** Slim images are published only under this prefix; single home for the check. */ diff --git a/apps/cli/src/shared/services/slim-images.unit.test.ts b/apps/cli/src/shared/services/slim-images.unit.test.ts index 654ea9b1fd..5064bd026a 100644 --- a/apps/cli/src/shared/services/slim-images.unit.test.ts +++ b/apps/cli/src/shared/services/slim-images.unit.test.ts @@ -4,6 +4,7 @@ import { vi } from "vitest"; import { dockerfileServiceImageRaw } from "./dockerfile-images.ts"; import { + imageTag, pinMatchesCurrentImage, slimCatalogPin, slimImageForAlias, @@ -13,93 +14,171 @@ import { usesSlimImageRuntime, } from "./slim-images.ts"; -describe("toSlimImage", () => { +// Only `auth` is pinned in this fixture catalog, at `v2.197.0-r0`, with a realistic +// (non-placeholder) fixture digest built to the real `ArtifactPin`/`NativePin` shape from +// `@supabase/stack/internal/artifacts`. Every other service is deliberately absent, so +// `toSlimImage` falls through to the upstream image for them — the permanent state for a +// non-slim-capable alias (kong, `pg14`, the job images): the Dockerfile's slim-capable lines +// are generated from the catalog now, so they never disagree with it. `vi.mock` factories are +// hoisted above every other top-level statement, so the fixture is inlined rather than +// referencing an outer const. +vi.mock("@supabase/stack/internal/artifacts", () => { + const digest = "d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c"; + const nativePin = { archive: digest, manifest: digest }; + return { + catalogPins: () => [ + { + service: "auth", + sourceService: "auth", + pin: { + upstreamVersion: "v2.197.0", + revision: 0, + image: `ghcr.io/supabase/cli/auth:v2.197.0-r0@sha256:${digest}`, + natives: { + "darwin-arm64": nativePin, + "linux-amd64": nativePin, + "linux-arm64": nativePin, + }, + }, + }, + ], + }; +}); + +const AUTH_FIXTURE_PIN_IMAGE = + "ghcr.io/supabase/cli/auth:v2.197.0-r0@sha256:d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c"; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("slimCatalogPin", () => { it.each([ - ["pg", "ghcr.io/supabase/cli/postgres"], - ["gotrue", "ghcr.io/supabase/cli/auth"], - ["postgrest", "ghcr.io/supabase/cli/postgrest"], - ["realtime", "ghcr.io/supabase/cli/realtime"], - ["storage", "ghcr.io/supabase/cli/storage"], - ["edgeruntime", "ghcr.io/supabase/cli/edge-runtime"], - ["studio", "ghcr.io/supabase/cli/studio"], - ["pgmeta", "ghcr.io/supabase/cli/pgmeta"], - ["logflare", "ghcr.io/supabase/cli/analytics"], - ["supavisor", "ghcr.io/supabase/cli/pooler"], - ["vector", "ghcr.io/supabase/cli/vector"], - ["imgproxy", "ghcr.io/supabase/cli/imgproxy"], - ["mailpit", "ghcr.io/supabase/cli/mailpit"], - ])("maps the %s manifest pin onto %s", (alias, repository) => { - const translated = toSlimImage(alias, dockerfileServiceImageRaw(alias)); - expect(translated.slice(0, translated.lastIndexOf(":"))).toBe(repository); - }); - - it("keeps a non-current pin instead of the catalog default", () => { - expect(toSlimImage("pg", "supabase/postgres:17.6.1.164")).toBe( - "ghcr.io/supabase/cli/postgres:17.6.1.164", - ); - expect(toSlimImage("studio", "supabase/studio:2026.08.17-sha-0c1da8f")).toBe( - "ghcr.io/supabase/cli/studio:2026.08.17-sha-0c1da8f", - ); + ["pg", "postgres"], + ["pg15", "postgres"], + ["gotrue", "auth"], + ["postgrest", "postgrest"], + ["realtime", "realtime"], + ["storage", "storage"], + ["edgeruntime", "edge-runtime"], + ["studio", "studio"], + ["pgmeta", "pgmeta"], + ["logflare", "analytics"], + ["supavisor", "pooler"], + ["vector", "vector"], + ["imgproxy", "imgproxy"], + ["mailpit", "mailpit"], + ])("maps the %s alias onto the %s slim service", (alias, service) => { + const pin = slimCatalogPin(alias, dockerfileServiceImageRaw(alias)); + expect(pin?.service).toBe(service); + }); + + it("keeps a non-current pin's version verbatim", () => { + expect(slimCatalogPin("pg", "supabase/postgres:17.6.1.164")).toEqual({ + service: "postgres", + version: "17.6.1.164", + }); + expect(slimCatalogPin("studio", "supabase/studio:2026.08.17-sha-0c1da8f")).toEqual({ + service: "studio", + version: "2026.08.17-sha-0c1da8f", + }); }); - // Fixed pins, not manifest pins: dependabot bumps the manifest, so spelling - // out a current pin here would fail on every bump. The `it.each` above covers - // the part that must track it (the repository each alias maps to). it("keeps a single v on pins already prefixed on docker.io", () => { - expect(toSlimImage("realtime", "supabase/realtime:v2.130.0")).toBe( - "ghcr.io/supabase/cli/realtime:v2.130.0", - ); - expect(toSlimImage("storage", "supabase/storage-api:v1.72.1")).toBe( - "ghcr.io/supabase/cli/storage:v1.72.1", - ); - expect(toSlimImage("gotrue", "supabase/gotrue:V2.196.0")).toBe( - "ghcr.io/supabase/cli/auth:v2.196.0", - ); + expect(slimCatalogPin("realtime", "supabase/realtime:v2.130.0")).toEqual({ + service: "realtime", + version: "v2.130.0", + }); + expect(slimCatalogPin("storage", "supabase/storage-api:v1.72.1")).toEqual({ + service: "storage", + version: "v1.72.1", + }); + expect(slimCatalogPin("gotrue", "supabase/gotrue:V2.196.0")).toEqual({ + service: "auth", + version: "v2.196.0", + }); }); it("v-prefixes pins whose slim tag scheme differs from docker.io's", () => { - expect(toSlimImage("supavisor", "supabase/supavisor:2.9.10")).toBe( - "ghcr.io/supabase/cli/pooler:v2.9.10", - ); - expect(toSlimImage("logflare", "supabase/logflare:1.50.4")).toBe( - "ghcr.io/supabase/cli/analytics:v1.50.4", - ); - expect(toSlimImage("pgmeta", "supabase/postgres-meta:v0.98.0")).toBe( - "ghcr.io/supabase/cli/pgmeta:v0.98.0", - ); + expect(slimCatalogPin("supavisor", "supabase/supavisor:2.9.10")).toEqual({ + service: "pooler", + version: "v2.9.10", + }); + expect(slimCatalogPin("logflare", "supabase/logflare:1.50.4")).toEqual({ + service: "analytics", + version: "v1.50.4", + }); + expect(slimCatalogPin("pgmeta", "supabase/postgres-meta:v0.98.0")).toEqual({ + service: "pgmeta", + version: "v0.98.0", + }); }); - it("keeps OrioleDB tags on docker.io", () => { - expect(toSlimImage("pg", "supabase/postgres:16.0.0.1-orioledb")).toBe( - "supabase/postgres:16.0.0.1-orioledb", - ); - expect(toSlimImage("pg", "supabase/postgres:orioledb-15.1.0.55")).toBe( - "supabase/postgres:orioledb-15.1.0.55", - ); + it("excludes OrioleDB tags", () => { expect(slimCatalogPin("pg", "supabase/postgres:16.0.0.1-orioledb")).toBeUndefined(); + expect(slimCatalogPin("pg", "supabase/postgres:orioledb-15.1.0.55")).toBeUndefined(); }); it("strips vector's docker.io -alpine variant suffix", () => { - expect(toSlimImage("vector", "timberio/vector:0.53.0-alpine")).toBe( - "ghcr.io/supabase/cli/vector:0.53.0", - ); + expect(slimCatalogPin("vector", "timberio/vector:0.53.0-alpine")).toEqual({ + service: "vector", + version: "0.53.0", + }); }); it("does not strip -alpine from a non-vector service's tag", () => { - expect(toSlimImage("studio", "supabase/studio:2026.08.17-alpine")).toBe( - "ghcr.io/supabase/cli/studio:2026.08.17-alpine", - ); + expect(slimCatalogPin("studio", "supabase/studio:2026.08.17-alpine")).toEqual({ + service: "studio", + version: "2026.08.17-alpine", + }); }); - it("passes through aliases with no slim build", () => { - for (const alias of ["kong", "differ", "migra", "pgprove"]) { - const image = dockerfileServiceImageRaw(alias); - expect(toSlimImage(alias, image)).toBe(image); + it("is absent for aliases with no slim build", () => { + for (const alias of ["kong", "pg14", "differ", "migra", "pgprove"]) { + expect(slimCatalogPin(alias, dockerfileServiceImageRaw(alias))).toBeUndefined(); } }); - it("passes through an untagged reference", () => { - expect(toSlimImage("pg", "supabase/postgres")).toBe("supabase/postgres"); + it("is absent for an untagged reference", () => { + expect(slimCatalogPin("pg", "supabase/postgres")).toBeUndefined(); + }); +}); + +describe("toSlimImage", () => { + it("returns the catalog's pinned image (with its digest) when the tag matches a catalog upstream version", () => { + expect(toSlimImage("gotrue", "supabase/gotrue:v2.197.0")).toBe(AUTH_FIXTURE_PIN_IMAGE); + // The alias-normalization prefix (V -> v) still applies before the catalog match. + expect(toSlimImage("gotrue", "supabase/gotrue:V2.197.0")).toBe(AUTH_FIXTURE_PIN_IMAGE); + }); + + it("returns undefined, keeping the upstream image, when the tag isn't in the catalog", () => { + expect(toSlimImage("gotrue", "supabase/gotrue:v2.100.0")).toBeUndefined(); + // `pg` has no entry at all in this fixture catalog. + expect(toSlimImage("pg", "supabase/postgres:17.6.1.164")).toBeUndefined(); + }); + + it("returns undefined for aliases and tags slimCatalogPin already excludes", () => { + expect(toSlimImage("pg", "supabase/postgres:16.0.0.1-orioledb")).toBeUndefined(); + expect(toSlimImage("kong", dockerfileServiceImageRaw("kong"))).toBeUndefined(); + expect(toSlimImage("pg", "supabase/postgres")).toBeUndefined(); + }); +}); + +describe("imageTag", () => { + it("returns the release tag, ignoring an @sha256 digest", () => { + expect( + imageTag( + "ghcr.io/supabase/cli/auth:v2.197.0-r0@sha256:d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c", + ), + ).toBe("v2.197.0-r0"); + }); + + it("returns the tag on a plain (digest-less) reference", () => { + expect(imageTag("supabase/gotrue:v2.197.0")).toBe("v2.197.0"); + }); + + it("returns undefined on an untagged reference", () => { + expect(imageTag("supabase/postgres")).toBeUndefined(); }); }); @@ -144,14 +223,20 @@ describe("slimImagesEnabled", () => { describe("slimImageForAlias", () => { it("is a no-op while the flag is off", () => { - expect(slimImageForAlias("pg", "supabase/postgres:17.6.1.165", false)).toBe( - "supabase/postgres:17.6.1.165", + expect(slimImageForAlias("gotrue", "supabase/gotrue:v2.197.0", false)).toBe( + "supabase/gotrue:v2.197.0", + ); + }); + + it("translates when the flag is on and the tag matches the catalog", () => { + expect(slimImageForAlias("gotrue", "supabase/gotrue:v2.197.0", true)).toBe( + AUTH_FIXTURE_PIN_IMAGE, ); }); - it("translates when the flag is on", () => { + it("keeps the upstream image when the flag is on but the tag isn't in the catalog", () => { expect(slimImageForAlias("pg", "supabase/postgres:17.6.1.165", true)).toBe( - "ghcr.io/supabase/cli/postgres:17.6.1.165", + "supabase/postgres:17.6.1.165", ); }); }); @@ -179,24 +264,30 @@ describe("pinMatchesCurrentImage", () => { }); describe("slimImageForCurrentPin", () => { - it("slim-translates the current pin and leaves a historical pin on docker.io", () => { - const current = dockerfileServiceImageRaw("storage"); - const currentTag = current.split(":")[1] ?? ""; - expect(slimImageForCurrentPin("storage", current, undefined, true)).toBe( - toSlimImage("storage", current), - ); - expect(slimImageForCurrentPin("storage", current, currentTag, true)).toBe( - toSlimImage("storage", current), + it("slim-translates the current pin using the catalog's pinned digest", () => { + const current = "supabase/gotrue:v2.197.0"; + expect(slimImageForCurrentPin("gotrue", current, undefined, true)).toBe(AUTH_FIXTURE_PIN_IMAGE); + expect(slimImageForCurrentPin("gotrue", current, "v2.197.0", true)).toBe( + AUTH_FIXTURE_PIN_IMAGE, ); - expect(slimImageForCurrentPin("storage", current, "v1.67.0", true)).toBe( - "supabase/storage-api:v1.67.0", + expect(slimImageForCurrentPin("gotrue", current, "v1.67.0", true)).toBe( + "supabase/gotrue:v1.67.0", ); }); it("is a no-op while the flag is off", () => { - const current = dockerfileServiceImageRaw("storage"); - expect(slimImageForCurrentPin("storage", current, "v1.67.0", false)).toBe( - "supabase/storage-api:v1.67.0", + const current = "supabase/gotrue:v2.197.0"; + expect(slimImageForCurrentPin("gotrue", current, "v1.67.0", false)).toBe( + "supabase/gotrue:v1.67.0", + ); + }); + + it("falls back to the upstream image on the linked-pin fallback path (a hosted version that doesn't match the pin)", () => { + const current = "supabase/gotrue:v2.197.0"; + // The linked project's hosted version (v1.67.0) differs from the catalog's pinned + // upstream version (v2.197.0): stay on the upstream (non-slim) image instead of guessing. + expect(slimImageForCurrentPin("gotrue", current, "v1.67.0", true)).toBe( + "supabase/gotrue:v1.67.0", ); }); }); diff --git a/apps/cli/tests/helpers/slim-images.ts b/apps/cli/tests/helpers/slim-images.ts new file mode 100644 index 0000000000..0bab6745fc --- /dev/null +++ b/apps/cli/tests/helpers/slim-images.ts @@ -0,0 +1,29 @@ +import { catalogPins } from "@supabase/stack/internal/artifacts"; + +import { slimCatalogPin } from "../../src/shared/services/slim-images.ts"; + +/** A regression to the docker.io fallback must fail an assertion built from this. */ +export const GHCR_SLIM_IMAGE_PATTERN = /^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/; + +/** + * The catalog's own pinned image for `alias`'s (docker.io) `image` — read straight from + * `catalogPins()`, independent of `toSlimImage`, so a test asserting against this actually + * exercises the catalog lookup instead of passing whether or not it resolves (a default + * Dockerfile tag is generated from the catalog, so it always matches a catalog pin). Only reuses `slimCatalogPin` for alias + * and tag normalization (`v`-prefixing), not the catalog image lookup itself. Throws when + * nothing is pinned, so a caller never silently falls back to a weaker assertion. + */ +export function expectedPinnedImage(alias: string, image: string): string { + const pin = slimCatalogPin(alias, image); + if (pin === undefined) { + throw new Error(`no slim catalog pin for ${alias} ${image}`); + } + const entry = catalogPins().find( + (candidate) => + candidate.sourceService === pin.service && candidate.pin.upstreamVersion === pin.version, + ); + if (entry === undefined) { + throw new Error(`no catalog pin for ${pin.service} ${pin.version}`); + } + return entry.pin.image; +} diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index cdcc4fb4ca..cba056ffe3 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -1,48 +1,191 @@ # 0026. Slim image and native artifact mirrors -**Status**: proposed -**Date**: 2026-09-17 +**Status**: accepted +**Date**: 2026-09-29 ## Problem Statement -Slim service images publish to `ghcr.io/supabase/cli/:`. Native archives publish to GitHub Releases. A broken build must be replaceable under the **same** upstream version string (`force=true`); bumping the tag is not an option. +slim-services used to publish each artifact under the exact upstream version string, so the +only way to fix bad packaging was `force=true`: overwrite the GitHub release assets +(`--clobber`), move the GHCR/ECR tags, and overwrite the S3 objects. That broke already-released +CLIs pinned to that version, and the CLI itself resolved artifacts inconsistently: some catalog +images were pinned by `@sha256` and some were tag-only, native archives had no pin at all, and +`manifest.json` was not integrity-checked despite supplying `entrypoint`/`cmd`. The runtime +checksum came from mutable sources — the release `SHA256SUMS` or a GHCR `:-native-` +tag — so one CLI version could run different bytes on different machines. -Some consumers cannot reach GitHub release assets or a single registry blob CDN. This ADR covers publishing those copies and the local stack's fail-through. +Some consumers also cannot reach GitHub release assets or a single registry blob CDN. This ADR +covers the immutable publish scheme, the CLI's content pins, and the local stack's fail-through +across mirrors. ## Decision -Publish each slim **image** to GHCR and AWS ECR Public (`public.ecr.aws/supabase/cli/`), digest-preserving, via `mirror-slim-image.yml` and `.github/scripts/mirror-slim-image.ts`. Publish each slim **native** archive as an OCI artifact on the same two repositories under `:version-native-` (not `:version-linux-*`, which are image platform tags). GitHub Releases remain the human HTTPS copy and `--clobber` on force. - -`force=true` always requests the ECR copy, including when the image digest is unchanged (natives may have moved). The **image** destination digest gates `publish-release` once the dispatch token exists. Native ECR copy is best-effort (`continue-on-error`) and must not fail that release. Native tags ride in a separate `natives[]` payload field. This mirror does not open a pull request to pin `packages/stack/src/Artifacts.ts`. A Dependabot Dockerfile bump commits that catalog pin onto the same pull request, so the docker.io tag, the slim image, and the native archive stay on one version. A PostgreSQL release line that is not the Dockerfile `pg` tag, and a same-version digest republish, stay put until that Dockerfile line changes. Do not prune untagged GHCR or ECR manifests: already-shipped CLIs still pin old image digests until that pull request merges. Natives follow the moved tag immediately. - -ECR Public tags are always mutable; `ecr-public create-repository` has no immutability flag. Reuse the existing `PROD_AWS_ROLE` in `supabase/cli`. - -A public S3 bucket on `*.amazonaws.com` holds native archives for hosts that cannot reach GitHub Releases ([infra/cli-artifacts](../../infra/cli-artifacts/README.md)). It is not a checksum authority. The stack tries the GitHub Release first and falls back to that bucket. The expected checksum comes from the release `SHA256SUMS` or, when that is blocked, from the archive layer of the `:version-native-` artifact on GHCR. An archive from either host is accepted only when it matches. - -The container runtime pulls a catalog image from GHCR first and falls back to the same reference on ECR Public. A failing image fallback still reports the primary's original error. When every native checksum source or archive mirror fails, the error names each source with its failure. +### Immutable revisions + +Every slim-services publish carries a release version `R = -r`, where `U` is the upstream +tag exactly as the service's `tag_pattern` matches it and `N` is a revision starting at `0`. +Revision `N` of `U` is taken exactly when the GitHub release `-U-rN` exists — the GitHub +release is the commit point. Allocation is `max(taken) + 1`, computed from the full paginated +release list before any push. Once a release exists, `gh release create` is create-only: nothing +ever writes to that `-rN` again, so **there is no same-version overwrite**. A hotfix publishes a +new `-rN`; it never touches a previously published revision's bytes. `R` is never fed to a +semver library — ordering is the service's upstream version key, then revision, numerically. + +**Legacy tags** (without `-rN`) are frozen: nothing parses, audits, rewrites, or unpublishes +them, so CLIs pinned to a legacy tag keep working. + +### The catalog is the single version table + +The CLI's stack catalog (`packages/stack/src/Artifacts.ts`) is the single version table for +every consumer of a slim-capable service: the new stack, legacy `supabase start`, and legacy +slim mode. Each slim-capable service pins exactly one committed slim-services release per +release line it carries (`ArtifactPin`, keyed by upstream version; only postgres carries more +than one line, its default plus an additional Postgres 15 pin). Everything else is derived from +that pin — never the other way around. + +`apps/cli/src/shared/services/Dockerfile`, and its byte-identical Go copy +(`apps/cli-go/pkg/config/templates/Dockerfile`), keep every existing consumer (the legacy TS +stack, the Go CLI embed, `mirror-template-images.yml`, `detect-unmirrored-images.ts`) unchanged, +but for a slim-capable alias its `FROM` line is now a **generated view** of the catalog's +`ArtifactPin.upstreamImage`: `apps/cli/scripts/render-service-dockerfile.ts` rewrites those lines +in place, and a CI check (`render-service-dockerfile.unit.test.ts`) fails on drift between the +Dockerfile and the catalog. Kong and Postgres 14 have no slim build and are bumped by hand (both +share `library/kong`'s or `supabase/postgres`'s Dependabot `ignore` entry — see "Tradeoffs" below). +The one-shot job images (migra, pg_prove, pgadmin-schema-diff) also have no slim build, but are +upstream-only: they are the images Dependabot still bumps. + +### Content pins, not runtime checksums + +Every `ArtifactPin` pins its release by content: + +- the slim image, as `ghcr.io/supabase/cli/:@sha256:`; +- each target's archive sha256 and manifest sha256 (`ArtifactPin.natives`); +- `upstreamImage`, the exact upstream image the release was built or mirrored from, normalized to + the Dockerfile's `FROM` form — this is what legacy non-slim mode resolves against, and what the + generator reads to render the Dockerfile. + +`SlimServicesSource` hash-checks the manifest against its pin before parsing it (its `version` +field must equal `R`) and hash-checks the archive against its pin while it streams. There is no +runtime checksum authority: **GitHub Releases, the S3 mirror, GHCR, and ECR Public are byte +mirrors only** for both images and native archives. None of them is consulted for the expected +hash — that comes only from the pin already committed to the catalog. + +### Sync and the S3-staleness check + +`.github/scripts/sync-artifacts-catalog.ts` writes those pins, in manual mode: given a service +and either `--upstream` (the highest committed revision of that upstream version) or `--release` +(exactly that committed `-r`, never "highest at apply time"), it reads the target +release's `SHA256SUMS` for the archive and manifest sha256 per target, and resolves the image +digest with `regctl manifest head`. Before writing the pin, it downloads each target's S3 archive +and manifest and hashes them against those same release sums. This exists because +`publish-release` does not wait for the ECR/S3 mirror to finish, so a freshly committed +revision's S3 copy can briefly lag. A missing object waits, bounded (`waitForExpectedRelease`'s +retry helper, generalized); an object that exists with the wrong bytes fails the sync +immediately — that's corruption, not lag, and means "run the mirror backfill", not "the CLI is +broken". Hosts that reach GitHub are unaffected — GitHub is the primary mirror — but a host that +can only reach S3 would otherwise fail verification with no fallback. + +### Hotfix and upgrade pickup + +The last step of slim-services' `publish-release` sends a `repository_dispatch` +(`slim-release-published`) to the CLI repo with `{service, upstream_version, revision, +release_version}`. `slim-release-published.yml` treats the dispatch as a trigger, not as the +payload to apply: it reconciles the named service against every committed (published, non-draft) +`-...-r` release it can currently see (`planSlimUpdates`), and opens or updates, per +release line the service carries: + +- a **hotfix**, when the pinned upstream version has a higher committed revision — branch + `slim-hotfix/[-]`, title `chore(stack): pin `; +- an **upgrade**, when the newest committed upstream on that line is newer than the pinned one — + branch `slim-bump/[-]`, title `chore(stack): bump to `. + +Both can be planned in the same run. Plan and apply run from the same checkout of the default +branch in one job, so a re-run always recomputes from the latest develop: a stale plan can never +be applied, and a superseded PR's branch is rewritten (force-pushed) in place rather than raced +by a new one — the workflow deliberately never auto-closes a superseded PR. A backlog republish +of an older upstream version naturally plans nothing. The fallback, if the push or PR step fails, +is a documented manual `bun .github/scripts/sync-artifacts-catalog.ts --service --release +-r` invocation, followed by `apps/cli/scripts/render-service-dockerfile.ts` — the release +itself is already committed by then, so a failure here means "open the pull request by hand", not +"republish". + +### Registry and bucket mirrors + +Publish each slim **image** to GHCR and AWS ECR Public (`public.ecr.aws/supabase/cli/`), +digest-preserving, via `mirror-slim-image.yml` and `.github/scripts/mirror-slim-image.ts`. +Publish each slim **native** archive as an OCI artifact on the same two repositories under +`:R-native-`. A public S3 bucket on `*.amazonaws.com` +([infra/cli-artifacts](../../infra/cli-artifacts/README.md)) holds native archives for hosts that +cannot reach GitHub Releases. The container runtime pulls a catalog image from GHCR first and +falls back to the same reference on ECR Public. A failing image fallback still reports the +primary's original error. When every native archive mirror fails to serve the pinned bytes, the +error names each mirror with its failure. + +ECR Public tags are always mutable; `ecr-public create-repository` has no immutability flag. +Mirror backfills of a committed revision copy the immutable GHCR bytes by digest, so re-running +one is idempotent. Reuse the existing `PROD_AWS_ROLE` in `supabase/cli`. ## Follow-up -- Native ECR copy is best-effort; a daily mirror audit should report native drift. That audit is not defined in this repository yet. +- Native ECR copy is best-effort; a daily mirror audit should report native drift and treat the + committed GHCR digest as the source of truth for backfills. That audit is not defined in this + repository yet. +- Unpublishing legacy (non-revision) artifacts is planned as a separate task; they stay published + and frozen until then. ## Rationale -ECR Public mirroring already exists for other CLI images and needs no new vendor. Native OCI on those same repos reuses `regctl` and that role. Extracting the workflow into bun scripts lets the copy, digest check, and native payload validation run in CI and locally. +ECR Public mirroring already exists for other CLI images and needs no new vendor. Native OCI on +those same repos reuses `regctl` and that role. Making the GitHub release the single commit +point, rather than a destination-specific immutability guard on each mirror, keeps the +invariant in one place: once `-U-rN` exists, every downstream copy of it is either correct +or considered stale and backfilled — never rewritten in place. ## Consequences -- Same-version overwrite works on GHCR, GitHub Releases, and ECR Public. -- Images come from the GHCR catalog pin with ECR Public as fallback, and natives from GitHub Releases with the S3 bucket as fallback. -- A GitHub native can be live while the ECR native is stale. -- Old CLI releases keep pulling the previous image digest. +- Hotfixing packaging never touches previously published bytes; it always publishes a new + revision. +- The CLI verifies every artifact it downloads against a pin already committed to its own + catalog — no destination is trusted to supply the expected hash at runtime. +- A same-version republish is no longer possible; every fix is a new, allocatable revision. +- A revision's ECR/S3 copy can briefly lag GitHub after publish; the sync's S3-staleness check + and the mirror backfill are what keep them converging. +- Old CLI releases keep pulling their originally pinned image digest and native bytes forever. + +### Tradeoffs of moving updates onto the catalog + +- Upstream and security fixes for slim-capable images now reach the CLI only once + slim-services publishes a release. Direct Dependabot discovery is gone for them; Dependabot's + `docker` ecosystem `ignore` list (`.github/dependabot.yml`) now excludes every slim-capable + image plus `library/kong` and `supabase/postgres`, so it bumps only the three images that + remain genuinely upstream-only: migra, pg_prove, pgadmin-schema-diff. +- Dependabot's 7-day cooldown no longer governs the excluded images — hotfix and upgrade PRs for + slim-capable ones land as soon as `slim-release-published` fires, on whatever cadence + slim-services publishes; kong and `pg14` are simply bumped by hand. +- `pg14` has no slim build and was already upstream-only; it is bumped by hand alongside kong, + not by Dependabot — Dependabot's `docker` ecosystem ignores `supabase/postgres` entirely now, + since it cannot tell `pg14`'s `FROM` line apart from `pg`'s and `pg15`'s by repository name + alone. +- The Deno 1 edge-runtime override (`apps/cli/src/shared/functions/functions.shared.ts`, and the + Go `deno1` constant) stays a separately pinned, upstream-only exception — it never goes through + the catalog. ## Alternatives considered -1. **Google Artifact Registry / GCS** — blob host `storage.googleapis.com` is on at least one major sandbox Trusted list. Rejected for this cut: new company-wide vendor. -2. **Public S3 bucket** — HTTPS GET on `*.amazonaws.com` can work without CloudFront. Adopted for native archives. The bucket is not a checksum authority, and image pulls stay on GHCR and ECR Public. +1. **Google Artifact Registry / GCS** — blob host `storage.googleapis.com` is on at least one + major sandbox Trusted list. Rejected for this cut: new company-wide vendor. +2. **Public S3 bucket** — HTTPS GET on `*.amazonaws.com` can work without CloudFront. Adopted for + native archives, as a byte mirror only; the CLI's checksum authority is its own catalog pin, + verified against the release's `SHA256SUMS` once at sync time, not read at runtime. 3. **npm packages** — allowlisted widely, but a published version cannot be replaced. -4. **Docker Hub `supabase/cli-*`** — deferred; blob CDN is also off some default allowlists, and names collide with upstream `supabase/`. -5. **Unpin slim images** — would make old CLIs see a moved tag. Rejected: conflicts with ADR 0017. +4. **Docker Hub `supabase/cli-*`** — deferred; blob CDN is also off some default allowlists, and + names collide with upstream `supabase/`. +5. **Same-version overwrite (`force=true` clobber)** — the original model. Rejected: it broke + already-released CLIs pinned to that version and made a republish silently move bytes out + from under them. Replaced by immutable `-rN` revisions. +6. **A destination-specific immutability guard (for example, an S3 conditional write)** — + rejected in favor of making the GitHub release the single commit point; every other + destination is just a copy that either matches it or is stale. ## Related diff --git a/infra/cli-artifacts/README.md b/infra/cli-artifacts/README.md index 9413175da1..07f0b01708 100644 --- a/infra/cli-artifacts/README.md +++ b/infra/cli-artifacts/README.md @@ -18,13 +18,22 @@ agent sandboxes that allow `*.amazonaws.com`. Objects are addressed as ``` -https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com///--.tar.zst -https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com///--.manifest.json -https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com///--.SHA256SUMS +https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com///--.tar.zst +https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com///--.manifest.json +https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com///--.SHA256SUMS ``` -which keeps the GitHub release asset names, so the `SHA256SUMS` lines match the archive name -without rewriting. +where `` is the immutable `-r` slim-services revision, which keeps +the GitHub release asset names so the `SHA256SUMS` lines match the archive name without +rewriting. + +This bucket, like GHCR and ECR, is a byte mirror only — it is never a checksum authority. The +CLI does not fetch `SHA256SUMS` (from here or from GitHub) at runtime: it pins each target's +archive and manifest sha256 directly in `packages/stack/src/Artifacts.ts`, written by +`.github/scripts/sync-artifacts-catalog.ts` after that script has already verified this bucket's +copy against the release's committed `SHA256SUMS` (see +[ADR 0026](../../docs/adr/0026-slim-artifact-mirrors.md)). A stale object here fails that sync, +not a CLI download. ## How it deploys diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 0cd6c8815a..4ffbe1290d 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -30,10 +30,42 @@ export class ArtifactError extends Data.TaggedError("ArtifactError")<{ readonly cause?: unknown; }> {} +/** Lowercase hexadecimal SHA-256 digest. */ +type Sha256 = string; + +/** Content digests of one native target's published archive and manifest. */ +export interface NativePin { + readonly archive: Sha256; + readonly manifest: Sha256; +} + +/** One published slim-services revision of an upstream version, pinned by content. */ +export interface ArtifactPin { + readonly upstreamVersion: string; + readonly revision: number; + /** `ghcr.io/supabase/cli/:@sha256:`. */ + readonly image: string; + /** + * The exact upstream image this release was built or mirrored from, as slim-services recorded + * it: the release manifest's `upstream_image` for a derived service, or the release's + * `oci-provenance.json` `source` for a mirrored one (vector, mailpit, imgproxy). Normalized to + * the Dockerfile's `FROM` form — no leading `docker.io/`, no digest — so legacy non-slim mode + * can use it as the upstream tag directly. + */ + readonly upstreamImage: string; + readonly natives: Readonly>; +} + +/** The published release version, `-r`. */ +const releaseVersion = (pin: ArtifactPin): string => `${pin.upstreamVersion}-r${pin.revision}`; + interface ArtifactResolution { readonly service: ServiceKind; + /** Upstream version. */ readonly version: string; + readonly releaseVersion: string; readonly image: string; + readonly natives: ArtifactPin["natives"]; readonly executablePath: string; readonly requiredRuntimePaths: ReadonlyArray; } @@ -48,94 +80,380 @@ export interface PreparedNativeArtifact { interface ArtifactDefinition { readonly sourceService: string; readonly defaultVersion: string; - readonly images: Readonly>; + /** Pins keyed by upstream version. */ + readonly pins: Readonly>; readonly requiredRuntimePaths: ReadonlyArray; readonly executablePath: string; } const definition = ( sourceService: string, - defaultVersion: string, - image: string, + pin: ArtifactPin, executablePath: string, requiredRuntimePaths: ReadonlyArray = [executablePath], - additionalImages: Readonly> = {}, + additionalPins: Readonly> = {}, ): ArtifactDefinition => ({ sourceService, - defaultVersion, - images: { [defaultVersion]: image, ...additionalImages }, + defaultVersion: pin.upstreamVersion, + pins: { [pin.upstreamVersion]: pin, ...additionalPins }, requiredRuntimePaths, executablePath, }); +const SLIM_IMAGE_GHCR_REGISTRY = "ghcr.io/supabase/cli/"; + const definitions: Readonly> = { database: definition( "postgres", - "17.6.1.173", - "ghcr.io/supabase/cli/postgres:17.6.1.173@sha256:9d6e542382946cad5eb1f11f1c8108a51297902ee42fe5098358816d3784ba5a", + { + upstreamVersion: "17.11.0.002", + revision: 0, + image: + "ghcr.io/supabase/cli/postgres:17.11.0.002-r0@sha256:5a551a204a41267c4f78a2e5b34657f5c6fdd39eddccd50262466481b95739cc", + upstreamImage: "supabase/postgres:17.11.0.002", + natives: { + "darwin-arm64": { + archive: "10410e402c77210a0543539d9bafcabd0c04923d8e796928cbd2a053cf7b4f4f", + manifest: "6c1f236324f02baf472152aafed68884cb115b3979fbf1c49209717485d36ac2", + }, + "linux-amd64": { + archive: "4a4410791bdeeda2e08fda400039b4319bb26d74e08e68951f38abc9dacd7c26", + manifest: "cd76249031db380831d773acea6fea8a54104d5ca97bb18db34d05e7bd124e5f", + }, + "linux-arm64": { + archive: "1d54954b2441990643f25a825e4e46ec50ecf538a63baa8e413e353e7faeb939", + manifest: "8dda36ffdb7b5b501873ff41c265bb061e8eb9aeed0e4372c5ad9af59110776b", + }, + }, + }, "bin/supabase-postgres-start", ["bin/supabase-postgres-start", "bin/pg_dump", "bin/pg_dumpall", "bin/pg_prove", "bin/psql"], { - "15.14.1.173": - "ghcr.io/supabase/cli/postgres:15.14.1.173@sha256:3abb20e89700d6211e741148b85c3a052bb290cae3b4f751311442f4a5fe2324", + "15.19.0.002": { + upstreamVersion: "15.19.0.002", + revision: 0, + image: + "ghcr.io/supabase/cli/postgres:15.19.0.002-r0@sha256:a3f343f19323497a5766fa4e86a51dba495a8eca354cc128d879b4632c6bd017", + upstreamImage: "supabase/postgres:15.19.0.002", + natives: { + "darwin-arm64": { + archive: "59b05ca76db9d807803840264d885a5ff435e552f294d7f22ca831fac1ffb4a3", + manifest: "b7956520ee15a8265635cabb318490f89c4737706f9349246da9921a17fbac07", + }, + "linux-amd64": { + archive: "bdf565e0b866ca7b2494b56b502963ef57d54ab1c99eac6855f760caaffda712", + manifest: "e1ecda082f1c414fea6356d0399eaf76d22a25b96808961a945aacb242f904c9", + }, + "linux-arm64": { + archive: "c7420f0eb0938397d90906265630359313936cc576035cb31c2bdef6451565c6", + manifest: "207feac260bcbc5607decca9e14e0422accc4a0ae1f7d24b8fe0992d284cc43c", + }, + }, + }, + }, + ), + rest: definition( + "postgrest", + { + upstreamVersion: "v16.4", + revision: 0, + image: + "ghcr.io/supabase/cli/postgrest:v16.4-r0@sha256:63a8d4acfdeb107b6568f4582759c78072100ef07951a7fbe58c9a51241138a7", + upstreamImage: "postgrest/postgrest:v16.4", + natives: { + "darwin-arm64": { + archive: "a1f5449d739404cbd042ec9dbabadea6dcf810db636e44c7176b859d5e78ab07", + manifest: "943cbeb7a3cf2f9dd0406152a178d814e81221d0061626b04ab2b042209d5e03", + }, + "linux-amd64": { + archive: "d9170378062dba1188c25fb05cd08a3397bb62afd63151812f831eed4fab1051", + manifest: "a3f3f29c4c4c5f7b004b507152f29316bc79df052652fb00ad5ad9bdd6f7be29", + }, + "linux-arm64": { + archive: "c1a99eddaac0c005b2520960d258c7fc79945ba0703eff3a2515aa119fac4477", + manifest: "7ced80e3f814d9748763983de74706c6e04a897f8c024366af156c8093cf150f", + }, + }, + }, + "bin/postgrest", + ), + auth: definition( + "auth", + { + upstreamVersion: "v2.197.0", + revision: 0, + image: + "ghcr.io/supabase/cli/auth:v2.197.0-r0@sha256:7eb303831d170865840dbb3f9018b48561c988704ad1d9a2bafabb4e93e92da5", + upstreamImage: "supabase/gotrue:v2.197.0", + natives: { + "darwin-arm64": { + archive: "eb365c4aea1e1cd04998ed16cd49b1b90e6a097ce633e5ee55f8bbbebdd349c7", + manifest: "178f748da1c886b07945a67a839d3a17fa546e01215a90329ded1a634831deb6", + }, + "linux-amd64": { + archive: "bd8f59ed1a817014afe71288c2275ad0396630f016f8cb5ab5129388860ec49b", + manifest: "ccad45ee54edf7506b055b85dcebee6db8c5b9ec57ac3c6d30baf38bb81b1deb", + }, + "linux-arm64": { + archive: "9598a8dd5a08707f65795505211c479b686c54e3dadbe64b3e1989855215cf0c", + manifest: "4f5f3a6284f0b161cf49a201fe551470d5b6e0e2b37dedb4dacdd0e42abaf3b1", + }, + }, }, + "bin/auth", ), - rest: definition("postgrest", "v16.2", "ghcr.io/supabase/cli/postgrest:v16.2", "bin/postgrest"), - auth: definition("auth", "v2.196.0", "ghcr.io/supabase/cli/auth:v2.196.0", "bin/auth"), realtime: definition( "realtime", - "v2.134.5", - "ghcr.io/supabase/cli/realtime:v2.134.5@sha256:ee672ffd06ca0a1712a06aea1307c134c366ec082a51b051a8d59a8ad0c72755", + { + upstreamVersion: "v2.140.3", + revision: 0, + image: + "ghcr.io/supabase/cli/realtime:v2.140.3-r0@sha256:af7b883647770351eef150cc16e1f12d7fe4b4920324e2e40487961ad3b22789", + upstreamImage: "supabase/realtime:v2.140.3", + natives: { + "darwin-arm64": { + archive: "e015ced15ed2110c2a0123b6e75edb2aeb12ac6bdbacffdfb296262a4215f97e", + manifest: "a833a92e7b2e34b6bfcf41c4c934587f85e96eb11acb9cce14852b8ffec312d6", + }, + "linux-amd64": { + archive: "d528a255b2cb20df25ab9440e29f026e2fb3c993e4ed7b0707e037d0cb3880d9", + manifest: "fc5445d4b28d41691091c8cdccde7cff0dbd06af2bf56e70401ec6a049fb8757", + }, + "linux-arm64": { + archive: "bb3b810540dbba2f9a94eb974799384cbffe67756cd91ebb3b8254d02d890e2e", + manifest: "1a33f5620a39a64b49645c25878a892122d695f8f1a26d3548fb5332d6e0d88b", + }, + }, + }, "bin/server", ["bin/server", "bin/prepare"], ), storage: definition( "storage", - "v1.79.28-r1", - "ghcr.io/supabase/cli/storage:v1.79.28-r1@sha256:95e0007f273e7c990ab81c44e021e4278b8953dd95ae2fbdc19fca047d4bd470", + { + upstreamVersion: "v1.79.28", + revision: 1, + image: + "ghcr.io/supabase/cli/storage:v1.79.28-r1@sha256:95e0007f273e7c990ab81c44e021e4278b8953dd95ae2fbdc19fca047d4bd470", + upstreamImage: "supabase/storage-api:v1.79.28", + natives: { + "darwin-arm64": { + archive: "bea019baff21e10b78291a9c23687ec15ebb45994d0d03c656ffda13e13108d4", + manifest: "8612ea30b918ac6590f30660b518ea41b80d9d591734fa3313a64c6607621cb2", + }, + "linux-amd64": { + archive: "c8ce124acfe46a2151052f160a81653f751277516f5c378d93ba7f18e7b3efe4", + manifest: "edce6bb24a84e5245956c1de3fa93d96d3813efbe2e5dac105b385ebae7e3f0d", + }, + "linux-arm64": { + archive: "8a51c2b8bc4d1076665321ae0da7eca81dbdff84dbc687a209a958955cf9cb69", + manifest: "befaaec0e8660d99a4040f81ece4277e4db27812b8e2630a4d0b15d16b3ed831", + }, + }, + }, "bin/storage", ["bin/storage", "bin/prepare"], ), imgproxy: definition( "imgproxy", - "v3.8.0", - "ghcr.io/supabase/cli/imgproxy:v3.8.0", + { + upstreamVersion: "v3.26.0", + revision: 0, + image: + "ghcr.io/supabase/cli/imgproxy:v3.26.0-r0@sha256:5871582cf6c5140d3b50b21e0d1aa236fdf46ee6133e3bfec1baf63545e01cf0", + upstreamImage: "ghcr.io/imgproxy/imgproxy:v3.26.0", + natives: { + "darwin-arm64": { + archive: "1f82084b056bf4806bf492bca0b6f963068d07a6776a2a122b61da940732683b", + manifest: "53b5c4f4f069db88c3919571dbee12af2373e5c90db291cf5d7456aa74e17ecb", + }, + "linux-amd64": { + archive: "7865edf054b8217c2daf9d42bce6bebe34f83df5c4183fbbe6495a3603dcd4a8", + manifest: "5ae7f22a6fbdc55f41ffc3334747a3475295676699ac1d2c372bc5fe2ad239bb", + }, + "linux-arm64": { + archive: "4528fd7877c9df3052cd6ec06c8cafd7e92977dd26490fb532964c19faec3b0a", + manifest: "5420e6d9b3661060b2dedfe157c18b56f22011c43cc3127031b4bbc65e936b59", + }, + }, + }, "bin/imgproxy", ), functions: definition( "edge-runtime", - "v1.77.1", - "ghcr.io/supabase/cli/edge-runtime:v1.77.1@sha256:db55555ba640180671be297179797ea39c8c6cf09a22a0b883923cb86938f5e1", + { + upstreamVersion: "v1.77.1", + revision: 0, + image: + "ghcr.io/supabase/cli/edge-runtime:v1.77.1-r0@sha256:98582ce39914bba6ac856a5bf6ed3c1584ce4616687ca359c8acd5f59dd9ff59", + upstreamImage: "supabase/edge-runtime:v1.77.1", + natives: { + "darwin-arm64": { + archive: "e52fdbedccf258fd9a427b19bd34569ca210c1ca257a85fc23f66b4e18c1d6a1", + manifest: "ef2590d8119fe1c2c877746ef72f58ed4d935ce23647b893fd5d36892b0496e0", + }, + "linux-amd64": { + archive: "e524630f0743319d77535f93748a894520fd52cf9c8fa8c2adb22e08ddb39419", + manifest: "3ff18dc0667373dfc108da7d206dbfbf844f2596a804319343d5793298416e9c", + }, + "linux-arm64": { + archive: "c0fe416e506087afd75771beeeb6cfd070d4d6056293a940e3bf453637ba7a5f", + manifest: "703581ebe6213c484955313fbec3ea6ad5f20a86f0c48eb0e29d1bbe9d1c286b", + }, + }, + }, "bin/edge-runtime", ), studio: definition( "studio", - "2026.09.04-sha-5a67366", - "ghcr.io/supabase/cli/studio:2026.09.04-sha-5a67366@sha256:9823a31668028f1846e87331bc21598d9cd74bcaa1466c72dab58c33c9c82720", + { + upstreamVersion: "2026.09.28-sha-5e59b60", + revision: 0, + image: + "ghcr.io/supabase/cli/studio:2026.09.28-sha-5e59b60-r0@sha256:4cf4f70978bb0866d1644b43cb0d23acc4b4ef7a898592043ad0d13cad8059be", + upstreamImage: "supabase/studio:2026.09.28-sha-5e59b60", + natives: { + "darwin-arm64": { + archive: "37f420f6af3d5ee7dab884e8c39fe2c3bcddcee0d90e53d1e075a05ae3cb4b73", + manifest: "b5d6209c4c28e594cc8b0ab961105465418f0021ba50a3fd92067ff5720ae820", + }, + "linux-amd64": { + archive: "19a9903732b71fba04ce342e4d13b83bf7579bce806981cf8f2b491ad792fc6e", + manifest: "5a4f1e316ad84ff058263601b9b144177ab0cd18443e826354551aec88013199", + }, + "linux-arm64": { + archive: "ce69544c9ec5dbae50e5da731770bc199e12f0f1d63b22ec7a43b6a0ba00cb4c", + manifest: "c27374ccc51cc6419f9bbd14f095f1a16db4bda369928effcc15a6366dcb3d1f", + }, + }, + }, "bin/studio", ), pgmeta: definition( "pgmeta", - "v0.99.0", - "ghcr.io/supabase/cli/pgmeta:v0.99.0@sha256:90de2dcf03ac548ae2d1d3e71b3cd10bde4c627572720a42e4c3946b7090292e", + { + upstreamVersion: "v0.99.0", + revision: 0, + image: + "ghcr.io/supabase/cli/pgmeta:v0.99.0-r0@sha256:c19f6bba3ab66fcf30c8737d2361ec9f8e12a4fa8a071481f53366dc13e83340", + upstreamImage: "supabase/postgres-meta:v0.99.0", + natives: { + "darwin-arm64": { + archive: "337f8cc6a23d93f3f9cfeed12de2a86d686ce6f4346ff9b334b5dfdcba7e890f", + manifest: "6b35ee42d334138562444842ed0662b97b3cbde504caa11e97407b137bc8b2ca", + }, + "linux-amd64": { + archive: "43156ba28901710bf02a333dc04c4b30754eb6a2334ff1d890a4a3ea55c02f22", + manifest: "dbd8a7eac705b6c6df16826f22f3317906842be8498ce44fa1229ae6f16273ad", + }, + "linux-arm64": { + archive: "a8565d6e550efa8a8481c7d542b612e7a6d50668321fcd2e668b2ee9bcb28ed7", + manifest: "1b67627c5ccde0f94a4997ab223a0ac072c70e9bf1bb86d3e4d0df42e90b4059", + }, + }, + }, "bin/pgmeta", ), - mail: definition("mailpit", "v1.30.2", "ghcr.io/supabase/cli/mailpit:v1.30.2", "bin/mailpit"), + mail: definition( + "mailpit", + { + upstreamVersion: "v1.31.3", + revision: 0, + image: + "ghcr.io/supabase/cli/mailpit:v1.31.3-r0@sha256:ed9b00c609e77e99c79b93f1178255ebc271868920f2c69a8d166bd5634ed10d", + upstreamImage: "axllent/mailpit:v1.31.3", + natives: { + "darwin-arm64": { + archive: "d460a6a55693a2a321ad83ebb83417b8078324751a744f982bb3512da0632712", + manifest: "43177dc88b6f625bec6d98096e39c10f4cd10956081df01b5b7cebeeae47766a", + }, + "linux-amd64": { + archive: "57e39cb39b2288313e26abe9ae9300f6af9332abd1f76f4d67be533741b2c942", + manifest: "36595519f40e406b17a0ce6e21f153400f217ae24b0365f0a5c495ff315b9f82", + }, + "linux-arm64": { + archive: "805b09d9008e9be2c6c9230e668bd2c6ab446eeae26389e91732f9e20b2c3f5f", + manifest: "973b76700a5a865243bd5a5c36664121ed99c538d58633921cc8fd7c675e655c", + }, + }, + }, + "bin/mailpit", + ), analytics: definition( "analytics", - "v1.50.9", - "ghcr.io/supabase/cli/analytics:v1.50.9@sha256:7db85cc6cb0cdeb4b71f2fadb49c0f9197bea0492daf7896f6ae69edad76d28e", + { + upstreamVersion: "v1.50.15", + revision: 0, + image: + "ghcr.io/supabase/cli/analytics:v1.50.15-r0@sha256:cea1595bafa6ab32df4854d407261ddef6e35394e300ae29770ba48c49c8dc7e", + upstreamImage: "supabase/logflare:1.50.15", + natives: { + "darwin-arm64": { + archive: "336c78c501aff270b0fcd2a42229b76d9f2feafa646f49a59c3de29358e3de17", + manifest: "d0e1a0c3fa0a4ebfcebbe490f290699424e3b8c92a542f50111ff88d6e47b8b5", + }, + "linux-amd64": { + archive: "a699ce1522dac43989987a9d152648ed0506fb887ffedff49c148da02cbbc7e7", + manifest: "ba8c55eeb06be91ad582156854babdc6a06d7a26fa6956bc1271eee4cae9d536", + }, + "linux-arm64": { + archive: "bb977b9cd0623e1b1ece9875377fdf29577f96991d2e5c72a38a1b664c6a7538", + manifest: "1739bbd4afb838e62b6191c4de8aa39685df9f5d83e1b7cc303bd096237ba08c", + }, + }, + }, "bin/logflare", ["bin/logflare", "bin/prepare"], ), - vector: definition("vector", "0.53.0", "ghcr.io/supabase/cli/vector:0.53.0", "bin/vector", [ + vector: definition( + "vector", + { + upstreamVersion: "0.58.0", + revision: 0, + image: + "ghcr.io/supabase/cli/vector:0.58.0-r0@sha256:5dcf67db0ee378caa87f3395cb9484ebe3e97bb0334d119f2ac33116e00c5773", + upstreamImage: "timberio/vector:0.58.0-alpine", + natives: { + "darwin-arm64": { + archive: "567245cf9a7d54eee45ecf74e1c9a61ca6d02cdca103edc7b32b005e54f4e632", + manifest: "a973a763b00599858ceae8f304714fb99f785860112e9e0812ef0df8f81dd2ee", + }, + "linux-amd64": { + archive: "697f4fae35be3026474695bef16336f6fcfd429ce8cfba884c595896e96c30ec", + manifest: "8989b8b061f08bd7653e9ae71c6ee0e5cf01a0b10f2d358fe3dbc671ec5b63e1", + }, + "linux-arm64": { + archive: "c66b8ad0a0dd0fdcb3e4ee36bae8040b7b23b44ec2b4023565ca335884268c1d", + manifest: "992467ec68a99a6413468b9311294abb7a1f86b7857ad37f1f3bd9e6aecc9dbb", + }, + }, + }, "bin/vector", - "share/doc/vector/config/vector.yaml", - ]), + ["bin/vector", "share/doc/vector/config/vector.yaml"], + ), pooler: definition( "pooler", - "v2.9.12", - "ghcr.io/supabase/cli/pooler:v2.9.12@sha256:12bb9dcb7ddace79bee173ccb7327c6646af2236679f3bd932a86b3a06479aac", + { + upstreamVersion: "v2.9.13", + revision: 0, + image: + "ghcr.io/supabase/cli/pooler:v2.9.13-r0@sha256:3a32b56d03675ed24e84408afcbb12fa52b7ec6e7741f913770fea5b66c2ddd3", + upstreamImage: "supabase/supavisor:2.9.13", + natives: { + "darwin-arm64": { + archive: "c05285be2a945a29d5be491661926f8c88d976540d49ddbcab10aa7276b29934", + manifest: "0fafa8dcf601ff3cea7326e82f7a5191ffe03e3e4edd0d2b155e80e8c9f53d78", + }, + "linux-amd64": { + archive: "d94e36f44267b4159fa55e1aea39320a68789b6af07f9cc87c27327ed6a00602", + manifest: "4f181d1d25da91f72ca37c22cdc46278a424a4edef6bdd2ea2c30f463c629a5a", + }, + "linux-arm64": { + archive: "124f3e6c95e6ba985239e013cab76c5be28fbd92cb4986b687dd5e827f0525ca", + manifest: "c6cdc298bc3a00d082a1c9c34f4131f13437df51fb0bf9a0ce857b0a2fd66460", + }, + }, + }, "bin/server", ["bin/server", "bin/prepare", "bin/provision-tenant"], ), @@ -174,10 +492,6 @@ const SLIM_NATIVE_GITHUB_RELEASES = "https://github.com/supabase/slim-services/r */ const SLIM_NATIVE_SUPABASE_S3_MIRROR = "https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com"; -const SLIM_NATIVE_GHCR_REGISTRY = "ghcr.io"; -const SLIM_NATIVE_GHCR_REPOSITORY = "supabase/cli"; - -const SLIM_IMAGE_GHCR_REGISTRY = "ghcr.io/supabase/cli/"; const SLIM_IMAGE_SUPABASE_ECR_MIRROR = "public.ecr.aws/supabase/cli/"; /** Mirrors carrying a catalog slim image under the same tag and digest, in fallback order. */ @@ -192,27 +506,21 @@ const artifactFor = ( target: NativeTarget, ): SlimServicesArtifact => { const sourceService = definitions[service].sourceService; - const releaseTag = `${sourceService}-${resolved.version}`; + const releaseTag = `${sourceService}-${resolved.releaseVersion}`; const assetName = `${releaseTag}-${target}`; const githubRelease = `${SLIM_NATIVE_GITHUB_RELEASES}/${releaseTag}`; - const supabaseS3 = `${SLIM_NATIVE_SUPABASE_S3_MIRROR}/${sourceService}/${resolved.version}`; + const supabaseS3 = `${SLIM_NATIVE_SUPABASE_S3_MIRROR}/${sourceService}/${resolved.releaseVersion}`; + const pin = resolved.natives[target]; return { provider: "supabase/slim-services", service: sourceService, - version: resolved.version, + version: resolved.releaseVersion, releaseTag, target, archive: "tar.zst", assetName, - checksums: [ - { kind: "sha256sums", url: `${githubRelease}/SHA256SUMS` }, - { - kind: "oci", - registry: SLIM_NATIVE_GHCR_REGISTRY, - repository: `${SLIM_NATIVE_GHCR_REPOSITORY}/${sourceService}`, - tag: `${resolved.version}-native-${target}`, - }, - ], + sha256: pin.archive, + manifestSha256: pin.manifest, mirrors: [ { downloadUrl: `${githubRelease}/${assetName}.tar.zst`, @@ -236,8 +544,8 @@ export const resolveArtifact = Effect.fn("Artifacts.resolveArtifact")(function* return yield* new ArtifactError({ message: `Unknown service kind: ${request.service}` }); const selected = definitions[request.service]; const version = request.version ?? selected.defaultVersion; - const image = Object.entries(selected.images).find(([candidate]) => candidate === version)?.[1]; - if (image === undefined) + const pin = Object.entries(selected.pins).find(([candidate]) => candidate === version)?.[1]; + if (pin === undefined) return yield* new ArtifactError({ message: `Unsupported ${request.service} artifact version: ${version}`, service: request.service, @@ -246,7 +554,9 @@ export const resolveArtifact = Effect.fn("Artifacts.resolveArtifact")(function* return { service: request.service, version, - image, + releaseVersion: releaseVersion(pin), + image: pin.image, + natives: pin.natives, executablePath: selected.executablePath, requiredRuntimePaths: selected.requiredRuntimePaths, }; @@ -254,13 +564,33 @@ export const resolveArtifact = Effect.fn("Artifacts.resolveArtifact")(function* /** Resolves a PostgreSQL major alias against the pinned database artifacts. */ export const postgresVersion = (version: string): string => - Object.keys(definitions.database.images).find( - (candidate) => candidate.split(".")[0] === version, - ) ?? version; + Object.keys(definitions.database.pins).find((candidate) => candidate.split(".")[0] === version) ?? + version; /** Service kinds in artifact catalog order. */ export const artifactServiceKinds = (): ReadonlyArray => Record.keys(definitions); +/** + * Every catalog pin in catalog order, including additional upstream lines. `isDefault` marks the + * pin `resolveArtifact` picks when no version is requested (postgres's 17.x line today); every + * other pin (postgres's 15.x additional line) carries `isDefault: false`. + */ +export const catalogPins = (): ReadonlyArray<{ + readonly service: ServiceKind; + readonly sourceService: string; + readonly pin: ArtifactPin; + readonly isDefault: boolean; +}> => + artifactServiceKinds().flatMap((service) => { + const { sourceService, defaultVersion, pins } = definitions[service]; + return Object.values(pins).map((pin) => ({ + service, + sourceService, + pin, + isDefault: pin.upstreamVersion === defaultVersion, + })); + }); + const artifactKey = (artifact: SlimServicesArtifact): string => `slim-services/${artifact.service}/${artifact.version}/${artifact.target}`; diff --git a/packages/stack/src/Artifacts.unit.test.ts b/packages/stack/src/Artifacts.unit.test.ts index 0e0ecf510d..50c55fd78c 100644 --- a/packages/stack/src/Artifacts.unit.test.ts +++ b/packages/stack/src/Artifacts.unit.test.ts @@ -1,5 +1,14 @@ import { expect, it } from "@effect/vitest"; -import { slimImageMirrors } from "./Artifacts.ts"; +import { catalogPins, slimImageMirrors } from "./Artifacts.ts"; + +it("carries a normalized upstreamImage for every catalog pin", () => { + for (const { pin } of catalogPins()) { + expect(pin.upstreamImage).not.toBe(""); + expect(pin.upstreamImage).not.toContain("docker.io/"); + expect(pin.upstreamImage).not.toContain("@sha256:"); + expect(pin.upstreamImage.split(":").at(-1)).not.toBe(""); + } +}); it("rewrites a GHCR catalog image onto ECR Public and keeps the tag and digest", () => { expect( diff --git a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts index 28e1c4c48d..269d9307c2 100644 --- a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts +++ b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts @@ -2,6 +2,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Context, Deferred, Effect, Fiber, FileSystem, Layer, Path, Ref } from "effect"; import { TestClock } from "effect/testing"; +import { catalogPins, resolveArtifact } from "../Artifacts.ts"; import { initialization } from "../Commands.ts"; import { makeArtifactStore, @@ -21,11 +22,20 @@ const target = ? "linux-arm64" : undefined; +// The real catalog's default auth pin, not hardcoded — `resolveArtifact({ service: "auth", +// version })` below would otherwise fail once a catalog bump moves past a literal. +const authVersion = catalogPins().find((entry) => entry.sourceService === "auth" && entry.isDefault) + ?.pin.upstreamVersion; +if (authVersion === undefined) { + throw new Error("no default auth catalog pin found"); +} + const prepareAuthArtifact = Effect.fn(function* (cacheRoot: string, script: string) { if (target === undefined) return yield* Effect.fail("Unsupported test platform"); const fs = yield* FileSystem.FileSystem; + const { releaseVersion } = yield* resolveArtifact({ service: "auth" }); const request: ArtifactRequest = { - key: `slim-services/auth/v2.196.0/${target}`, + key: `slim-services/auth/${releaseVersion}/${target}`, requiredRuntimePaths: ["bin/auth"], executablePath: "bin/auth", }; @@ -94,7 +104,7 @@ it.live.skipIf(target === undefined || process.platform === "win32")( const error = yield* runner .run({ command: initialization.auth({ - version: "v2.196.0", + version: authVersion, databaseUrl: "postgresql://invalid:invalid@127.0.0.1:1/postgres", }), credentials, @@ -137,7 +147,7 @@ it.live.skipIf(target === undefined || process.platform === "win32")( const command = yield* runner .run({ command: initialization.auth({ - version: "v2.196.0", + version: authVersion, databaseUrl: "postgresql://invalid:invalid@127.0.0.1:1/postgres", }), credentials, @@ -195,7 +205,7 @@ it.effect.skipIf(target === undefined || process.platform === "win32")( const command = yield* runner .run({ command: initialization.auth({ - version: "v2.196.0", + version: authVersion, databaseUrl: "postgresql://invalid:invalid@127.0.0.1:1/postgres", }), credentials, diff --git a/packages/stack/src/internal/artifacts.ts b/packages/stack/src/internal/artifacts.ts index cce55dec8c..0ee05f1fac 100644 --- a/packages/stack/src/internal/artifacts.ts +++ b/packages/stack/src/internal/artifacts.ts @@ -2,8 +2,12 @@ export { ArtifactError, artifactServiceKinds, + catalogPins, defaultRuntime, postgresVersion, prepareNativeArtifact, resolveArtifact, + type ArtifactPin, + type NativePin, + type ServiceKind, } from "../Artifacts.ts"; diff --git a/packages/stack/src/preparation/Integrity.ts b/packages/stack/src/preparation/Integrity.ts index 381aef46ca..a376dc6618 100644 --- a/packages/stack/src/preparation/Integrity.ts +++ b/packages/stack/src/preparation/Integrity.ts @@ -35,7 +35,7 @@ export const verifySha256 = Effect.fn("Integrity.verifySha256")(function* ( ); const actual = digestHex(digest); if (actual !== canonical) - return yield* integrityError("Artifact SHA-256 does not match the catalog", { + return yield* integrityError(`expected ${canonical}, got ${actual}`, { expected: canonical, actual, }); diff --git a/packages/stack/src/preparation/SlimServicesSource.ts b/packages/stack/src/preparation/SlimServicesSource.ts index f4453347b6..97ed1487e5 100644 --- a/packages/stack/src/preparation/SlimServicesSource.ts +++ b/packages/stack/src/preparation/SlimServicesSource.ts @@ -7,6 +7,7 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { errorChainMessage } from "../internal/error-message.ts"; import type { ArtifactRequest, ArtifactSource } from "./ArtifactStore.ts"; import { PreparationError } from "./Errors.ts"; +import { verifySha256 } from "./Integrity.ts"; /** One host serving the archive and manifest of a slim-services release asset. */ interface SlimServicesMirror { @@ -14,33 +15,20 @@ interface SlimServicesMirror { readonly manifestUrl: string; } -/** - * Where the expected archive digest comes from: a release `SHA256SUMS` file, or the archive layer - * of the native OCI artifact in a registry that allows anonymous pulls. - */ -type SlimServicesChecksumSource = - | { readonly kind: "sha256sums"; readonly url: string } - | { - readonly kind: "oci"; - readonly registry: string; - readonly repository: string; - readonly tag: string; - }; - export interface SlimServicesArtifact { readonly provider: "supabase/slim-services"; readonly service: string; + /** Release version, `-r`. */ readonly version: string; readonly releaseTag: string; readonly target: "darwin-arm64" | "linux-amd64" | "linux-arm64"; readonly archive: "tar.zst"; readonly assetName: string; - /** Checksum authorities, tried in order; a mirror's own checksum file counts only when listed here. */ - readonly checksums: readonly [ - SlimServicesChecksumSource, - ...ReadonlyArray, - ]; - /** Hosts carrying the same release assets, tried in order until one serves them. */ + /** Pinned archive digest. Mirrors only serve bytes; they never supply a digest. */ + readonly sha256: string; + /** Pinned manifest digest, verified before the manifest is parsed. */ + readonly manifestSha256: string; + /** Hosts carrying the same release assets, tried in order until one serves the pinned bytes. */ readonly mirrors: readonly [SlimServicesMirror, ...ReadonlyArray]; readonly requiredRuntimePaths: ReadonlyArray; readonly executablePath: string; @@ -97,9 +85,9 @@ const systemTarBoundary: TarBoundary = { ); }), }; -const responseFor = (url: string, headers?: Readonly>) => +const responseFor = (url: string) => Effect.flatMap(HttpClient.HttpClient, (client) => - HttpClient.followRedirects(client).get(url, { headers }), + HttpClient.followRedirects(client).get(url), ).pipe( Effect.flatMap((response) => Effect.gen(function* () { @@ -145,9 +133,8 @@ const withTransferRetry = const fetchBytes = Effect.fn("SlimServicesSource.fetchBytes")(function* ( url: string, backoff: Schedule.Schedule, - headers?: Readonly>, ) { - return yield* responseFor(url, headers).pipe( + return yield* responseFor(url).pipe( Effect.flatMap((response) => response.arrayBuffer), Effect.map((bytes) => new Uint8Array(bytes)), withTransferRetry(url, backoff), @@ -274,93 +261,6 @@ const firstSuccess = ( ); }; -const checksumFor = (contents: string, archiveName: string): string | undefined => - contents - .split(/\r?\n/u) - .map((line) => line.trim().match(/^([a-f0-9]{64})\s+[* ]?(.+)$/iu)) - .find((match) => match?.[2] === archiveName || match?.[2]?.endsWith(`/${archiveName}`))?.[1]; - -const ARCHIVE_MEDIA_TYPE = "application/vnd.supabase.slim.archive.v1.tar+zstd"; - -const RegistryToken = Schema.Struct({ token: Schema.String }); - -const NativeOciManifest = Schema.Struct({ - layers: Schema.Array(Schema.Struct({ mediaType: Schema.String, digest: Schema.String })), -}); - -const decodeJson = (schema: S, bytes: Uint8Array, message: string) => - Schema.decodeEffect(Schema.fromJsonString(schema))(new TextDecoder().decode(bytes)).pipe( - Effect.mapError((cause) => new PreparationError({ message, cause })), - ); - -/** Reads the archive layer digest of a native OCI artifact through an anonymous pull token. */ -const ociArchiveDigest = Effect.fn("SlimServicesSource.ociArchiveDigest")(function* ( - source: Extract, - backoff: Schedule.Schedule, -) { - const tokenUrl = `https://${source.registry}/token?scope=repository:${source.repository}:pull&service=${source.registry}`; - const { token } = yield* decodeJson( - RegistryToken, - yield* fetchBytes(tokenUrl, backoff), - "Slim-services registry token is invalid", - ); - const manifest = yield* decodeJson( - NativeOciManifest, - yield* fetchBytes( - `https://${source.registry}/v2/${source.repository}/manifests/${source.tag}`, - backoff, - { - accept: "application/vnd.oci.image.manifest.v1+json", - authorization: `Bearer ${token}`, - }, - ), - "Slim-services native OCI manifest is invalid", - ); - const digest = manifest.layers - .find((layer) => layer.mediaType === ARCHIVE_MEDIA_TYPE) - ?.digest.match(/^sha256:([a-f0-9]{64})$/u)?.[1]; - if (digest === undefined) - return yield* new PreparationError({ - message: "Slim-services native OCI manifest has no archive layer", - }); - return digest; -}); - -const describeChecksumSource = (source: SlimServicesChecksumSource): string => - source.kind === "sha256sums" - ? source.url - : `${source.registry}/${source.repository}:${source.tag}`; - -export const slimServicesChecksum = Effect.fn("SlimServicesSource.checksum")(function* ( - artifact: SlimServicesArtifact, - backoff: Schedule.Schedule = transferBackoff, -) { - return yield* firstSuccess( - "Unable to resolve the slim-services checksum", - artifact, - artifact.checksums, - describeChecksumSource, - (source) => - source.kind === "oci" - ? ociArchiveDigest(source, backoff) - : fetchBytes(source.url, backoff).pipe( - Effect.map((bytes) => new TextDecoder().decode(bytes)), - Effect.flatMap((contents) => { - const checksum = checksumFor(contents, `${artifact.assetName}.tar.zst`); - return checksum === undefined || !/^[a-f0-9]{64}$/iu.test(checksum) - ? Effect.fail( - new PreparationError({ - message: "Slim-services checksum is missing", - service: artifact.service, - version: artifact.version, - }), - ) - : Effect.succeed(checksum.toLowerCase()); - }), - ), - ); -}); - const manifestSchema = Schema.Struct({ service: Schema.String, version: Schema.String, @@ -375,6 +275,18 @@ const verifiedManifest = Effect.fn("SlimServicesSource.verifiedManifest")(functi backoff: Schedule.Schedule, ) { const manifestBytes = yield* fetchBytes(mirror.manifestUrl, backoff); + yield* verifySha256(manifestBytes, artifact.manifestSha256).pipe( + Effect.mapError( + (cause) => + new PreparationError({ + message: "Slim-services manifest does not match its pin", + service: artifact.service, + version: artifact.version, + target: artifact.target, + cause, + }), + ), + ); const manifest = yield* Schema.decodeEffect(Schema.fromJsonString(manifestSchema))( new TextDecoder().decode(manifestBytes), ).pipe( @@ -497,10 +409,7 @@ export const makeSlimServicesSource = ( return artifact; }); return { - checksum: (request) => - resolveArtifact(request).pipe( - Effect.flatMap((artifact) => slimServicesChecksum(artifact, backoff)), - ), + checksum: (request) => resolveArtifact(request).pipe(Effect.map(({ sha256 }) => sha256)), materialize: Effect.fn("SlimServicesSource.materialize")( function* (request, destination, expectedSha256, onProgress) { const fs = yield* FileSystem.FileSystem; diff --git a/packages/stack/src/preparation/slim-services.integration.test.ts b/packages/stack/src/preparation/slim-services.integration.test.ts index 9423210184..c321aae3c1 100644 --- a/packages/stack/src/preparation/slim-services.integration.test.ts +++ b/packages/stack/src/preparation/slim-services.integration.test.ts @@ -17,12 +17,11 @@ import { import { createServer, type Server } from "node:http"; import { zstdCompress } from "node:zlib"; import { FetchHttpClient } from "effect/unstable/http"; -import { prepareNativeArtifact, resolveArtifact } from "../Artifacts.ts"; -import { makeArtifactStore, type ArtifactRequest } from "./ArtifactStore.ts"; +import { catalogPins, prepareNativeArtifact, resolveArtifact } from "../Artifacts.ts"; +import { makeArtifactStore, type ArtifactRequest, type ArtifactSource } from "./ArtifactStore.ts"; import { digestHex } from "./Integrity.ts"; import { makeSlimServicesSource, - slimServicesChecksum, type SlimServicesArtifact, type ZstdDecompressor, } from "./SlimServicesSource.ts"; @@ -34,15 +33,31 @@ const waitForAbort = (signal?: AbortSignal | null): Promise => const waitForRelease = (released: Deferred.Deferred): Promise => Effect.runPromise(Deferred.await(released)); +const manifestBytes = (version: string, service = "demo"): Uint8Array => + new TextEncoder().encode(JSON.stringify({ service, version, target: "linux-amd64" })); + +const demoManifest = manifestBytes("v1.0.0-r0"); + +/** Passes every manifest check except its pin. */ +const commandManifest = new TextEncoder().encode( + JSON.stringify({ + service: "demo", + version: "v1.0.0-r0", + target: "linux-amd64", + cmd: ["bin/evil"], + }), +); + const artifact: SlimServicesArtifact = { provider: "supabase/slim-services", service: "demo", - version: "v1.0.0", - releaseTag: "demo-v1.0.0", + version: "v1.0.0-r0", + releaseTag: "demo-v1.0.0-r0", target: "linux-amd64", archive: "tar.zst", - assetName: "demo-v1.0.0-linux-amd64", - checksums: [{ kind: "sha256sums", url: "https://example.test/SHA256SUMS" }], + assetName: "demo-v1.0.0-r0-linux-amd64", + sha256: "0".repeat(64), + manifestSha256: "0".repeat(64), mirrors: [ { downloadUrl: "https://example.test/demo.tar.zst", @@ -52,6 +67,34 @@ const artifact: SlimServicesArtifact = { requiredRuntimePaths: ["bin/demo"], executablePath: "bin/demo", }; + +const releaseMirror = { + downloadUrl: "https://release.test/demo-v1.0.0-r0-linux-amd64.tar.zst", + manifestUrl: "https://release.test/demo-v1.0.0-r0-linux-amd64.manifest.json", +}; +const bucketMirror = { + downloadUrl: "https://bucket.test/demo-v1.0.0-r0-linux-amd64.tar.zst", + manifestUrl: "https://bucket.test/demo-v1.0.0-r0-linux-amd64.manifest.json", +}; + +const sha256Of = Effect.fn(function* (bytes: Uint8Array) { + const crypto = yield* Crypto.Crypto; + return digestHex(yield* crypto.digest("SHA-256", bytes)); +}); + +/** Pins `base` to the digests of the given archive and manifest bytes. */ +const pinned = Effect.fn(function* ( + archive: Uint8Array, + manifest: Uint8Array = demoManifest, + base: SlimServicesArtifact = artifact, +) { + return { + ...base, + sha256: yield* sha256Of(archive), + manifestSha256: yield* sha256Of(manifest), + } satisfies SlimServicesArtifact; +}); + const request: ArtifactRequest = { key: "demo/v1", requiredRuntimePaths: ["bin/demo"], @@ -138,59 +181,62 @@ const withFetch = (fetcher: FetchLike, effect: Effect.Effect) Effect.provideService(FetchHttpClient.Fetch, fetcher), ); -const registryChecksum = { - kind: "oci", - registry: "registry.test", - repository: "supabase/cli/demo", - tag: "v1.0.0-native-linux-amd64", -} as const; +/** Serves each host's manifest and archive, answers 403 elsewhere, and records every URL. */ +const serving = + ( + hosts: Readonly< + Record + >, + requested: Array = [], + ): FetchLike => + (input) => { + const url = requestUrl(input); + requested.push(url); + const host = hosts[new URL(url).host]; + if (host === undefined) return Promise.resolve(new Response("", { status: 403 })); + return Promise.resolve( + new Response(url.endsWith(".manifest.json") ? host.manifest : host.archive), + ); + }; -/** Anonymous registry token and native manifest; the manifest requires the issued token. */ -const registryResponse = ( - input: Parameters[0], - init: Parameters[1], - archiveSha256: string, - registry = "registry.test", - repository = "supabase/cli/demo", - tag = "v1.0.0-native-linux-amd64", -): Response | undefined => { - const url = requestUrl(input); - if (url === `https://${registry}/token?scope=repository:${repository}:pull&service=${registry}`) - return new Response(JSON.stringify({ token: "anonymous" })); - if (url !== `https://${registry}/v2/${repository}/manifests/${tag}`) return undefined; - if (new Headers(init?.headers).get("authorization") !== "Bearer anonymous") - return new Response("", { status: 401 }); - return new Response( - JSON.stringify({ - layers: [ - { - mediaType: "application/vnd.supabase.slim.archive.v1.tar+zstd", - digest: `sha256:${archiveSha256}`, - }, - { - mediaType: "application/vnd.supabase.slim.checksum.v1", - digest: `sha256:${"f".repeat(64)}`, - }, - ], - }), - ); -}; +const fixtureSource = (content: string): ArtifactSource => ({ + checksum: () => Effect.succeed("0".repeat(64)), + materialize: (entry, destination) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + for (const file of entry.requiredRuntimePaths) { + yield* fs.makeDirectory(`${destination}/${file.slice(0, file.lastIndexOf("/"))}`, { + recursive: true, + }); + yield* fs.writeFileString(`${destination}/${file}`, content); + yield* fs.chmod(`${destination}/${file}`, 0o755); + } + }).pipe( + Effect.mapError( + (cause) => new PreparationError({ message: "Unable to write cache fixture", cause }), + ), + ), +}); describe("slim-services artifact source", () => { it.live("follows release redirects through the supplied Node HTTP client", () => Effect.scoped( Effect.gen(function* () { + const archive = yield* compress(tar("bin/demo", "demo")); const server = yield* Effect.acquireRelease( Effect.tryPromise({ try: () => // oxlint-disable-next-line effecttsgo/new-promise -- node server listen exposes a callback lifecycle. new Promise((resolve, reject) => { const value = createServer((request, response) => { - if (request.url === "/redirect") { + const url = request.url ?? ""; + if (url.startsWith("/redirect/")) { response.statusCode = 302; - response.setHeader("location", "/checksums"); + response.setHeader("location", url.slice("/redirect".length)); + response.end(); + return; } - response.end("a".repeat(64) + " demo-v1.0.0-linux-amd64.tar.zst\n"); + response.end(url.endsWith(".manifest.json") ? demoManifest : archive); }); value.once("error", reject); value.listen(0, "127.0.0.1", () => resolve(value)); @@ -207,50 +253,62 @@ describe("slim-services artifact source", () => { const address = server.address(); if (address === null || typeof address === "string") return yield* Effect.die("redirect server did not expose a port"); - const redirected: SlimServicesArtifact = { + const origin = `http://127.0.0.1:${address.port}/redirect`; + const redirected = yield* pinned(archive, demoManifest, { ...artifact, - checksums: [{ kind: "sha256sums", url: `http://127.0.0.1:${address.port}/redirect` }], - }; - const checksum = yield* slimServicesChecksum(redirected); - expect(checksum).toBe("a".repeat(64)); + mirrors: [ + { + downloadUrl: `${origin}/demo.tar.zst`, + manifestUrl: `${origin}/demo.manifest.json`, + }, + ], + }); + const fs = yield* FileSystem.FileSystem; + const destination = yield* fs.makeTempDirectoryScoped({ + prefix: "slim-services-redirect-", + }); + yield* makeSlimServicesSource(() => redirected).materialize( + request, + destination, + redirected.sha256, + ); + expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); }).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ), ); - it.live("verifies checksums and extracts a manifest-matched archive using injected fetch", () => + it.live("prepares a pinned archive and manifest without fetching any checksum", () => Effect.scoped( Effect.gen(function* () { const archive = yield* compress(tar("bin/demo", "demo")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const checksums = expected + " demo-v1.0.0-linux-amd64.tar.zst\n"; - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) return Promise.resolve(new Response(checksums)); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; - expect(yield* withFetch(fetcher, slimServicesChecksum(artifact))).toBe(expected); - const source = makeSlimServicesSource(() => artifact); + const demo = yield* pinned(archive); + const requested: string[] = []; + const fetcher = serving({ "example.test": { archive, manifest: demoManifest } }, requested); const fs = yield* FileSystem.FileSystem; - const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-source-" }); - yield* withFetch(fetcher, source.materialize(request, destination, expected)); - expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-store-" }); + const store = yield* makeArtifactStore({ + cacheRoot: root, + source: makeSlimServicesSource(() => demo), + }); + + const prepared = yield* withFetch(fetcher, store.prepare(request)); + expect(prepared.outcome).toBe("downloaded"); + expect(prepared.sha256).toBe(demo.sha256); + expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); + expect(requested).toEqual([demo.mirrors[0].manifestUrl, demo.mirrors[0].downloadUrl]); + + const cached = yield* withFetch(fetcher, store.prepare(request)); + expect(cached.outcome).toBe("cached"); + expect(requested).toHaveLength(2); }).pipe(Effect.provide(NodeServices.layer)), ), ); - it.live("retries a gateway error on the checksum and a truncated archive transfer", () => + it.live("retries a gateway error on the manifest and a truncated archive transfer", () => Effect.scoped( Effect.gen(function* () { const archive = yield* compress(tar("bin/demo", "demo")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); + const demo = yield* pinned(archive); const attempts = new Map(); const count = (url: string): number => { const next = (attempts.get(url) ?? 0) + 1; @@ -266,17 +324,9 @@ describe("slim-services artifact source", () => { }); const flaky: FetchLike = (input) => { const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve( - count(url) === 1 - ? new Response("", { status: 504 }) - : new Response(`${expected} demo-v1.0.0-linux-amd64.tar.zst\n`), - ); if (url.endsWith("manifest.json")) return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), + count(url) === 1 ? new Response("", { status: 504 }) : new Response(demoManifest), ); return Promise.resolve( count(url) === 1 ? new Response(truncated()) : new Response(archive), @@ -284,78 +334,68 @@ describe("slim-services artifact source", () => { }; const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-retry-" }); - const source = makeSlimServicesSource(() => artifact, { backoff: immediate }); - expect(yield* withFetch(flaky, source.checksum(request))).toBe(expected); - expect(attempts.get("https://example.test/SHA256SUMS")).toBe(2); - yield* withFetch(flaky, source.materialize(request, destination, expected)); + const source = makeSlimServicesSource(() => demo, { backoff: immediate }); + yield* withFetch(flaky, source.materialize(request, destination, demo.sha256)); expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); - expect(attempts.get(artifact.mirrors[0].downloadUrl)).toBe(2); + expect(attempts.get(demo.mirrors[0].manifestUrl)).toBe(2); + expect(attempts.get(demo.mirrors[0].downloadUrl)).toBe(2); }).pipe(Effect.provide(NodeServices.layer)), ), ); it.live("spends five attempts on a persistent gateway error and one on a missing asset", () => - Effect.gen(function* () { - let gateway = 0; - const exhausted = yield* withFetch(() => { - gateway += 1; - return Promise.resolve(new Response("", { status: 504 })); - }, slimServicesChecksum(artifact, immediate).pipe(Effect.exit)); - expect(errorOf(exhausted)).toBeInstanceOf(PreparationError); - expect(gateway).toBe(5); - - let missing = 0; - const failed = yield* withFetch(() => { - missing += 1; - return Promise.resolve(new Response("", { status: 404 })); - }, slimServicesChecksum(artifact, immediate).pipe(Effect.exit)); - expect(errorOf(failed)).toBeInstanceOf(PreparationError); - expect(missing).toBe(1); - }), + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const source = makeSlimServicesSource(() => artifact, { backoff: immediate }); + + let gateway = 0; + const exhausted = yield* withFetch( + () => { + gateway += 1; + return Promise.resolve(new Response("", { status: 504 })); + }, + source + .materialize( + request, + yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-gateway-" }), + artifact.sha256, + ) + .pipe(Effect.exit), + ); + expect(errorOf(exhausted)).toBeInstanceOf(PreparationError); + expect(gateway).toBe(5); + + let missing = 0; + const failed = yield* withFetch( + () => { + missing += 1; + return Promise.resolve(new Response("", { status: 404 })); + }, + source + .materialize( + request, + yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-missing-" }), + artifact.sha256, + ) + .pipe(Effect.exit), + ); + expect(errorOf(failed)).toBeInstanceOf(PreparationError); + expect(missing).toBe(1); + }).pipe(Effect.provide(NodeServices.layer)), + ), ); it.live("prepares the artifact from the next mirror when the release host is blocked", () => Effect.scoped( Effect.gen(function* () { const archive = yield* compress(tar("bin/demo", "demo")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const mirrored: SlimServicesArtifact = { + const mirrored = yield* pinned(archive, demoManifest, { ...artifact, - checksums: [ - { kind: "sha256sums", url: "https://release.test/SHA256SUMS" }, - registryChecksum, - ], - mirrors: [ - { - downloadUrl: "https://release.test/demo-v1.0.0-linux-amd64.tar.zst", - manifestUrl: "https://release.test/demo-v1.0.0-linux-amd64.manifest.json", - }, - { - downloadUrl: "https://bucket.test/demo-v1.0.0-linux-amd64.tar.zst", - manifestUrl: "https://bucket.test/demo-v1.0.0-linux-amd64.manifest.json", - }, - ], - }; - const blocked = new Set(); - const bucket: string[] = []; - const fetcher: FetchLike = (input, init) => { - const url = requestUrl(input); - if (url.startsWith("https://release.test/")) { - blocked.add(url); - return Promise.resolve(new Response("", { status: 403 })); - } - const registry = registryResponse(input, init, expected); - if (registry !== undefined) return Promise.resolve(registry); - bucket.push(url); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; + mirrors: [releaseMirror, bucketMirror], + }); + const requested: string[] = []; + const fetcher = serving({ "bucket.test": { archive, manifest: demoManifest } }, requested); const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-fallback-" }); const store = yield* makeArtifactStore({ @@ -365,100 +405,140 @@ describe("slim-services artifact source", () => { const prepared = yield* withFetch(fetcher, store.prepare(request)); expect(prepared.outcome).toBe("downloaded"); expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); - expect([...blocked]).toEqual([ - "https://release.test/SHA256SUMS", - "https://release.test/demo-v1.0.0-linux-amd64.manifest.json", + expect(requested).toEqual([ + releaseMirror.manifestUrl, + bucketMirror.manifestUrl, + bucketMirror.downloadUrl, ]); - expect(bucket).toEqual([ - "https://bucket.test/demo-v1.0.0-linux-amd64.manifest.json", - "https://bucket.test/demo-v1.0.0-linux-amd64.tar.zst", + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); + + it.live("falls through to the next mirror when the primary serves a tampered archive", () => + Effect.scoped( + Effect.gen(function* () { + const archive = yield* compress(tar("bin/demo", "demo")); + const tampered = yield* compress(tar("bin/demo", "evil")); + const mirrored = yield* pinned(archive, demoManifest, { + ...artifact, + mirrors: [releaseMirror, bucketMirror], + }); + const requested: string[] = []; + const fetcher = serving( + { + "release.test": { archive: tampered, manifest: demoManifest }, + "bucket.test": { archive, manifest: demoManifest }, + }, + requested, + ); + const fs = yield* FileSystem.FileSystem; + const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-archive-" }); + yield* withFetch( + fetcher, + makeSlimServicesSource(() => mirrored).materialize(request, destination, mirrored.sha256), + ); + expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); + expect(requested).toEqual([ + releaseMirror.manifestUrl, + releaseMirror.downloadUrl, + bucketMirror.manifestUrl, + bucketMirror.downloadUrl, ]); }).pipe(Effect.provide(NodeServices.layer)), ), ); - it.live("names every failed checksum source in the aggregated error", () => - Effect.gen(function* () { - const mirrored: SlimServicesArtifact = { - ...artifact, - checksums: [ - { kind: "sha256sums", url: "https://release.test/SHA256SUMS" }, - registryChecksum, - ], - }; - const requested: string[] = []; - const failed = yield* withFetch((input) => { - requested.push(requestUrl(input)); - return Promise.resolve(new Response("", { status: 403 })); - }, slimServicesChecksum(mirrored, immediate).pipe(Effect.exit)); - expect(errorOf(failed)?.message).toBe( - "Unable to resolve the slim-services checksum: https://release.test/SHA256SUMS " + - "(Unable to download https://release.test/SHA256SUMS: HTTP 403); " + - "registry.test/supabase/cli/demo:v1.0.0-native-linux-amd64 (Unable to download " + - "https://registry.test/token?scope=repository:supabase/cli/demo:pull&service=registry.test: HTTP 403)", - ); - expect(errorOf(failed)).toMatchObject({ - service: artifact.service, - version: artifact.version, - }); - expect(requested).toEqual([ - "https://release.test/SHA256SUMS", - "https://registry.test/token?scope=repository:supabase/cli/demo:pull&service=registry.test", - ]); - }), + it.live("falls through to the next mirror when the primary serves a tampered manifest", () => + Effect.scoped( + Effect.gen(function* () { + const archive = yield* compress(tar("bin/demo", "demo")); + const mirrored = yield* pinned(archive, demoManifest, { + ...artifact, + mirrors: [releaseMirror, bucketMirror], + }); + const requested: string[] = []; + const fetcher = serving( + { + "release.test": { archive, manifest: commandManifest }, + "bucket.test": { archive, manifest: demoManifest }, + }, + requested, + ); + const fs = yield* FileSystem.FileSystem; + const destination = yield* fs.makeTempDirectoryScoped({ + prefix: "slim-services-manifest-", + }); + yield* withFetch( + fetcher, + makeSlimServicesSource(() => mirrored).materialize(request, destination, mirrored.sha256), + ); + expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); + expect(requested).toEqual([ + releaseMirror.manifestUrl, + bucketMirror.manifestUrl, + bucketMirror.downloadUrl, + ]); + }).pipe(Effect.provide(NodeServices.layer)), + ), ); - it.live("accepts a fallback archive only when it matches the checksum", () => + it.live("names every mirror and its mismatch when none serves the pinned bytes", () => Effect.scoped( Effect.gen(function* () { const archive = yield* compress(tar("bin/demo", "demo")); const tampered = yield* compress(tar("bin/demo", "evil")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const mirrored = (fallback: Uint8Array): SlimServicesArtifact => ({ + const tamperedManifest = manifestBytes("v1.0.0-r0", "evil"); + const mirrored = yield* pinned(archive, demoManifest, { ...artifact, - mirrors: [ - { - downloadUrl: "https://release.test/demo.tar.zst", - manifestUrl: "https://release.test/demo.manifest.json", - }, - { - downloadUrl: `https://bucket.test/${fallback === archive ? "good" : "bad"}.tar.zst`, - manifestUrl: "https://bucket.test/demo.manifest.json", - }, - ], + mirrors: [releaseMirror, bucketMirror], + }); + const fetcher = serving({ + "release.test": { archive: tampered, manifest: demoManifest }, + "bucket.test": { archive, manifest: tamperedManifest }, }); - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(url.endsWith("good.tar.zst") ? archive : tampered)); - }; const fs = yield* FileSystem.FileSystem; - - const recovered = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-recover-" }); - yield* withFetch( + const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-reject-" }); + const failed = yield* withFetch( fetcher, - makeSlimServicesSource(() => mirrored(archive)).materialize(request, recovered, expected), + makeSlimServicesSource(() => mirrored) + .materialize(request, destination, mirrored.sha256) + .pipe(Effect.exit), ); - expect(yield* fs.readFileString(`${recovered}/bin/demo`)).toBe("demo"); + expect(errorOf(failed)?.message).toBe( + "Unable to download the slim-services archive: " + + `${releaseMirror.downloadUrl} (Unable to download slim-services archive: ` + + `expected ${mirrored.sha256}, got ${yield* sha256Of(tampered)}); ` + + `${bucketMirror.downloadUrl} (Slim-services manifest does not match its pin: ` + + `expected ${mirrored.manifestSha256}, got ${yield* sha256Of(tamperedManifest)})`, + ); + expect(yield* fs.exists(`${destination}/bin/demo`)).toBe(false); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); - const rejected = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-reject-" }); + it.live("rejects a manifest that names the upstream version instead of the release", () => + Effect.scoped( + Effect.gen(function* () { + const archive = yield* compress(tar("bin/demo", "demo")); + const upstreamManifest = manifestBytes("v1.0.0"); + const demo = yield* pinned(archive, upstreamManifest); + const requested: string[] = []; + const fetcher = serving( + { "example.test": { archive, manifest: upstreamManifest } }, + requested, + ); + const fs = yield* FileSystem.FileSystem; + const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-version-" }); const failed = yield* withFetch( fetcher, - makeSlimServicesSource(() => mirrored(tampered)) - .materialize(request, rejected, expected) + makeSlimServicesSource(() => demo) + .materialize(request, destination, demo.sha256) .pipe(Effect.exit), ); - const message = errorOf(failed)?.message; - expect(message).toContain("Unable to download the slim-services archive"); - expect(message).toContain("https://release.test/demo.tar.zst"); - expect(message).toContain("https://bucket.test/bad.tar.zst"); - expect(yield* fs.exists(`${rejected}/bin/demo`)).toBe(false); + expect(errorOf(failed)?.message).toBe( + "Slim-services manifest does not match the catalog artifact", + ); + expect(requested).toEqual([demo.mirrors[0].manifestUrl]); }).pipe(Effect.provide(NodeServices.layer)), ), ); @@ -467,26 +547,14 @@ describe("slim-services artifact source", () => { Effect.scoped( Effect.gen(function* () { const archive = yield* compress(tar("../outside", "unsafe")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve(new Response(expected + " demo-v1.0.0-linux-amd64.tar.zst\n")); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; - const source = makeSlimServicesSource(() => artifact); + const demo = yield* pinned(archive); const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-unsafe-" }); const failed = yield* withFetch( - fetcher, - source.materialize(request, destination, expected).pipe(Effect.exit), + serving({ "example.test": { archive, manifest: demoManifest } }), + makeSlimServicesSource(() => demo) + .materialize(request, destination, demo.sha256) + .pipe(Effect.exit), ); expect(errorOf(failed)).toBeInstanceOf(PreparationError); expect(yield* fs.exists(`${destination}/outside`)).toBe(false); @@ -503,48 +571,21 @@ describe("slim-services artifact source", () => { { name: "bin/current", link: "demo" }, ]), ); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve(new Response(expected + " demo-v1.0.0-linux-amd64.tar.zst\n")); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; + const demo = yield* pinned(archive); const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-links-" }); yield* withFetch( - fetcher, - makeSlimServicesSource(() => artifact).materialize(request, destination, expected), + serving({ "example.test": { archive, manifest: demoManifest } }), + makeSlimServicesSource(() => demo).materialize(request, destination, demo.sha256), ); expect(yield* fs.readFileString(`${destination}/bin/current`)).toBe("demo"); const malformed = yield* compress(new Uint8Array([1, 2, 3])); - const malformedDigest = digestHex(yield* crypto.digest("SHA-256", malformed)); - const malformedFetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve( - new Response(malformedDigest + " demo-v1.0.0-linux-amd64.tar.zst\n"), - ); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(malformed)); - }; + const malformedDemo = yield* pinned(malformed); const failed = yield* withFetch( - malformedFetcher, - makeSlimServicesSource(() => artifact) - .materialize(request, destination, malformedDigest) + serving({ "example.test": { archive: malformed, manifest: demoManifest } }), + makeSlimServicesSource(() => malformedDemo) + .materialize(request, destination, malformedDemo.sha256) .pipe(Effect.exit), ); expect(errorOf(failed)).toBeInstanceOf(PreparationError); @@ -561,28 +602,15 @@ describe("slim-services artifact source", () => { { name: "bin/escape", link: "../../outside" }, ]), ); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve(new Response(`${expected} demo-v1.0.0-linux-amd64.tar.zst\n`)); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; + const demo = yield* pinned(archive); const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-link-escape-", }); const failed = yield* withFetch( - fetcher, - makeSlimServicesSource(() => artifact) - .materialize(request, destination, expected) + serving({ "example.test": { archive, manifest: demoManifest } }), + makeSlimServicesSource(() => demo) + .materialize(request, destination, demo.sha256) .pipe(Effect.exit), ); expect(errorOf(failed)).toBeInstanceOf(PreparationError); @@ -594,20 +622,12 @@ describe("slim-services artifact source", () => { it.live("interrupts an in-flight download without publishing a staging artifact", () => Effect.scoped( Effect.gen(function* () { + const demo = yield* pinned(new Uint8Array()); const started = yield* Deferred.make(); let signal: AbortSignal | undefined; const fetcher: FetchLike = (input, init) => { - const url = requestUrl(input); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve( - new Response("0".repeat(64) + " demo-v1.0.0-linux-amd64.tar.zst\n"), - ); + if (requestUrl(input).endsWith("manifest.json")) + return Promise.resolve(new Response(demoManifest)); signal = init?.signal ?? undefined; Deferred.doneUnsafe(started, Effect.void); return waitForAbort(signal); @@ -619,11 +639,7 @@ describe("slim-services artifact source", () => { const fiber = yield* Effect.forkChild( withFetch( fetcher, - makeSlimServicesSource(() => artifact).materialize( - request, - destination, - "0".repeat(64), - ), + makeSlimServicesSource(() => demo).materialize(request, destination, demo.sha256), ), { startImmediately: true }, ); @@ -635,39 +651,25 @@ describe("slim-services artifact source", () => { ), ); - it.live("rejects a streamed checksum mismatch before publishing and retries cleanly", () => + it.live("rejects an archive that misses its pin before publishing and retries cleanly", () => Effect.scoped( Effect.gen(function* () { const archive = yield* compress(tar("bin/demo", "demo")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - let validChecksum = false; - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve( - new Response( - `${validChecksum ? expected : "0".repeat(64)} demo-v1.0.0-linux-amd64.tar.zst\n`, - ), - ); - return Promise.resolve(new Response(archive)); - }; + const demo = yield* pinned(archive); + let current: SlimServicesArtifact = { ...demo, sha256: "0".repeat(64) }; const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-store-integrity-", }); - const source = makeSlimServicesSource(() => artifact); - const store = yield* makeArtifactStore({ cacheRoot: root, source }); + const store = yield* makeArtifactStore({ + cacheRoot: root, + source: makeSlimServicesSource(() => current), + }); + const fetcher = serving({ "example.test": { archive, manifest: demoManifest } }); const failed = yield* withFetch(fetcher, store.prepare(request).pipe(Effect.exit)); expect(Exit.isFailure(failed)).toBe(true); expect(yield* fs.exists(`${root}/demo/v1`)).toBe(false); - validChecksum = true; + current = demo; const prepared = yield* withFetch(fetcher, store.prepare(request)); expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); }).pipe(Effect.provide(NodeServices.layer)), @@ -677,21 +679,13 @@ describe("slim-services artifact source", () => { it.live("cancels a streamed response after transfer starts and removes its staging file", () => Effect.scoped( Effect.gen(function* () { + const demo = yield* pinned(new Uint8Array()); const started = yield* Deferred.make(); let signal: AbortSignal | undefined; let canceled = false; const fetcher: FetchLike = (input, init) => { - const url = requestUrl(input); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve( - new Response("0".repeat(64) + " demo-v1.0.0-linux-amd64.tar.zst\n"), - ); + if (requestUrl(input).endsWith("manifest.json")) + return Promise.resolve(new Response(demoManifest)); signal = init?.signal ?? undefined; let pulls = 0; const released = Deferred.makeUnsafe(); @@ -718,11 +712,7 @@ describe("slim-services artifact source", () => { const fiber = yield* Effect.forkChild( withFetch( fetcher, - makeSlimServicesSource(() => artifact).materialize( - request, - destination, - "0".repeat(64), - ), + makeSlimServicesSource(() => demo).materialize(request, destination, demo.sha256), ), { startImmediately: true }, ); @@ -739,8 +729,7 @@ describe("slim-services artifact source", () => { Effect.scoped( Effect.gen(function* () { const archive = yield* compress(tar("bin/demo", "demo")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); + const demo = yield* pinned(archive); const started = yield* Deferred.make(); let destroyed = false; const decompressor: ZstdDecompressor = { @@ -752,27 +741,15 @@ describe("slim-services artifact source", () => { }); }), }; - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve(new Response(`${expected} demo-v1.0.0-linux-amd64.tar.zst\n`)); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-zstd-" }); const fiber = yield* Effect.forkChild( withFetch( - fetcher, - makeSlimServicesSource(() => artifact, { decompressor }).materialize( + serving({ "example.test": { archive, manifest: demoManifest } }), + makeSlimServicesSource(() => demo, { decompressor }).materialize( request, destination, - expected, + demo.sha256, ), ), { startImmediately: true }, @@ -790,114 +767,106 @@ describe("slim-services artifact source", () => { Effect.gen(function* () { const longName = `bin/${"long-function-name-".repeat(8)}.js`; const archive = yield* compress(paxTar(longName)); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve(new Response(expected + " demo-v1.0.0-linux-amd64.tar.zst\n")); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; + const demo = yield* pinned(archive); const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-pax-" }); yield* withFetch( - fetcher, - makeSlimServicesSource(() => artifact).materialize(request, destination, expected), + serving({ "example.test": { archive, manifest: demoManifest } }), + makeSlimServicesSource(() => demo).materialize(request, destination, demo.sha256), ); expect(yield* fs.exists(`${destination}/${longName}`)).toBe(true); }).pipe(Effect.provide(NodeServices.layer)), ), ); - - it.live("publishes the extracted slim artifact through the verified store", () => - Effect.scoped( - Effect.gen(function* () { - const archive = yield* compress(tar("bin/demo", "demo")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const checksums = `${expected} demo-v1.0.0-linux-amd64.tar.zst\n`; - let checksumRequests = 0; - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) { - checksumRequests += 1; - return Promise.resolve(new Response(checksums)); - } - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; - const source = makeSlimServicesSource(() => artifact); - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-store-" }); - const store = yield* makeArtifactStore({ cacheRoot: root, source }); - const prepared = yield* withFetch(fetcher, store.prepare(request)); - expect(prepared.outcome).toBe("downloaded"); - expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); - expect(yield* fs.exists(`${prepared.path}/.artifact.json`)).toBe(true); - expect(checksumRequests).toBe(1); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); }); describe("native artifact catalog", () => { - it.live("verifies an S3 download against GHCR when GitHub release assets are blocked", () => + const linux = { os: "linux", arch: "x64" }; + const s3 = "supabase-cli-artifacts.s3.us-east-1.amazonaws.com"; + + it.live( + "requests release-versioned assets from GitHub, then S3, and rejects unpinned bytes", + () => + Effect.scoped( + Effect.gen(function* () { + const { version, releaseVersion } = yield* resolveArtifact({ service: "rest" }); + expect(releaseVersion).toMatch(/-r(0|[1-9][0-9]*)$/u); + expect(releaseVersion.startsWith(`${version}-r`)).toBe(true); + const asset = `postgrest-${releaseVersion}-linux-amd64`; + const archive = yield* compress(tar("bin/postgrest", "postgrest")); + const manifest = manifestBytes(releaseVersion, "postgrest"); + const requested: string[] = []; + const fetcher = serving( + { "github.com": { archive, manifest }, [s3]: { archive, manifest } }, + requested, + ); + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-catalog-" }); + const failed = yield* withFetch( + fetcher, + prepareNativeArtifact({ service: "rest" }, cacheRoot, linux).pipe(Effect.exit), + ); + const github = `https://github.com/supabase/slim-services/releases/download/postgrest-${releaseVersion}`; + const bucket = `https://${s3}/postgrest/${releaseVersion}`; + expect(requested).toEqual([ + `${github}/${asset}.manifest.json`, + `${bucket}/${asset}.manifest.json`, + ]); + const message = errorOf(failed)?.message ?? ""; + expect(message).toContain( + `${github}/${asset}.tar.zst (Slim-services manifest does not match its pin`, + ); + expect(message).toContain( + `${bucket}/${asset}.tar.zst (Slim-services manifest does not match its pin`, + ); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); + + it.live("keys the native cache by release version and ignores a legacy upstream entry", () => Effect.scoped( Effect.gen(function* () { - const { version } = yield* resolveArtifact({ service: "rest" }); - const asset = `postgrest-${version}-linux-amd64`; - const archive = yield* compress(tar("bin/postgrest", "postgrest")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const served: string[] = []; - const fetcher: FetchLike = (input, init) => { - const url = requestUrl(input); - if (url.startsWith("https://github.com/")) - return Promise.resolve(new Response("", { status: 403 })); - served.push(url); - const registry = registryResponse( - input, - init, - expected, - "ghcr.io", - "supabase/cli/postgrest", - `${version}-native-linux-amd64`, - ); - if (registry !== undefined) return Promise.resolve(registry); - if (url.endsWith(".manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "postgrest", version, target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; + const resolved = yield* resolveArtifact({ service: "rest" }); const fs = yield* FileSystem.FileSystem; - const cacheRoot = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-catalog-" }); - const prepared = yield* withFetch( - fetcher, - prepareNativeArtifact({ service: "rest" }, cacheRoot, { os: "linux", arch: "x64" }), + const cacheRoot = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-cache-" }); + const requested: string[] = []; + const seed = (version: string, content: string) => + makeArtifactStore({ cacheRoot, source: fixtureSource(content) }).pipe( + Effect.flatMap((store) => + withFetch( + serving({}, requested), + store.prepare({ + key: `slim-services/postgrest/${version}/linux-amd64`, + requiredRuntimePaths: resolved.requiredRuntimePaths, + executablePath: resolved.executablePath, + }), + ), + ), + ); + const prepare = withFetch( + serving({}, requested), + prepareNativeArtifact({ service: "rest" }, cacheRoot, linux), ); - expect(yield* fs.readFileString(prepared.executable)).toBe("postgrest"); - const bucket = `https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com/postgrest/${version}`; - expect(served).toEqual([ - "https://ghcr.io/token?scope=repository:supabase/cli/postgrest:pull&service=ghcr.io", - `https://ghcr.io/v2/supabase/cli/postgrest/manifests/${version}-native-linux-amd64`, - `${bucket}/${asset}.manifest.json`, - `${bucket}/${asset}.tar.zst`, - ]); + + yield* seed(resolved.version, "legacy"); + expect(Exit.isFailure(yield* prepare.pipe(Effect.exit))).toBe(true); + expect(requested).not.toEqual([]); + + const seeded = yield* seed(resolved.releaseVersion, "pinned"); + requested.length = 0; + const prepared = yield* prepare; + expect(prepared.root).toBe(seeded.path); + expect(yield* fs.readFileString(prepared.executable)).toBe("pinned"); + expect(requested).toEqual([]); }).pipe(Effect.provide(NodeServices.layer)), ), ); + + it("catalog has no placeholder pins", () => { + const digests = catalogPins().flatMap(({ pin }) => [ + pin.image.slice(pin.image.lastIndexOf(":") + 1), + ...Object.values(pin.natives).flatMap((native) => [native.archive, native.manifest]), + ]); + expect(digests).not.toContain("0".repeat(64)); + }); }); diff --git a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts index 4d3bbc131e..a89e190335 100644 --- a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts +++ b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts @@ -2,7 +2,7 @@ import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Crypto, Data, Effect, Exit, FileSystem, Path, Ref, Schema, Scope, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { resolveArtifact } from "../Artifacts.ts"; +import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; import { makeContainerRuntime } from "../runtime/Container.ts"; import { makeDockerDatabaseStorage } from "../storage/DockerDatabaseStorage.ts"; import { makeDockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; @@ -10,7 +10,8 @@ import { shellQuote } from "../storage/DockerSnapshotBackend.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { makeDatabaseSnapshots, type SnapshotScope } from "./DatabaseSnapshot.ts"; -const version = "17.6.1.173"; +// Derived from the real catalog (not hardcoded), so a Postgres pin bump never makes this go stale. +const version = postgresVersion("17"); class ShellError extends Data.TaggedError("ShellError")<{ readonly message: string }> {} @@ -510,8 +511,10 @@ for (const { name, make } of engines) const saved = yield* shell(`${tool}cat ${descriptor}`); const target = yield* engine.instance("target"); + // Any version different from `version` proves the mismatch check; this one is a + // synthetic sentinel, not a real catalog pin, so it stays correct across catalog bumps. yield* shell( - `printf '%s' ${shellQuote(saved.replace(version, "15.14.1.173"))} > ${descriptor}`, + `printf '%s' ${shellQuote(saved.replace(version, "0.0.0-version-mismatch"))} > ${descriptor}`, ); expect(yield* target.restoreSnapshot("key")).toBe(false); expect(yield* contents(target.data)).toBe(""); diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index 6e7484b6d7..9f14cc70ce 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -24,6 +24,7 @@ import { systemError } from "effect/PlatformError"; import * as Net from "node:net"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- the collision fixture owns a local HTTP listener. import * as NodeHttp from "node:http"; +import { catalogPins, resolveArtifact, type ServiceKind } from "../Artifacts.ts"; import { makeArtifactStore, type ArtifactRequest, @@ -48,12 +49,21 @@ import * as Pooler from "./Pooler.ts"; type TestCreation = RecipeCreation<"rest", Record> & { readonly service: "rest"; - readonly version: "v16.2"; + readonly version: string; }; +// The real catalog's default postgrest pin, not hardcoded — `makeProcessRecipe` resolves this +// through the real catalog, which would otherwise fail once a bump moves past a literal. +const postgrestVersion = catalogPins().find( + (entry) => entry.sourceService === "postgrest" && entry.isDefault, +)?.pin.upstreamVersion; +if (postgrestVersion === undefined) { + throw new Error("no default postgrest catalog pin found"); +} + const creation: TestCreation = { service: "rest", - version: "v16.2", + version: postgrestVersion, config: {}, }; @@ -459,6 +469,7 @@ const platform = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); const nativeFixtureArtifact = Effect.fn(function* ( cacheRoot: string, artifact: { + readonly service: ServiceKind; readonly name: string; readonly executablePath: string; readonly files: Readonly>; @@ -475,8 +486,9 @@ const nativeFixtureArtifact = Effect.fn(function* ( : undefined; if (target === undefined) return yield* Effect.fail(`Unsupported test platform: ${platformName}`); + const { releaseVersion } = yield* resolveArtifact({ service: artifact.service }); const request: ArtifactRequest = { - key: `slim-services/${artifact.name}/${target}`, + key: `slim-services/${artifact.name}/${releaseVersion}/${target}`, requiredRuntimePaths: Object.keys(artifact.files), executablePath: artifact.executablePath, }; @@ -506,7 +518,8 @@ const nativeFixtureArtifact = Effect.fn(function* ( const nativeRestArtifact = (cacheRoot: string, program = "") => nativeFixtureArtifact(cacheRoot, { - name: "postgrest/v16.2", + service: "rest", + name: "postgrest", executablePath: "bin/postgrest", files: { "bin/postgrest": `#!${process.execPath}\n${program}` }, }); @@ -530,7 +543,8 @@ const nativePoolerArtifact = (cacheRoot: string) => { `});\n`; const oneShot = `#!${process.execPath}\nprocess.exit(0);\n`; return nativeFixtureArtifact(cacheRoot, { - name: "pooler/v2.9.12", + service: "pooler", + name: "pooler", executablePath: "bin/server", files: { "bin/server": server, "bin/prepare": oneShot, "bin/provision-tenant": oneShot }, }); diff --git a/packages/stack/src/services/Vector.ts b/packages/stack/src/services/Vector.ts index da59642e4b..aeab546a93 100644 --- a/packages/stack/src/services/Vector.ts +++ b/packages/stack/src/services/Vector.ts @@ -20,8 +20,35 @@ export interface Endpoints extends Schema.Schema.Type {} export const Creation = serviceCreation("vector", Config, Endpoints); export interface Creation extends Schema.Schema.Type {} -// Vector has no API flag or env var, so the recipe loads this alongside the pipeline config. -const apiConfig = 'api:\n enabled: true\n address: "${VECTOR_API_ADDRESS}"\n'; +/** + * Vector has no API flag or env var, so the recipe loads this alongside the pipeline config. + * `address` is templated directly into the file content at write time (not through Vector's own + * `${VAR}` config interpolation, which 0.58 disabled by default) — this is the only stack-owned + * config Vector loads that ever varied per launch, so no interpolation flag is needed at all. + */ +const apiConfigFor = (address: string | undefined): string => + address === undefined + ? "api:\n enabled: false\n" + : `api:\n enabled: true\n address: "${address}"\n`; + +/** + * Every `${VAR}` name the recipe's own `env` sets and documents for a pipeline config + * (`Config.analyticsUrl`/`apiKey`, mirrored into `LOGFLARE_URL`/`LOGFLARE_PRIVATE_ACCESS_TOKEN`). + * A caller-supplied pipeline (`Config.configPath`) may reference these the same way the recipe's + * own API config used to. `renderKnownPlaceholders` substitutes only this closed, recipe-owned + * set directly into the file at write time — never a caller's or the process's arbitrary + * environment — so a caller config keeps working on Vector 0.58 without the recipe ever passing + * `--dangerously-allow-env-var-interpolation` (which would expose every env var, not just these). + */ +const renderKnownPlaceholders = ( + content: string, + values: Readonly>, +): string => + Object.entries(values).reduce( + (rendered, [name, value]) => + value === undefined ? rendered : rendered.replaceAll(`\${${name}}`, value), + content, + ); // Vector requires at least one source and sink; the default forwards nothing. const defaultPipelineConfig = [ @@ -73,7 +100,10 @@ const makeSpec = ( const configRoot = path.join(instanceRoot, "runtime", "vector"); const apiConfigPath = path.join(configRoot, "vector-api.yaml"); const defaultPipelinePath = path.join(configRoot, "vector.yaml"); - const pipelinePath = (creation: Creation) => creation.config.configPath ?? defaultPipelinePath; + const renderedPipelinePath = path.join(configRoot, "vector.rendered.yaml"); + /** What Vector actually loads: a caller's file is rendered to a recipe-owned copy first. */ + const resolvedPipelinePath = (creation: Creation) => + creation.config.configPath === undefined ? defaultPipelinePath : renderedPipelinePath; const ownedInstance = (operation: string) => /^[a-zA-Z0-9_-]+$/u.test(instanceId) ? Effect.void @@ -83,30 +113,47 @@ const makeSpec = ( executable: "bin/vector", ports: { http: 9001 }, healthPath: "/health", - env: (creation, endpoints, container) => + env: (creation, _endpoints, _container) => requiredInput("vector", "analyticsUrl", creation.config.analyticsUrl).pipe( - Effect.map((analyticsUrl) => { - const http = endpoints.get("http"); - return { - ...(http === undefined - ? {} - : { VECTOR_API_ADDRESS: `${container ? "0.0.0.0" : "127.0.0.1"}:${http.port}` }), - LOGFLARE_URL: analyticsUrl, - ...(creation.config.apiKey === undefined - ? {} - : { LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey }), - }; - }), + Effect.map((analyticsUrl) => ({ + LOGFLARE_URL: analyticsUrl, + ...(creation.config.apiKey === undefined + ? {} + : { LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey }), + })), ), - args: (creation, _endpoints, context) => - Effect.succeed( - context.container + args: (creation, endpoints, context) => + Effect.gen(function* () { + const http = endpoints.get("http"); + const address = + http === undefined + ? undefined + : `${context.container ? "0.0.0.0" : "127.0.0.1"}:${http.port}`; + yield* writeAtomically(fs, path, apiConfigPath, apiConfigFor(address)); + if (creation.config.configPath !== undefined) { + const source = yield* fs.readFileString(creation.config.configPath).pipe( + Effect.mapError( + (cause) => + new ServiceError({ + operation: "prepare", + message: "Unable to read Vector configPath", + cause, + }), + ), + ); + const rendered = renderKnownPlaceholders(source, { + LOGFLARE_URL: creation.config.analyticsUrl, + LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey, + }); + yield* writeAtomically(fs, path, renderedPipelinePath, rendered); + } + return context.container ? ["--config", containerPipelinePath, "--config", containerApiPath] - : ["--config", pipelinePath(creation), "--config", apiConfigPath], - ), + : ["--config", resolvedPipelinePath(creation), "--config", apiConfigPath]; + }), mounts: (creation) => Effect.succeed([ - { source: pipelinePath(creation), target: containerPipelinePath, readOnly: true }, + { source: resolvedPipelinePath(creation), target: containerPipelinePath, readOnly: true }, { source: apiConfigPath, target: containerApiPath, readOnly: true }, ]), startupCommands: [], @@ -136,6 +183,7 @@ const makeSpec = ( const owned = yield* Effect.all([ canonical(apiConfigPath), canonical(defaultPipelinePath), + canonical(renderedPipelinePath), ]).pipe( Effect.mapError( (cause) => @@ -152,7 +200,9 @@ const makeSpec = ( message: "Vector configPath must not point at a stack-owned Vector config file", }); } - yield* writeAtomically(fs, path, apiConfigPath, apiConfig); + // A safe placeholder until `args` writes the real address — the listening port isn't + // known until endpoints are reserved for an actual launch, which happens after `prepare`. + yield* writeAtomically(fs, path, apiConfigPath, apiConfigFor(undefined)); if (callerPath === undefined) yield* writeAtomically(fs, path, defaultPipelinePath, defaultPipelineConfig); }), @@ -162,6 +212,7 @@ const makeSpec = ( yield* ownedInstance("destroy"); yield* fs.remove(apiConfigPath, { force: true }); yield* fs.remove(defaultPipelinePath, { force: true }); + yield* fs.remove(renderedPipelinePath, { force: true }); if (yield* fs.exists(configRoot)) for (const entry of yield* fs.readDirectory(configRoot)) if (entry.startsWith(temporaryPrefix)) diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index 7a0be70b33..14021aa972 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -1058,7 +1058,9 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.writeFileString(path.join(nativeRoot, "data", "PG_VERSION"), "17\n"); yield* fs.writeFileString( path.join(nativeRoot, ".supabase-database-ready.json"), - '{"version":"17.6.1.173","runtime":"native","profile":"supabase"}', + // `makeDatabaseSnapshots` below resolves `version: "17"` through the real catalog + // (`postgresVersion`), so the ready marker must carry that same resolved version. + `{"version":"${postgresVersion("17")}","runtime":"native","profile":"supabase"}`, ); const nativeSnapshots = yield* makeDatabaseSnapshots({ instanceRoot: nativeRoot,