From 21d6cad430a2c3a09af579150ce236d70bf17b37 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 29 Sep 2026 09:37:58 +0200 Subject: [PATCH 01/45] feat(stack): pin slim artifacts by content and drop runtime checksum lookups --- packages/stack/src/Artifacts.ts | 174 ++-- ...dRunner.initialization.integration.test.ts | 4 +- packages/stack/src/internal/artifacts.ts | 4 + packages/stack/src/preparation/Integrity.ts | 2 +- .../src/preparation/SlimServicesSource.ts | 137 +-- .../slim-services.integration.test.ts | 866 +++++++++--------- .../ProcessRecipe.integration.test.ts | 11 +- 7 files changed, 551 insertions(+), 647 deletions(-) diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 1637340042..57e597cd8e 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -30,10 +30,34 @@ export class ArtifactError extends Data.TaggedError("ArtifactError")<{ readonly cause?: unknown; }> {} +/** Lowercase hexadecimal SHA-256 digest. */ +type Sha256 = string; + +/** Content digests of one native target's published archive and manifest. */ +export interface NativePin { + readonly archive: Sha256; + readonly manifest: Sha256; +} + +/** One published slim-services revision of an upstream version, pinned by content. */ +export interface ArtifactPin { + readonly upstreamVersion: string; + readonly revision: number; + /** `ghcr.io/supabase/cli/:@sha256:`. */ + readonly image: string; + readonly natives: Readonly>; +} + +/** The published release version, `-r`. */ +const releaseVersion = (pin: ArtifactPin): string => `${pin.upstreamVersion}-r${pin.revision}`; + interface ArtifactResolution { readonly service: ServiceKind; + /** Upstream version. */ readonly version: string; + readonly releaseVersion: string; readonly image: string; + readonly natives: ArtifactPin["natives"]; readonly executablePath: string; readonly requiredRuntimePaths: ReadonlyArray; } @@ -48,97 +72,81 @@ export interface PreparedNativeArtifact { interface ArtifactDefinition { readonly sourceService: string; readonly defaultVersion: string; - readonly images: Readonly>; + /** Pins keyed by upstream version. */ + readonly pins: Readonly>; readonly requiredRuntimePaths: ReadonlyArray; readonly executablePath: string; } const definition = ( sourceService: string, - defaultVersion: string, - image: string, + pin: ArtifactPin, executablePath: string, requiredRuntimePaths: ReadonlyArray = [executablePath], - additionalImages: Readonly> = {}, + additionalPins: Readonly> = {}, ): ArtifactDefinition => ({ sourceService, - defaultVersion, - images: { [defaultVersion]: image, ...additionalImages }, + defaultVersion: pin.upstreamVersion, + pins: { [pin.upstreamVersion]: pin, ...additionalPins }, requiredRuntimePaths, executablePath, }); +const SLIM_IMAGE_GHCR_REGISTRY = "ghcr.io/supabase/cli/"; + +/** Stand-in digest of every catalog pin that no published revision backs yet. */ +export const PLACEHOLDER_PINS = "0".repeat(64); + +const placeholderPin = (repository: string, upstreamVersion: string): ArtifactPin => { + const native = { archive: PLACEHOLDER_PINS, manifest: PLACEHOLDER_PINS }; + return { + upstreamVersion, + revision: 0, + image: `${SLIM_IMAGE_GHCR_REGISTRY}${repository}:${upstreamVersion}-r0@sha256:${PLACEHOLDER_PINS}`, + natives: { "darwin-arm64": native, "linux-amd64": native, "linux-arm64": native }, + }; +}; + const definitions: Readonly> = { database: definition( "postgres", - "17.6.1.173", - "ghcr.io/supabase/cli/postgres:17.6.1.173@sha256:9d6e542382946cad5eb1f11f1c8108a51297902ee42fe5098358816d3784ba5a", + placeholderPin("postgres", "17.6.1.173"), "bin/supabase-postgres-start", ["bin/supabase-postgres-start", "bin/pg_dump", "bin/pg_dumpall", "bin/pg_prove", "bin/psql"], - { - "15.14.1.173": - "ghcr.io/supabase/cli/postgres:15.14.1.173@sha256:3abb20e89700d6211e741148b85c3a052bb290cae3b4f751311442f4a5fe2324", - }, + { "15.14.1.173": placeholderPin("postgres", "15.14.1.173") }, ), - rest: definition("postgrest", "v16.2", "ghcr.io/supabase/cli/postgrest:v16.2", "bin/postgrest"), - auth: definition("auth", "v2.196.0", "ghcr.io/supabase/cli/auth:v2.196.0", "bin/auth"), - realtime: definition( - "realtime", - "v2.134.5", - "ghcr.io/supabase/cli/realtime:v2.134.5@sha256:ee672ffd06ca0a1712a06aea1307c134c366ec082a51b051a8d59a8ad0c72755", + rest: definition("postgrest", placeholderPin("postgrest", "v16.2"), "bin/postgrest"), + auth: definition("auth", placeholderPin("auth", "v2.196.0"), "bin/auth"), + realtime: definition("realtime", placeholderPin("realtime", "v2.134.5"), "bin/server", [ "bin/server", - ["bin/server", "bin/prepare"], - ), - storage: definition( - "storage", - "v1.73.0", - "ghcr.io/supabase/cli/storage:v1.73.0@sha256:69590a75f916837641976d4018e5ead7c7d2c2305312d9bfb06d86aec8fb1cdd", + "bin/prepare", + ]), + storage: definition("storage", placeholderPin("storage", "v1.73.0"), "bin/storage", [ "bin/storage", - ["bin/storage", "bin/prepare"], - ), - imgproxy: definition( - "imgproxy", - "v3.8.0", - "ghcr.io/supabase/cli/imgproxy:v3.8.0", - "bin/imgproxy", - ), + "bin/prepare", + ]), + imgproxy: definition("imgproxy", placeholderPin("imgproxy", "v3.8.0"), "bin/imgproxy"), functions: definition( "edge-runtime", - "v1.77.1", - "ghcr.io/supabase/cli/edge-runtime:v1.77.1@sha256:db55555ba640180671be297179797ea39c8c6cf09a22a0b883923cb86938f5e1", + placeholderPin("edge-runtime", "v1.77.1"), "bin/edge-runtime", ), - studio: definition( - "studio", - "2026.09.04-sha-5a67366", - "ghcr.io/supabase/cli/studio:2026.09.04-sha-5a67366@sha256:9823a31668028f1846e87331bc21598d9cd74bcaa1466c72dab58c33c9c82720", - "bin/studio", - ), - pgmeta: definition( - "pgmeta", - "v0.99.0", - "ghcr.io/supabase/cli/pgmeta:v0.99.0@sha256:90de2dcf03ac548ae2d1d3e71b3cd10bde4c627572720a42e4c3946b7090292e", - "bin/pgmeta", - ), - mail: definition("mailpit", "v1.30.2", "ghcr.io/supabase/cli/mailpit:v1.30.2", "bin/mailpit"), - analytics: definition( - "analytics", - "v1.50.9", - "ghcr.io/supabase/cli/analytics:v1.50.9@sha256:7db85cc6cb0cdeb4b71f2fadb49c0f9197bea0492daf7896f6ae69edad76d28e", + studio: definition("studio", placeholderPin("studio", "2026.09.04-sha-5a67366"), "bin/studio"), + pgmeta: definition("pgmeta", placeholderPin("pgmeta", "v0.99.0"), "bin/pgmeta"), + mail: definition("mailpit", placeholderPin("mailpit", "v1.30.2"), "bin/mailpit"), + analytics: definition("analytics", placeholderPin("analytics", "v1.50.9"), "bin/logflare", [ "bin/logflare", - ["bin/logflare", "bin/prepare"], - ), - vector: definition("vector", "0.53.0", "ghcr.io/supabase/cli/vector:0.53.0", "bin/vector", [ + "bin/prepare", + ]), + vector: definition("vector", placeholderPin("vector", "0.53.0"), "bin/vector", [ "bin/vector", "share/doc/vector/config/vector.yaml", ]), - pooler: definition( - "pooler", - "v2.9.12", - "ghcr.io/supabase/cli/pooler:v2.9.12@sha256:12bb9dcb7ddace79bee173ccb7327c6646af2236679f3bd932a86b3a06479aac", + pooler: definition("pooler", placeholderPin("pooler", "v2.9.12"), "bin/server", [ "bin/server", - ["bin/server", "bin/prepare", "bin/provision-tenant"], - ), + "bin/prepare", + "bin/provision-tenant", + ]), }; const targetForPlatform = (platform: { @@ -174,10 +182,6 @@ const SLIM_NATIVE_GITHUB_RELEASES = "https://github.com/supabase/slim-services/r */ const SLIM_NATIVE_SUPABASE_S3_MIRROR = "https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com"; -const SLIM_NATIVE_GHCR_REGISTRY = "ghcr.io"; -const SLIM_NATIVE_GHCR_REPOSITORY = "supabase/cli"; - -const SLIM_IMAGE_GHCR_REGISTRY = "ghcr.io/supabase/cli/"; const SLIM_IMAGE_SUPABASE_ECR_MIRROR = "public.ecr.aws/supabase/cli/"; /** Mirrors carrying a catalog slim image under the same tag and digest, in fallback order. */ @@ -192,27 +196,21 @@ const artifactFor = ( target: NativeTarget, ): SlimServicesArtifact => { const sourceService = definitions[service].sourceService; - const releaseTag = `${sourceService}-${resolved.version}`; + const releaseTag = `${sourceService}-${resolved.releaseVersion}`; const assetName = `${releaseTag}-${target}`; const githubRelease = `${SLIM_NATIVE_GITHUB_RELEASES}/${releaseTag}`; - const supabaseS3 = `${SLIM_NATIVE_SUPABASE_S3_MIRROR}/${sourceService}/${resolved.version}`; + const supabaseS3 = `${SLIM_NATIVE_SUPABASE_S3_MIRROR}/${sourceService}/${resolved.releaseVersion}`; + const pin = resolved.natives[target]; return { provider: "supabase/slim-services", service: sourceService, - version: resolved.version, + version: resolved.releaseVersion, releaseTag, target, archive: "tar.zst", assetName, - checksums: [ - { kind: "sha256sums", url: `${githubRelease}/SHA256SUMS` }, - { - kind: "oci", - registry: SLIM_NATIVE_GHCR_REGISTRY, - repository: `${SLIM_NATIVE_GHCR_REPOSITORY}/${sourceService}`, - tag: `${resolved.version}-native-${target}`, - }, - ], + sha256: pin.archive, + manifestSha256: pin.manifest, mirrors: [ { downloadUrl: `${githubRelease}/${assetName}.tar.zst`, @@ -236,8 +234,8 @@ export const resolveArtifact = Effect.fn("Artifacts.resolveArtifact")(function* return yield* new ArtifactError({ message: `Unknown service kind: ${request.service}` }); const selected = definitions[request.service]; const version = request.version ?? selected.defaultVersion; - const image = Object.entries(selected.images).find(([candidate]) => candidate === version)?.[1]; - if (image === undefined) + const pin = Object.entries(selected.pins).find(([candidate]) => candidate === version)?.[1]; + if (pin === undefined) return yield* new ArtifactError({ message: `Unsupported ${request.service} artifact version: ${version}`, service: request.service, @@ -246,7 +244,9 @@ export const resolveArtifact = Effect.fn("Artifacts.resolveArtifact")(function* return { service: request.service, version, - image, + releaseVersion: releaseVersion(pin), + image: pin.image, + natives: pin.natives, executablePath: selected.executablePath, requiredRuntimePaths: selected.requiredRuntimePaths, }; @@ -254,13 +254,23 @@ export const resolveArtifact = Effect.fn("Artifacts.resolveArtifact")(function* /** Resolves a PostgreSQL major alias against the pinned database artifacts. */ export const postgresVersion = (version: string): string => - Object.keys(definitions.database.images).find( - (candidate) => candidate.split(".")[0] === version, - ) ?? version; + Object.keys(definitions.database.pins).find((candidate) => candidate.split(".")[0] === version) ?? + version; /** Service kinds in artifact catalog order. */ export const artifactServiceKinds = (): ReadonlyArray => Record.keys(definitions); +/** Every catalog pin in catalog order, including additional upstream lines. */ +export const catalogPins = (): ReadonlyArray<{ + readonly service: ServiceKind; + readonly sourceService: string; + readonly pin: ArtifactPin; +}> => + artifactServiceKinds().flatMap((service) => { + const { sourceService, pins } = definitions[service]; + return Object.values(pins).map((pin) => ({ service, sourceService, pin })); + }); + const artifactKey = (artifact: SlimServicesArtifact): string => `slim-services/${artifact.service}/${artifact.version}/${artifact.target}`; diff --git a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts index 28e1c4c48d..ea12cd3348 100644 --- a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts +++ b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts @@ -2,6 +2,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Context, Deferred, Effect, Fiber, FileSystem, Layer, Path, Ref } from "effect"; import { TestClock } from "effect/testing"; +import { resolveArtifact } from "../Artifacts.ts"; import { initialization } from "../Commands.ts"; import { makeArtifactStore, @@ -24,8 +25,9 @@ const target = const prepareAuthArtifact = Effect.fn(function* (cacheRoot: string, script: string) { if (target === undefined) return yield* Effect.fail("Unsupported test platform"); const fs = yield* FileSystem.FileSystem; + const { releaseVersion } = yield* resolveArtifact({ service: "auth" }); const request: ArtifactRequest = { - key: `slim-services/auth/v2.196.0/${target}`, + key: `slim-services/auth/${releaseVersion}/${target}`, requiredRuntimePaths: ["bin/auth"], executablePath: "bin/auth", }; diff --git a/packages/stack/src/internal/artifacts.ts b/packages/stack/src/internal/artifacts.ts index cce55dec8c..0ee05f1fac 100644 --- a/packages/stack/src/internal/artifacts.ts +++ b/packages/stack/src/internal/artifacts.ts @@ -2,8 +2,12 @@ export { ArtifactError, artifactServiceKinds, + catalogPins, defaultRuntime, postgresVersion, prepareNativeArtifact, resolveArtifact, + type ArtifactPin, + type NativePin, + type ServiceKind, } from "../Artifacts.ts"; diff --git a/packages/stack/src/preparation/Integrity.ts b/packages/stack/src/preparation/Integrity.ts index 381aef46ca..a376dc6618 100644 --- a/packages/stack/src/preparation/Integrity.ts +++ b/packages/stack/src/preparation/Integrity.ts @@ -35,7 +35,7 @@ export const verifySha256 = Effect.fn("Integrity.verifySha256")(function* ( ); const actual = digestHex(digest); if (actual !== canonical) - return yield* integrityError("Artifact SHA-256 does not match the catalog", { + return yield* integrityError(`expected ${canonical}, got ${actual}`, { expected: canonical, actual, }); diff --git a/packages/stack/src/preparation/SlimServicesSource.ts b/packages/stack/src/preparation/SlimServicesSource.ts index f4453347b6..97ed1487e5 100644 --- a/packages/stack/src/preparation/SlimServicesSource.ts +++ b/packages/stack/src/preparation/SlimServicesSource.ts @@ -7,6 +7,7 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { errorChainMessage } from "../internal/error-message.ts"; import type { ArtifactRequest, ArtifactSource } from "./ArtifactStore.ts"; import { PreparationError } from "./Errors.ts"; +import { verifySha256 } from "./Integrity.ts"; /** One host serving the archive and manifest of a slim-services release asset. */ interface SlimServicesMirror { @@ -14,33 +15,20 @@ interface SlimServicesMirror { readonly manifestUrl: string; } -/** - * Where the expected archive digest comes from: a release `SHA256SUMS` file, or the archive layer - * of the native OCI artifact in a registry that allows anonymous pulls. - */ -type SlimServicesChecksumSource = - | { readonly kind: "sha256sums"; readonly url: string } - | { - readonly kind: "oci"; - readonly registry: string; - readonly repository: string; - readonly tag: string; - }; - export interface SlimServicesArtifact { readonly provider: "supabase/slim-services"; readonly service: string; + /** Release version, `-r`. */ readonly version: string; readonly releaseTag: string; readonly target: "darwin-arm64" | "linux-amd64" | "linux-arm64"; readonly archive: "tar.zst"; readonly assetName: string; - /** Checksum authorities, tried in order; a mirror's own checksum file counts only when listed here. */ - readonly checksums: readonly [ - SlimServicesChecksumSource, - ...ReadonlyArray, - ]; - /** Hosts carrying the same release assets, tried in order until one serves them. */ + /** Pinned archive digest. Mirrors only serve bytes; they never supply a digest. */ + readonly sha256: string; + /** Pinned manifest digest, verified before the manifest is parsed. */ + readonly manifestSha256: string; + /** Hosts carrying the same release assets, tried in order until one serves the pinned bytes. */ readonly mirrors: readonly [SlimServicesMirror, ...ReadonlyArray]; readonly requiredRuntimePaths: ReadonlyArray; readonly executablePath: string; @@ -97,9 +85,9 @@ const systemTarBoundary: TarBoundary = { ); }), }; -const responseFor = (url: string, headers?: Readonly>) => +const responseFor = (url: string) => Effect.flatMap(HttpClient.HttpClient, (client) => - HttpClient.followRedirects(client).get(url, { headers }), + HttpClient.followRedirects(client).get(url), ).pipe( Effect.flatMap((response) => Effect.gen(function* () { @@ -145,9 +133,8 @@ const withTransferRetry = const fetchBytes = Effect.fn("SlimServicesSource.fetchBytes")(function* ( url: string, backoff: Schedule.Schedule, - headers?: Readonly>, ) { - return yield* responseFor(url, headers).pipe( + return yield* responseFor(url).pipe( Effect.flatMap((response) => response.arrayBuffer), Effect.map((bytes) => new Uint8Array(bytes)), withTransferRetry(url, backoff), @@ -274,93 +261,6 @@ const firstSuccess = ( ); }; -const checksumFor = (contents: string, archiveName: string): string | undefined => - contents - .split(/\r?\n/u) - .map((line) => line.trim().match(/^([a-f0-9]{64})\s+[* ]?(.+)$/iu)) - .find((match) => match?.[2] === archiveName || match?.[2]?.endsWith(`/${archiveName}`))?.[1]; - -const ARCHIVE_MEDIA_TYPE = "application/vnd.supabase.slim.archive.v1.tar+zstd"; - -const RegistryToken = Schema.Struct({ token: Schema.String }); - -const NativeOciManifest = Schema.Struct({ - layers: Schema.Array(Schema.Struct({ mediaType: Schema.String, digest: Schema.String })), -}); - -const decodeJson = (schema: S, bytes: Uint8Array, message: string) => - Schema.decodeEffect(Schema.fromJsonString(schema))(new TextDecoder().decode(bytes)).pipe( - Effect.mapError((cause) => new PreparationError({ message, cause })), - ); - -/** Reads the archive layer digest of a native OCI artifact through an anonymous pull token. */ -const ociArchiveDigest = Effect.fn("SlimServicesSource.ociArchiveDigest")(function* ( - source: Extract, - backoff: Schedule.Schedule, -) { - const tokenUrl = `https://${source.registry}/token?scope=repository:${source.repository}:pull&service=${source.registry}`; - const { token } = yield* decodeJson( - RegistryToken, - yield* fetchBytes(tokenUrl, backoff), - "Slim-services registry token is invalid", - ); - const manifest = yield* decodeJson( - NativeOciManifest, - yield* fetchBytes( - `https://${source.registry}/v2/${source.repository}/manifests/${source.tag}`, - backoff, - { - accept: "application/vnd.oci.image.manifest.v1+json", - authorization: `Bearer ${token}`, - }, - ), - "Slim-services native OCI manifest is invalid", - ); - const digest = manifest.layers - .find((layer) => layer.mediaType === ARCHIVE_MEDIA_TYPE) - ?.digest.match(/^sha256:([a-f0-9]{64})$/u)?.[1]; - if (digest === undefined) - return yield* new PreparationError({ - message: "Slim-services native OCI manifest has no archive layer", - }); - return digest; -}); - -const describeChecksumSource = (source: SlimServicesChecksumSource): string => - source.kind === "sha256sums" - ? source.url - : `${source.registry}/${source.repository}:${source.tag}`; - -export const slimServicesChecksum = Effect.fn("SlimServicesSource.checksum")(function* ( - artifact: SlimServicesArtifact, - backoff: Schedule.Schedule = transferBackoff, -) { - return yield* firstSuccess( - "Unable to resolve the slim-services checksum", - artifact, - artifact.checksums, - describeChecksumSource, - (source) => - source.kind === "oci" - ? ociArchiveDigest(source, backoff) - : fetchBytes(source.url, backoff).pipe( - Effect.map((bytes) => new TextDecoder().decode(bytes)), - Effect.flatMap((contents) => { - const checksum = checksumFor(contents, `${artifact.assetName}.tar.zst`); - return checksum === undefined || !/^[a-f0-9]{64}$/iu.test(checksum) - ? Effect.fail( - new PreparationError({ - message: "Slim-services checksum is missing", - service: artifact.service, - version: artifact.version, - }), - ) - : Effect.succeed(checksum.toLowerCase()); - }), - ), - ); -}); - const manifestSchema = Schema.Struct({ service: Schema.String, version: Schema.String, @@ -375,6 +275,18 @@ const verifiedManifest = Effect.fn("SlimServicesSource.verifiedManifest")(functi backoff: Schedule.Schedule, ) { const manifestBytes = yield* fetchBytes(mirror.manifestUrl, backoff); + yield* verifySha256(manifestBytes, artifact.manifestSha256).pipe( + Effect.mapError( + (cause) => + new PreparationError({ + message: "Slim-services manifest does not match its pin", + service: artifact.service, + version: artifact.version, + target: artifact.target, + cause, + }), + ), + ); const manifest = yield* Schema.decodeEffect(Schema.fromJsonString(manifestSchema))( new TextDecoder().decode(manifestBytes), ).pipe( @@ -497,10 +409,7 @@ export const makeSlimServicesSource = ( return artifact; }); return { - checksum: (request) => - resolveArtifact(request).pipe( - Effect.flatMap((artifact) => slimServicesChecksum(artifact, backoff)), - ), + checksum: (request) => resolveArtifact(request).pipe(Effect.map(({ sha256 }) => sha256)), materialize: Effect.fn("SlimServicesSource.materialize")( function* (request, destination, expectedSha256, onProgress) { const fs = yield* FileSystem.FileSystem; diff --git a/packages/stack/src/preparation/slim-services.integration.test.ts b/packages/stack/src/preparation/slim-services.integration.test.ts index 9423210184..d4e043f415 100644 --- a/packages/stack/src/preparation/slim-services.integration.test.ts +++ b/packages/stack/src/preparation/slim-services.integration.test.ts @@ -17,12 +17,16 @@ import { import { createServer, type Server } from "node:http"; import { zstdCompress } from "node:zlib"; import { FetchHttpClient } from "effect/unstable/http"; -import { prepareNativeArtifact, resolveArtifact } from "../Artifacts.ts"; -import { makeArtifactStore, type ArtifactRequest } from "./ArtifactStore.ts"; +import { + catalogPins, + PLACEHOLDER_PINS, + prepareNativeArtifact, + resolveArtifact, +} from "../Artifacts.ts"; +import { makeArtifactStore, type ArtifactRequest, type ArtifactSource } from "./ArtifactStore.ts"; import { digestHex } from "./Integrity.ts"; import { makeSlimServicesSource, - slimServicesChecksum, type SlimServicesArtifact, type ZstdDecompressor, } from "./SlimServicesSource.ts"; @@ -34,15 +38,31 @@ const waitForAbort = (signal?: AbortSignal | null): Promise => const waitForRelease = (released: Deferred.Deferred): Promise => Effect.runPromise(Deferred.await(released)); +const manifestBytes = (version: string, service = "demo"): Uint8Array => + new TextEncoder().encode(JSON.stringify({ service, version, target: "linux-amd64" })); + +const demoManifest = manifestBytes("v1.0.0-r0"); + +/** Passes every manifest check except its pin. */ +const commandManifest = new TextEncoder().encode( + JSON.stringify({ + service: "demo", + version: "v1.0.0-r0", + target: "linux-amd64", + cmd: ["bin/evil"], + }), +); + const artifact: SlimServicesArtifact = { provider: "supabase/slim-services", service: "demo", - version: "v1.0.0", - releaseTag: "demo-v1.0.0", + version: "v1.0.0-r0", + releaseTag: "demo-v1.0.0-r0", target: "linux-amd64", archive: "tar.zst", - assetName: "demo-v1.0.0-linux-amd64", - checksums: [{ kind: "sha256sums", url: "https://example.test/SHA256SUMS" }], + assetName: "demo-v1.0.0-r0-linux-amd64", + sha256: "0".repeat(64), + manifestSha256: "0".repeat(64), mirrors: [ { downloadUrl: "https://example.test/demo.tar.zst", @@ -52,6 +72,34 @@ const artifact: SlimServicesArtifact = { requiredRuntimePaths: ["bin/demo"], executablePath: "bin/demo", }; + +const releaseMirror = { + downloadUrl: "https://release.test/demo-v1.0.0-r0-linux-amd64.tar.zst", + manifestUrl: "https://release.test/demo-v1.0.0-r0-linux-amd64.manifest.json", +}; +const bucketMirror = { + downloadUrl: "https://bucket.test/demo-v1.0.0-r0-linux-amd64.tar.zst", + manifestUrl: "https://bucket.test/demo-v1.0.0-r0-linux-amd64.manifest.json", +}; + +const sha256Of = Effect.fn(function* (bytes: Uint8Array) { + const crypto = yield* Crypto.Crypto; + return digestHex(yield* crypto.digest("SHA-256", bytes)); +}); + +/** Pins `base` to the digests of the given archive and manifest bytes. */ +const pinned = Effect.fn(function* ( + archive: Uint8Array, + manifest: Uint8Array = demoManifest, + base: SlimServicesArtifact = artifact, +) { + return { + ...base, + sha256: yield* sha256Of(archive), + manifestSha256: yield* sha256Of(manifest), + } satisfies SlimServicesArtifact; +}); + const request: ArtifactRequest = { key: "demo/v1", requiredRuntimePaths: ["bin/demo"], @@ -138,59 +186,62 @@ const withFetch = (fetcher: FetchLike, effect: Effect.Effect) Effect.provideService(FetchHttpClient.Fetch, fetcher), ); -const registryChecksum = { - kind: "oci", - registry: "registry.test", - repository: "supabase/cli/demo", - tag: "v1.0.0-native-linux-amd64", -} as const; +/** Serves each host's manifest and archive, answers 403 elsewhere, and records every URL. */ +const serving = + ( + hosts: Readonly< + Record + >, + requested: Array = [], + ): FetchLike => + (input) => { + const url = requestUrl(input); + requested.push(url); + const host = hosts[new URL(url).host]; + if (host === undefined) return Promise.resolve(new Response("", { status: 403 })); + return Promise.resolve( + new Response(url.endsWith(".manifest.json") ? host.manifest : host.archive), + ); + }; -/** Anonymous registry token and native manifest; the manifest requires the issued token. */ -const registryResponse = ( - input: Parameters[0], - init: Parameters[1], - archiveSha256: string, - registry = "registry.test", - repository = "supabase/cli/demo", - tag = "v1.0.0-native-linux-amd64", -): Response | undefined => { - const url = requestUrl(input); - if (url === `https://${registry}/token?scope=repository:${repository}:pull&service=${registry}`) - return new Response(JSON.stringify({ token: "anonymous" })); - if (url !== `https://${registry}/v2/${repository}/manifests/${tag}`) return undefined; - if (new Headers(init?.headers).get("authorization") !== "Bearer anonymous") - return new Response("", { status: 401 }); - return new Response( - JSON.stringify({ - layers: [ - { - mediaType: "application/vnd.supabase.slim.archive.v1.tar+zstd", - digest: `sha256:${archiveSha256}`, - }, - { - mediaType: "application/vnd.supabase.slim.checksum.v1", - digest: `sha256:${"f".repeat(64)}`, - }, - ], - }), - ); -}; +const fixtureSource = (content: string): ArtifactSource => ({ + checksum: () => Effect.succeed("0".repeat(64)), + materialize: (entry, destination) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + for (const file of entry.requiredRuntimePaths) { + yield* fs.makeDirectory(`${destination}/${file.slice(0, file.lastIndexOf("/"))}`, { + recursive: true, + }); + yield* fs.writeFileString(`${destination}/${file}`, content); + yield* fs.chmod(`${destination}/${file}`, 0o755); + } + }).pipe( + Effect.mapError( + (cause) => new PreparationError({ message: "Unable to write cache fixture", cause }), + ), + ), +}); describe("slim-services artifact source", () => { it.live("follows release redirects through the supplied Node HTTP client", () => Effect.scoped( Effect.gen(function* () { + const archive = yield* compress(tar("bin/demo", "demo")); const server = yield* Effect.acquireRelease( Effect.tryPromise({ try: () => // oxlint-disable-next-line effecttsgo/new-promise -- node server listen exposes a callback lifecycle. new Promise((resolve, reject) => { const value = createServer((request, response) => { - if (request.url === "/redirect") { + const url = request.url ?? ""; + if (url.startsWith("/redirect/")) { response.statusCode = 302; - response.setHeader("location", "/checksums"); + response.setHeader("location", url.slice("/redirect".length)); + response.end(); + return; } - response.end("a".repeat(64) + " demo-v1.0.0-linux-amd64.tar.zst\n"); + response.end(url.endsWith(".manifest.json") ? demoManifest : archive); }); value.once("error", reject); value.listen(0, "127.0.0.1", () => resolve(value)); @@ -207,50 +258,62 @@ describe("slim-services artifact source", () => { const address = server.address(); if (address === null || typeof address === "string") return yield* Effect.die("redirect server did not expose a port"); - const redirected: SlimServicesArtifact = { + const origin = `http://127.0.0.1:${address.port}/redirect`; + const redirected = yield* pinned(archive, demoManifest, { ...artifact, - checksums: [{ kind: "sha256sums", url: `http://127.0.0.1:${address.port}/redirect` }], - }; - const checksum = yield* slimServicesChecksum(redirected); - expect(checksum).toBe("a".repeat(64)); + mirrors: [ + { + downloadUrl: `${origin}/demo.tar.zst`, + manifestUrl: `${origin}/demo.manifest.json`, + }, + ], + }); + const fs = yield* FileSystem.FileSystem; + const destination = yield* fs.makeTempDirectoryScoped({ + prefix: "slim-services-redirect-", + }); + yield* makeSlimServicesSource(() => redirected).materialize( + request, + destination, + redirected.sha256, + ); + expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); }).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ), ); - it.live("verifies checksums and extracts a manifest-matched archive using injected fetch", () => + it.live("prepares a pinned archive and manifest without fetching any checksum", () => Effect.scoped( Effect.gen(function* () { const archive = yield* compress(tar("bin/demo", "demo")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const checksums = expected + " demo-v1.0.0-linux-amd64.tar.zst\n"; - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) return Promise.resolve(new Response(checksums)); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; - expect(yield* withFetch(fetcher, slimServicesChecksum(artifact))).toBe(expected); - const source = makeSlimServicesSource(() => artifact); + const demo = yield* pinned(archive); + const requested: string[] = []; + const fetcher = serving({ "example.test": { archive, manifest: demoManifest } }, requested); const fs = yield* FileSystem.FileSystem; - const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-source-" }); - yield* withFetch(fetcher, source.materialize(request, destination, expected)); - expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-store-" }); + const store = yield* makeArtifactStore({ + cacheRoot: root, + source: makeSlimServicesSource(() => demo), + }); + + const prepared = yield* withFetch(fetcher, store.prepare(request)); + expect(prepared.outcome).toBe("downloaded"); + expect(prepared.sha256).toBe(demo.sha256); + expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); + expect(requested).toEqual([demo.mirrors[0].manifestUrl, demo.mirrors[0].downloadUrl]); + + const cached = yield* withFetch(fetcher, store.prepare(request)); + expect(cached.outcome).toBe("cached"); + expect(requested).toHaveLength(2); }).pipe(Effect.provide(NodeServices.layer)), ), ); - it.live("retries a gateway error on the checksum and a truncated archive transfer", () => + it.live("retries a gateway error on the manifest and a truncated archive transfer", () => Effect.scoped( Effect.gen(function* () { const archive = yield* compress(tar("bin/demo", "demo")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); + const demo = yield* pinned(archive); const attempts = new Map(); const count = (url: string): number => { const next = (attempts.get(url) ?? 0) + 1; @@ -266,17 +329,9 @@ describe("slim-services artifact source", () => { }); const flaky: FetchLike = (input) => { const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve( - count(url) === 1 - ? new Response("", { status: 504 }) - : new Response(`${expected} demo-v1.0.0-linux-amd64.tar.zst\n`), - ); if (url.endsWith("manifest.json")) return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), + count(url) === 1 ? new Response("", { status: 504 }) : new Response(demoManifest), ); return Promise.resolve( count(url) === 1 ? new Response(truncated()) : new Response(archive), @@ -284,78 +339,68 @@ describe("slim-services artifact source", () => { }; const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-retry-" }); - const source = makeSlimServicesSource(() => artifact, { backoff: immediate }); - expect(yield* withFetch(flaky, source.checksum(request))).toBe(expected); - expect(attempts.get("https://example.test/SHA256SUMS")).toBe(2); - yield* withFetch(flaky, source.materialize(request, destination, expected)); + const source = makeSlimServicesSource(() => demo, { backoff: immediate }); + yield* withFetch(flaky, source.materialize(request, destination, demo.sha256)); expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); - expect(attempts.get(artifact.mirrors[0].downloadUrl)).toBe(2); + expect(attempts.get(demo.mirrors[0].manifestUrl)).toBe(2); + expect(attempts.get(demo.mirrors[0].downloadUrl)).toBe(2); }).pipe(Effect.provide(NodeServices.layer)), ), ); it.live("spends five attempts on a persistent gateway error and one on a missing asset", () => - Effect.gen(function* () { - let gateway = 0; - const exhausted = yield* withFetch(() => { - gateway += 1; - return Promise.resolve(new Response("", { status: 504 })); - }, slimServicesChecksum(artifact, immediate).pipe(Effect.exit)); - expect(errorOf(exhausted)).toBeInstanceOf(PreparationError); - expect(gateway).toBe(5); - - let missing = 0; - const failed = yield* withFetch(() => { - missing += 1; - return Promise.resolve(new Response("", { status: 404 })); - }, slimServicesChecksum(artifact, immediate).pipe(Effect.exit)); - expect(errorOf(failed)).toBeInstanceOf(PreparationError); - expect(missing).toBe(1); - }), + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const source = makeSlimServicesSource(() => artifact, { backoff: immediate }); + + let gateway = 0; + const exhausted = yield* withFetch( + () => { + gateway += 1; + return Promise.resolve(new Response("", { status: 504 })); + }, + source + .materialize( + request, + yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-gateway-" }), + artifact.sha256, + ) + .pipe(Effect.exit), + ); + expect(errorOf(exhausted)).toBeInstanceOf(PreparationError); + expect(gateway).toBe(5); + + let missing = 0; + const failed = yield* withFetch( + () => { + missing += 1; + return Promise.resolve(new Response("", { status: 404 })); + }, + source + .materialize( + request, + yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-missing-" }), + artifact.sha256, + ) + .pipe(Effect.exit), + ); + expect(errorOf(failed)).toBeInstanceOf(PreparationError); + expect(missing).toBe(1); + }).pipe(Effect.provide(NodeServices.layer)), + ), ); it.live("prepares the artifact from the next mirror when the release host is blocked", () => Effect.scoped( Effect.gen(function* () { const archive = yield* compress(tar("bin/demo", "demo")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const mirrored: SlimServicesArtifact = { + const mirrored = yield* pinned(archive, demoManifest, { ...artifact, - checksums: [ - { kind: "sha256sums", url: "https://release.test/SHA256SUMS" }, - registryChecksum, - ], - mirrors: [ - { - downloadUrl: "https://release.test/demo-v1.0.0-linux-amd64.tar.zst", - manifestUrl: "https://release.test/demo-v1.0.0-linux-amd64.manifest.json", - }, - { - downloadUrl: "https://bucket.test/demo-v1.0.0-linux-amd64.tar.zst", - manifestUrl: "https://bucket.test/demo-v1.0.0-linux-amd64.manifest.json", - }, - ], - }; - const blocked = new Set(); - const bucket: string[] = []; - const fetcher: FetchLike = (input, init) => { - const url = requestUrl(input); - if (url.startsWith("https://release.test/")) { - blocked.add(url); - return Promise.resolve(new Response("", { status: 403 })); - } - const registry = registryResponse(input, init, expected); - if (registry !== undefined) return Promise.resolve(registry); - bucket.push(url); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; + mirrors: [releaseMirror, bucketMirror], + }); + const requested: string[] = []; + const fetcher = serving({ "bucket.test": { archive, manifest: demoManifest } }, requested); const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-fallback-" }); const store = yield* makeArtifactStore({ @@ -365,100 +410,140 @@ describe("slim-services artifact source", () => { const prepared = yield* withFetch(fetcher, store.prepare(request)); expect(prepared.outcome).toBe("downloaded"); expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); - expect([...blocked]).toEqual([ - "https://release.test/SHA256SUMS", - "https://release.test/demo-v1.0.0-linux-amd64.manifest.json", + expect(requested).toEqual([ + releaseMirror.manifestUrl, + bucketMirror.manifestUrl, + bucketMirror.downloadUrl, ]); - expect(bucket).toEqual([ - "https://bucket.test/demo-v1.0.0-linux-amd64.manifest.json", - "https://bucket.test/demo-v1.0.0-linux-amd64.tar.zst", + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); + + it.live("falls through to the next mirror when the primary serves a tampered archive", () => + Effect.scoped( + Effect.gen(function* () { + const archive = yield* compress(tar("bin/demo", "demo")); + const tampered = yield* compress(tar("bin/demo", "evil")); + const mirrored = yield* pinned(archive, demoManifest, { + ...artifact, + mirrors: [releaseMirror, bucketMirror], + }); + const requested: string[] = []; + const fetcher = serving( + { + "release.test": { archive: tampered, manifest: demoManifest }, + "bucket.test": { archive, manifest: demoManifest }, + }, + requested, + ); + const fs = yield* FileSystem.FileSystem; + const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-archive-" }); + yield* withFetch( + fetcher, + makeSlimServicesSource(() => mirrored).materialize(request, destination, mirrored.sha256), + ); + expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); + expect(requested).toEqual([ + releaseMirror.manifestUrl, + releaseMirror.downloadUrl, + bucketMirror.manifestUrl, + bucketMirror.downloadUrl, ]); }).pipe(Effect.provide(NodeServices.layer)), ), ); - it.live("names every failed checksum source in the aggregated error", () => - Effect.gen(function* () { - const mirrored: SlimServicesArtifact = { - ...artifact, - checksums: [ - { kind: "sha256sums", url: "https://release.test/SHA256SUMS" }, - registryChecksum, - ], - }; - const requested: string[] = []; - const failed = yield* withFetch((input) => { - requested.push(requestUrl(input)); - return Promise.resolve(new Response("", { status: 403 })); - }, slimServicesChecksum(mirrored, immediate).pipe(Effect.exit)); - expect(errorOf(failed)?.message).toBe( - "Unable to resolve the slim-services checksum: https://release.test/SHA256SUMS " + - "(Unable to download https://release.test/SHA256SUMS: HTTP 403); " + - "registry.test/supabase/cli/demo:v1.0.0-native-linux-amd64 (Unable to download " + - "https://registry.test/token?scope=repository:supabase/cli/demo:pull&service=registry.test: HTTP 403)", - ); - expect(errorOf(failed)).toMatchObject({ - service: artifact.service, - version: artifact.version, - }); - expect(requested).toEqual([ - "https://release.test/SHA256SUMS", - "https://registry.test/token?scope=repository:supabase/cli/demo:pull&service=registry.test", - ]); - }), + it.live("falls through to the next mirror when the primary serves a tampered manifest", () => + Effect.scoped( + Effect.gen(function* () { + const archive = yield* compress(tar("bin/demo", "demo")); + const mirrored = yield* pinned(archive, demoManifest, { + ...artifact, + mirrors: [releaseMirror, bucketMirror], + }); + const requested: string[] = []; + const fetcher = serving( + { + "release.test": { archive, manifest: commandManifest }, + "bucket.test": { archive, manifest: demoManifest }, + }, + requested, + ); + const fs = yield* FileSystem.FileSystem; + const destination = yield* fs.makeTempDirectoryScoped({ + prefix: "slim-services-manifest-", + }); + yield* withFetch( + fetcher, + makeSlimServicesSource(() => mirrored).materialize(request, destination, mirrored.sha256), + ); + expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); + expect(requested).toEqual([ + releaseMirror.manifestUrl, + bucketMirror.manifestUrl, + bucketMirror.downloadUrl, + ]); + }).pipe(Effect.provide(NodeServices.layer)), + ), ); - it.live("accepts a fallback archive only when it matches the checksum", () => + it.live("names every mirror and its mismatch when none serves the pinned bytes", () => Effect.scoped( Effect.gen(function* () { const archive = yield* compress(tar("bin/demo", "demo")); const tampered = yield* compress(tar("bin/demo", "evil")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const mirrored = (fallback: Uint8Array): SlimServicesArtifact => ({ + const tamperedManifest = manifestBytes("v1.0.0-r0", "evil"); + const mirrored = yield* pinned(archive, demoManifest, { ...artifact, - mirrors: [ - { - downloadUrl: "https://release.test/demo.tar.zst", - manifestUrl: "https://release.test/demo.manifest.json", - }, - { - downloadUrl: `https://bucket.test/${fallback === archive ? "good" : "bad"}.tar.zst`, - manifestUrl: "https://bucket.test/demo.manifest.json", - }, - ], + mirrors: [releaseMirror, bucketMirror], + }); + const fetcher = serving({ + "release.test": { archive: tampered, manifest: demoManifest }, + "bucket.test": { archive, manifest: tamperedManifest }, }); - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(url.endsWith("good.tar.zst") ? archive : tampered)); - }; const fs = yield* FileSystem.FileSystem; - - const recovered = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-recover-" }); - yield* withFetch( + const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-reject-" }); + const failed = yield* withFetch( fetcher, - makeSlimServicesSource(() => mirrored(archive)).materialize(request, recovered, expected), + makeSlimServicesSource(() => mirrored) + .materialize(request, destination, mirrored.sha256) + .pipe(Effect.exit), + ); + expect(errorOf(failed)?.message).toBe( + "Unable to download the slim-services archive: " + + `${releaseMirror.downloadUrl} (Unable to download slim-services archive: ` + + `expected ${mirrored.sha256}, got ${yield* sha256Of(tampered)}); ` + + `${bucketMirror.downloadUrl} (Slim-services manifest does not match its pin: ` + + `expected ${mirrored.manifestSha256}, got ${yield* sha256Of(tamperedManifest)})`, ); - expect(yield* fs.readFileString(`${recovered}/bin/demo`)).toBe("demo"); + expect(yield* fs.exists(`${destination}/bin/demo`)).toBe(false); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); - const rejected = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-reject-" }); + it.live("rejects a manifest that names the upstream version instead of the release", () => + Effect.scoped( + Effect.gen(function* () { + const archive = yield* compress(tar("bin/demo", "demo")); + const upstreamManifest = manifestBytes("v1.0.0"); + const demo = yield* pinned(archive, upstreamManifest); + const requested: string[] = []; + const fetcher = serving( + { "example.test": { archive, manifest: upstreamManifest } }, + requested, + ); + const fs = yield* FileSystem.FileSystem; + const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-version-" }); const failed = yield* withFetch( fetcher, - makeSlimServicesSource(() => mirrored(tampered)) - .materialize(request, rejected, expected) + makeSlimServicesSource(() => demo) + .materialize(request, destination, demo.sha256) .pipe(Effect.exit), ); - const message = errorOf(failed)?.message; - expect(message).toContain("Unable to download the slim-services archive"); - expect(message).toContain("https://release.test/demo.tar.zst"); - expect(message).toContain("https://bucket.test/bad.tar.zst"); - expect(yield* fs.exists(`${rejected}/bin/demo`)).toBe(false); + expect(errorOf(failed)?.message).toBe( + "Slim-services manifest does not match the catalog artifact", + ); + expect(requested).toEqual([demo.mirrors[0].manifestUrl]); }).pipe(Effect.provide(NodeServices.layer)), ), ); @@ -467,26 +552,14 @@ describe("slim-services artifact source", () => { Effect.scoped( Effect.gen(function* () { const archive = yield* compress(tar("../outside", "unsafe")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve(new Response(expected + " demo-v1.0.0-linux-amd64.tar.zst\n")); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; - const source = makeSlimServicesSource(() => artifact); + const demo = yield* pinned(archive); const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-unsafe-" }); const failed = yield* withFetch( - fetcher, - source.materialize(request, destination, expected).pipe(Effect.exit), + serving({ "example.test": { archive, manifest: demoManifest } }), + makeSlimServicesSource(() => demo) + .materialize(request, destination, demo.sha256) + .pipe(Effect.exit), ); expect(errorOf(failed)).toBeInstanceOf(PreparationError); expect(yield* fs.exists(`${destination}/outside`)).toBe(false); @@ -503,48 +576,21 @@ describe("slim-services artifact source", () => { { name: "bin/current", link: "demo" }, ]), ); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve(new Response(expected + " demo-v1.0.0-linux-amd64.tar.zst\n")); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; + const demo = yield* pinned(archive); const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-links-" }); yield* withFetch( - fetcher, - makeSlimServicesSource(() => artifact).materialize(request, destination, expected), + serving({ "example.test": { archive, manifest: demoManifest } }), + makeSlimServicesSource(() => demo).materialize(request, destination, demo.sha256), ); expect(yield* fs.readFileString(`${destination}/bin/current`)).toBe("demo"); const malformed = yield* compress(new Uint8Array([1, 2, 3])); - const malformedDigest = digestHex(yield* crypto.digest("SHA-256", malformed)); - const malformedFetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve( - new Response(malformedDigest + " demo-v1.0.0-linux-amd64.tar.zst\n"), - ); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(malformed)); - }; + const malformedDemo = yield* pinned(malformed); const failed = yield* withFetch( - malformedFetcher, - makeSlimServicesSource(() => artifact) - .materialize(request, destination, malformedDigest) + serving({ "example.test": { archive: malformed, manifest: demoManifest } }), + makeSlimServicesSource(() => malformedDemo) + .materialize(request, destination, malformedDemo.sha256) .pipe(Effect.exit), ); expect(errorOf(failed)).toBeInstanceOf(PreparationError); @@ -561,28 +607,15 @@ describe("slim-services artifact source", () => { { name: "bin/escape", link: "../../outside" }, ]), ); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve(new Response(`${expected} demo-v1.0.0-linux-amd64.tar.zst\n`)); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; + const demo = yield* pinned(archive); const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-link-escape-", }); const failed = yield* withFetch( - fetcher, - makeSlimServicesSource(() => artifact) - .materialize(request, destination, expected) + serving({ "example.test": { archive, manifest: demoManifest } }), + makeSlimServicesSource(() => demo) + .materialize(request, destination, demo.sha256) .pipe(Effect.exit), ); expect(errorOf(failed)).toBeInstanceOf(PreparationError); @@ -594,20 +627,12 @@ describe("slim-services artifact source", () => { it.live("interrupts an in-flight download without publishing a staging artifact", () => Effect.scoped( Effect.gen(function* () { + const demo = yield* pinned(new Uint8Array()); const started = yield* Deferred.make(); let signal: AbortSignal | undefined; const fetcher: FetchLike = (input, init) => { - const url = requestUrl(input); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve( - new Response("0".repeat(64) + " demo-v1.0.0-linux-amd64.tar.zst\n"), - ); + if (requestUrl(input).endsWith("manifest.json")) + return Promise.resolve(new Response(demoManifest)); signal = init?.signal ?? undefined; Deferred.doneUnsafe(started, Effect.void); return waitForAbort(signal); @@ -619,11 +644,7 @@ describe("slim-services artifact source", () => { const fiber = yield* Effect.forkChild( withFetch( fetcher, - makeSlimServicesSource(() => artifact).materialize( - request, - destination, - "0".repeat(64), - ), + makeSlimServicesSource(() => demo).materialize(request, destination, demo.sha256), ), { startImmediately: true }, ); @@ -635,39 +656,25 @@ describe("slim-services artifact source", () => { ), ); - it.live("rejects a streamed checksum mismatch before publishing and retries cleanly", () => + it.live("rejects an archive that misses its pin before publishing and retries cleanly", () => Effect.scoped( Effect.gen(function* () { const archive = yield* compress(tar("bin/demo", "demo")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - let validChecksum = false; - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve( - new Response( - `${validChecksum ? expected : "0".repeat(64)} demo-v1.0.0-linux-amd64.tar.zst\n`, - ), - ); - return Promise.resolve(new Response(archive)); - }; + const demo = yield* pinned(archive); + let current: SlimServicesArtifact = { ...demo, sha256: "0".repeat(64) }; const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-store-integrity-", }); - const source = makeSlimServicesSource(() => artifact); - const store = yield* makeArtifactStore({ cacheRoot: root, source }); + const store = yield* makeArtifactStore({ + cacheRoot: root, + source: makeSlimServicesSource(() => current), + }); + const fetcher = serving({ "example.test": { archive, manifest: demoManifest } }); const failed = yield* withFetch(fetcher, store.prepare(request).pipe(Effect.exit)); expect(Exit.isFailure(failed)).toBe(true); expect(yield* fs.exists(`${root}/demo/v1`)).toBe(false); - validChecksum = true; + current = demo; const prepared = yield* withFetch(fetcher, store.prepare(request)); expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); }).pipe(Effect.provide(NodeServices.layer)), @@ -677,21 +684,13 @@ describe("slim-services artifact source", () => { it.live("cancels a streamed response after transfer starts and removes its staging file", () => Effect.scoped( Effect.gen(function* () { + const demo = yield* pinned(new Uint8Array()); const started = yield* Deferred.make(); let signal: AbortSignal | undefined; let canceled = false; const fetcher: FetchLike = (input, init) => { - const url = requestUrl(input); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve( - new Response("0".repeat(64) + " demo-v1.0.0-linux-amd64.tar.zst\n"), - ); + if (requestUrl(input).endsWith("manifest.json")) + return Promise.resolve(new Response(demoManifest)); signal = init?.signal ?? undefined; let pulls = 0; const released = Deferred.makeUnsafe(); @@ -718,11 +717,7 @@ describe("slim-services artifact source", () => { const fiber = yield* Effect.forkChild( withFetch( fetcher, - makeSlimServicesSource(() => artifact).materialize( - request, - destination, - "0".repeat(64), - ), + makeSlimServicesSource(() => demo).materialize(request, destination, demo.sha256), ), { startImmediately: true }, ); @@ -739,8 +734,7 @@ describe("slim-services artifact source", () => { Effect.scoped( Effect.gen(function* () { const archive = yield* compress(tar("bin/demo", "demo")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); + const demo = yield* pinned(archive); const started = yield* Deferred.make(); let destroyed = false; const decompressor: ZstdDecompressor = { @@ -752,27 +746,15 @@ describe("slim-services artifact source", () => { }); }), }; - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve(new Response(`${expected} demo-v1.0.0-linux-amd64.tar.zst\n`)); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-zstd-" }); const fiber = yield* Effect.forkChild( withFetch( - fetcher, - makeSlimServicesSource(() => artifact, { decompressor }).materialize( + serving({ "example.test": { archive, manifest: demoManifest } }), + makeSlimServicesSource(() => demo, { decompressor }).materialize( request, destination, - expected, + demo.sha256, ), ), { startImmediately: true }, @@ -790,114 +772,106 @@ describe("slim-services artifact source", () => { Effect.gen(function* () { const longName = `bin/${"long-function-name-".repeat(8)}.js`; const archive = yield* compress(paxTar(longName)); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) - return Promise.resolve(new Response(expected + " demo-v1.0.0-linux-amd64.tar.zst\n")); - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; + const demo = yield* pinned(archive); const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-pax-" }); yield* withFetch( - fetcher, - makeSlimServicesSource(() => artifact).materialize(request, destination, expected), + serving({ "example.test": { archive, manifest: demoManifest } }), + makeSlimServicesSource(() => demo).materialize(request, destination, demo.sha256), ); expect(yield* fs.exists(`${destination}/${longName}`)).toBe(true); }).pipe(Effect.provide(NodeServices.layer)), ), ); - - it.live("publishes the extracted slim artifact through the verified store", () => - Effect.scoped( - Effect.gen(function* () { - const archive = yield* compress(tar("bin/demo", "demo")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const checksums = `${expected} demo-v1.0.0-linux-amd64.tar.zst\n`; - let checksumRequests = 0; - const fetcher: FetchLike = (input) => { - const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) { - checksumRequests += 1; - return Promise.resolve(new Response(checksums)); - } - if (url.endsWith("manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; - const source = makeSlimServicesSource(() => artifact); - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-store-" }); - const store = yield* makeArtifactStore({ cacheRoot: root, source }); - const prepared = yield* withFetch(fetcher, store.prepare(request)); - expect(prepared.outcome).toBe("downloaded"); - expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); - expect(yield* fs.exists(`${prepared.path}/.artifact.json`)).toBe(true); - expect(checksumRequests).toBe(1); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); }); describe("native artifact catalog", () => { - it.live("verifies an S3 download against GHCR when GitHub release assets are blocked", () => + const linux = { os: "linux", arch: "x64" }; + const s3 = "supabase-cli-artifacts.s3.us-east-1.amazonaws.com"; + + it.live( + "requests release-versioned assets from GitHub, then S3, and rejects unpinned bytes", + () => + Effect.scoped( + Effect.gen(function* () { + const { version, releaseVersion } = yield* resolveArtifact({ service: "rest" }); + expect(releaseVersion).toMatch(/-r(0|[1-9][0-9]*)$/u); + expect(releaseVersion.startsWith(`${version}-r`)).toBe(true); + const asset = `postgrest-${releaseVersion}-linux-amd64`; + const archive = yield* compress(tar("bin/postgrest", "postgrest")); + const manifest = manifestBytes(releaseVersion, "postgrest"); + const requested: string[] = []; + const fetcher = serving( + { "github.com": { archive, manifest }, [s3]: { archive, manifest } }, + requested, + ); + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-catalog-" }); + const failed = yield* withFetch( + fetcher, + prepareNativeArtifact({ service: "rest" }, cacheRoot, linux).pipe(Effect.exit), + ); + const github = `https://github.com/supabase/slim-services/releases/download/postgrest-${releaseVersion}`; + const bucket = `https://${s3}/postgrest/${releaseVersion}`; + expect(requested).toEqual([ + `${github}/${asset}.manifest.json`, + `${bucket}/${asset}.manifest.json`, + ]); + const message = errorOf(failed)?.message ?? ""; + expect(message).toContain( + `${github}/${asset}.tar.zst (Slim-services manifest does not match its pin`, + ); + expect(message).toContain( + `${bucket}/${asset}.tar.zst (Slim-services manifest does not match its pin`, + ); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); + + it.live("keys the native cache by release version and ignores a legacy upstream entry", () => Effect.scoped( Effect.gen(function* () { - const { version } = yield* resolveArtifact({ service: "rest" }); - const asset = `postgrest-${version}-linux-amd64`; - const archive = yield* compress(tar("bin/postgrest", "postgrest")); - const crypto = yield* Crypto.Crypto; - const expected = digestHex(yield* crypto.digest("SHA-256", archive)); - const served: string[] = []; - const fetcher: FetchLike = (input, init) => { - const url = requestUrl(input); - if (url.startsWith("https://github.com/")) - return Promise.resolve(new Response("", { status: 403 })); - served.push(url); - const registry = registryResponse( - input, - init, - expected, - "ghcr.io", - "supabase/cli/postgrest", - `${version}-native-linux-amd64`, - ); - if (registry !== undefined) return Promise.resolve(registry); - if (url.endsWith(".manifest.json")) - return Promise.resolve( - new Response( - JSON.stringify({ service: "postgrest", version, target: "linux-amd64" }), - ), - ); - return Promise.resolve(new Response(archive)); - }; + const resolved = yield* resolveArtifact({ service: "rest" }); const fs = yield* FileSystem.FileSystem; - const cacheRoot = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-catalog-" }); - const prepared = yield* withFetch( - fetcher, - prepareNativeArtifact({ service: "rest" }, cacheRoot, { os: "linux", arch: "x64" }), + const cacheRoot = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-cache-" }); + const requested: string[] = []; + const seed = (version: string, content: string) => + makeArtifactStore({ cacheRoot, source: fixtureSource(content) }).pipe( + Effect.flatMap((store) => + withFetch( + serving({}, requested), + store.prepare({ + key: `slim-services/postgrest/${version}/linux-amd64`, + requiredRuntimePaths: resolved.requiredRuntimePaths, + executablePath: resolved.executablePath, + }), + ), + ), + ); + const prepare = withFetch( + serving({}, requested), + prepareNativeArtifact({ service: "rest" }, cacheRoot, linux), ); - expect(yield* fs.readFileString(prepared.executable)).toBe("postgrest"); - const bucket = `https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com/postgrest/${version}`; - expect(served).toEqual([ - "https://ghcr.io/token?scope=repository:supabase/cli/postgrest:pull&service=ghcr.io", - `https://ghcr.io/v2/supabase/cli/postgrest/manifests/${version}-native-linux-amd64`, - `${bucket}/${asset}.manifest.json`, - `${bucket}/${asset}.tar.zst`, - ]); + + yield* seed(resolved.version, "legacy"); + expect(Exit.isFailure(yield* prepare.pipe(Effect.exit))).toBe(true); + expect(requested).not.toEqual([]); + + const seeded = yield* seed(resolved.releaseVersion, "pinned"); + requested.length = 0; + const prepared = yield* prepare; + expect(prepared.root).toBe(seeded.path); + expect(yield* fs.readFileString(prepared.executable)).toBe("pinned"); + expect(requested).toEqual([]); }).pipe(Effect.provide(NodeServices.layer)), ), ); + + it.fails("catalog has no placeholder pins", () => { + const digests = catalogPins().flatMap(({ pin }) => [ + pin.image.slice(pin.image.lastIndexOf(":") + 1), + ...Object.values(pin.natives).flatMap((native) => [native.archive, native.manifest]), + ]); + expect(digests).not.toContain(PLACEHOLDER_PINS); + }); }); diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index 6e7484b6d7..36909f2216 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -24,6 +24,7 @@ import { systemError } from "effect/PlatformError"; import * as Net from "node:net"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- the collision fixture owns a local HTTP listener. import * as NodeHttp from "node:http"; +import { prepareNativeArtifact, resolveArtifact, type ServiceKind } from "../Artifacts.ts"; import { makeArtifactStore, type ArtifactRequest, @@ -459,6 +460,7 @@ const platform = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); const nativeFixtureArtifact = Effect.fn(function* ( cacheRoot: string, artifact: { + readonly service: ServiceKind; readonly name: string; readonly executablePath: string; readonly files: Readonly>; @@ -475,8 +477,9 @@ const nativeFixtureArtifact = Effect.fn(function* ( : undefined; if (target === undefined) return yield* Effect.fail(`Unsupported test platform: ${platformName}`); + const { releaseVersion } = yield* resolveArtifact({ service: artifact.service }); const request: ArtifactRequest = { - key: `slim-services/${artifact.name}/${target}`, + key: `slim-services/${artifact.name}/${releaseVersion}/${target}`, requiredRuntimePaths: Object.keys(artifact.files), executablePath: artifact.executablePath, }; @@ -506,7 +509,8 @@ const nativeFixtureArtifact = Effect.fn(function* ( const nativeRestArtifact = (cacheRoot: string, program = "") => nativeFixtureArtifact(cacheRoot, { - name: "postgrest/v16.2", + service: "rest", + name: "postgrest", executablePath: "bin/postgrest", files: { "bin/postgrest": `#!${process.execPath}\n${program}` }, }); @@ -530,7 +534,8 @@ const nativePoolerArtifact = (cacheRoot: string) => { `});\n`; const oneShot = `#!${process.execPath}\nprocess.exit(0);\n`; return nativeFixtureArtifact(cacheRoot, { - name: "pooler/v2.9.12", + service: "pooler", + name: "pooler", executablePath: "bin/server", files: { "bin/server": server, "bin/prepare": oneShot, "bin/provision-tenant": oneShot }, }); From 35089176b302f671dcad879e63a586da140e2972 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 29 Sep 2026 09:59:43 +0200 Subject: [PATCH 02/45] feat(ci): sync catalog pins from immutable slim revisions --- .../scripts/sync-artifacts-catalog.test.ts | 338 +++++++--- .github/scripts/sync-artifacts-catalog.ts | 610 +++++++++++++----- 2 files changed, 699 insertions(+), 249 deletions(-) diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index 809dce551f..2b40d7e975 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -1,132 +1,322 @@ import { describe, expect, test } from "bun:test"; -import { InvalidPayloadError } from "./slim-mirror-payload.ts"; +import { parseDockerfileServiceImages } from "../../apps/cli/src/shared/services/parse-dockerfile-service-images.ts"; +import { isOrioleImage, slimCatalogPin } from "../../apps/cli/src/shared/services/slim-images.ts"; +import { InvalidPayloadError, nativeFileNames, nativeObjectUrl } from "./slim-mirror-payload.ts"; import { CATALOG_PATH, planArtifactCatalogUpdate, - type ReleasePublication, + refreshCatalogPin, + type RevisionIo, } from "./sync-artifacts-catalog.ts"; -const DIGEST_B = `sha256:${"b".repeat(64)}`; -const POSTGRES_17 = `ghcr.io/supabase/cli/postgres:17.6.1.168@sha256:${"9".repeat(64)}`; -const POSTGRES_15 = `ghcr.io/supabase/cli/postgres:15.14.1.168@sha256:${"f".repeat(64)}`; +const NATIVE_TARGETS = ["darwin-arm64", "linux-amd64", "linux-arm64"] as const; + +const hex = (seed: string): string => new Bun.CryptoHasher("sha256").update(seed).digest("hex"); +const digest = (seed: string): string => `sha256:${hex(seed)}`; + +type NativeDigests = Record<(typeof NATIVE_TARGETS)[number], { archive: string; manifest: string }>; + +const nativeDigests = (seed: string): NativeDigests => ({ + "darwin-arm64": { archive: hex(`${seed}a`), manifest: hex(`${seed}A`) }, + "linux-amd64": { archive: hex(`${seed}b`), manifest: hex(`${seed}B`) }, + "linux-arm64": { archive: hex(`${seed}c`), manifest: hex(`${seed}C`) }, +}); + +/** Builds a `SHA256SUMS` body via the same file-name helper the sync script reads. */ +function checksumsFor(service: string, releaseVersion: string, digests: NativeDigests): string { + return NATIVE_TARGETS.map((target) => { + const files = nativeFileNames(service, releaseVersion, target); + return `${digests[target].archive} ${files.archive}\n${digests[target].manifest} ${files.manifest}`; + }).join("\n"); +} + +/** An `io` whose S3 copies always match the given checksums. */ +function matchingS3( + service: string, + releaseVersion: string, + digests: NativeDigests, +): RevisionIo["s3Sha256"] { + const byUrl = new Map(); + for (const target of NATIVE_TARGETS) { + const files = nativeFileNames(service, releaseVersion, target); + byUrl.set(nativeObjectUrl(service, releaseVersion, files.archive), digests[target].archive); + byUrl.set(nativeObjectUrl(service, releaseVersion, files.manifest), digests[target].manifest); + } + return async (url) => byUrl.get(url); +} const fixture = `const workloadCatalog = { database: definition( "postgres", - "17.6.1.168", - "${POSTGRES_17}", + placeholderPin("postgres", "17.6.1.168"), "bin/supabase-postgres-start", ["bin/supabase-postgres-start"], - { "15.14.1.168": "${POSTGRES_15}" }, + { "15.14.1.168": placeholderPin("postgres", "15.14.1.168") }, ), - rest: definition("postgrest", "v16.2", "ghcr.io/supabase/cli/postgrest:v16.2", "bin/postgrest"), + rest: definition("postgrest", placeholderPin("postgrest", "v16.2"), "bin/postgrest"), + storage: definition("storage", placeholderPin("storage", "v1.73.0"), "bin/storage", ["bin/storage"]), analytics: definition( "analytics", - "v1.50.9", - "ghcr.io/supabase/cli/analytics:v1.50.9@sha256:${"a".repeat(64)}", + placeholderPin("analytics", "v1.50.9"), "bin/logflare", ), }; `; -const published = (digest?: string): ReleasePublication => ({ status: "published", digest }); - describe("planArtifactCatalogUpdate", () => { - test("pins Dockerfile tags that changed and leaves the other postgres line", async () => { + test("pins to the highest committed revision, writing every native target's digests", async () => { + const digests = nativeDigests("c"); + const io: RevisionIo = { + listReleaseTags: async () => [ + "postgrest-v16.3-r0", + "postgrest-v16.3-r1", + "postgrest-v16.3-r2", + "unrelated-tag", + ], + fetchChecksums: async (service, releaseVersion) => + service === "postgrest" && releaseVersion === "v16.3-r2" + ? checksumsFor("postgrest", "v16.3-r2", digests) + : undefined, + imageDigest: async () => digest("d"), + s3Sha256: matchingS3("postgrest", "v16.3-r2", digests), + }; + const plan = await planArtifactCatalogUpdate({ - baseDockerfile: `FROM supabase/postgres:17.6.1.168 AS pg -FROM postgrest/postgrest:v16.2 AS postgrest -FROM supabase/logflare:1.50.9 AS logflare -`, - dockerfile: `FROM supabase/postgres:17.6.1.171 AS pg -FROM library/kong:2.8.1 AS kong -FROM postgrest/postgrest:v16.3 AS postgrest -FROM supabase/logflare:1.50.9 AS logflare -`, + baseDockerfile: "FROM postgrest/postgrest:v16.2 AS postgrest\n", + dockerfile: "FROM postgrest/postgrest:v16.3 AS postgrest\n", catalog: fixture, - publication: async (service) => published(service === "postgres" ? DIGEST_B : undefined), + io, }); - expect(plan.updates.map((update) => `${update.service} ${update.version}`)).toEqual([ - "postgres 17.6.1.171", - "postgrest v16.3", - ]); - expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgres:17.6.1.171@${DIGEST_B}"`); - expect(plan.source).toContain(`"${POSTGRES_15}"`); - expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgrest:v16.3"`); - expect(plan.source).toContain(`"v1.50.9"`); expect(plan.skipped).toEqual([]); + expect(plan.updates).toEqual([ + { + service: "postgrest", + version: "v16.3", + revision: 2, + previousVersion: "v16.2", + target: "default", + }, + ]); + expect(plan.source).toContain('upstreamVersion: "v16.3"'); + expect(plan.source).toContain("revision: 2"); + expect(plan.source).toContain( + `image: "ghcr.io/supabase/cli/postgrest:v16.3-r2@${digest("d")}"`, + ); + for (const target of NATIVE_TARGETS) { + expect(plan.source).toContain(`"${target}": { archive: "${digests[target].archive}"`); + expect(plan.source).toContain(`manifest: "${digests[target].manifest}" }`); + } + // The other postgres line is untouched. + expect(plan.source).toContain('placeholderPin("postgres", "15.14.1.168")'); }); - test("a missing release blocks the commit and OrioleDB does not", async () => { - const movesPostgres = await planArtifactCatalogUpdate({ - baseDockerfile: "FROM supabase/postgres:17.6.1.168 AS pg\n", - dockerfile: `FROM supabase/postgres:17.6.1.171 AS pg -FROM postgrest/postgrest:v16.3 AS postgrest -`, + test("a legacy release with no -rN tag is a blocking missing pin", async () => { + const io: RevisionIo = { + listReleaseTags: async () => ["postgrest-v16.3"], + fetchChecksums: async () => undefined, + imageDigest: async () => undefined, + s3Sha256: async () => undefined, + }; + + const plan = await planArtifactCatalogUpdate({ + baseDockerfile: "FROM postgrest/postgrest:v16.2 AS postgrest\n", + dockerfile: "FROM postgrest/postgrest:v16.3 AS postgrest\n", catalog: fixture, - publication: async () => ({ status: "missing" }), + io, }); - expect(movesPostgres.updates).toEqual([]); - expect(movesPostgres.source).toBe(fixture); - expect(movesPostgres.skipped.every((skip) => skip.blocking)).toBe(true); - - const oriole = await planArtifactCatalogUpdate({ - baseDockerfile: - "FROM supabase/postgres:17.6.1.168 AS pg\nFROM postgrest/postgrest:v16.2 AS postgrest\n", - dockerfile: `FROM supabase/postgres:16.0.0.1-orioledb AS pg -FROM postgrest/postgrest:v16.3 AS postgrest -`, + + expect(plan.updates).toEqual([]); + expect(plan.source).toBe(fixture); + expect(plan.skipped).toEqual([ + { + alias: "postgrest", + reason: "postgrest:v16.3 has no published slim-services revision.", + blocking: true, + }, + ]); + }); + + test("a longer upstream version's release tag does not satisfy a prefix collision", async () => { + const io: RevisionIo = { + // A decoy tag for a different (longer) upstream version must not count as v1.79.2's. + listReleaseTags: async () => ["storage-v1.79.23-r0"], + fetchChecksums: async () => undefined, + imageDigest: async () => undefined, + s3Sha256: async () => undefined, + }; + + const plan = await planArtifactCatalogUpdate({ + baseDockerfile: "FROM supabase/storage-api:v1.73.0 AS storage\n", + dockerfile: "FROM supabase/storage-api:v1.79.2 AS storage\n", catalog: fixture, - publication: async () => published(), + io, }); - expect(oriole.updates.map((update) => update.service)).toEqual(["postgrest"]); - expect(oriole.skipped).toEqual([ + + expect(plan.updates).toEqual([]); + expect(plan.skipped).toEqual([ { - alias: "pg", - reason: "pg supabase/postgres:16.0.0.1-orioledb has no slim image.", - blocking: false, + alias: "storage", + reason: "storage:v1.79.2 has no published slim-services revision.", + blocking: true, }, ]); - expect(oriole.source).toContain(`"${POSTGRES_17}"`); }); - test("refuses a backward pin and a tag that would escape the catalog string", async () => { - const backward = await planArtifactCatalogUpdate({ - baseDockerfile: "FROM supabase/postgres:17.6.1.170 AS pg\n", - dockerfile: "FROM supabase/postgres:17.6.1.171 AS pg\n", - catalog: fixture.replaceAll("17.6.1.168", "17.6.1.173"), - publication: async () => { - throw new Error("publication lookup"); + test("a stale S3 copy is a blocking error", async () => { + const digests = nativeDigests("e"); + const io: RevisionIo = { + listReleaseTags: async () => ["postgrest-v16.3-r0"], + fetchChecksums: async () => checksumsFor("postgrest", "v16.3-r0", digests), + imageDigest: async () => digest("f"), + s3Sha256: async (url) => { + const files = nativeFileNames("postgrest", "v16.3-r0", "darwin-arm64"); + if (url === nativeObjectUrl("postgrest", "v16.3-r0", files.archive)) return hex("stale"); + return matchingS3("postgrest", "v16.3-r0", digests)(url); }, + }; + + const plan = await planArtifactCatalogUpdate({ + baseDockerfile: "FROM postgrest/postgrest:v16.2 AS postgrest\n", + dockerfile: "FROM postgrest/postgrest:v16.3 AS postgrest\n", + catalog: fixture, + io, }); - expect(backward.updates).toEqual([]); - expect(backward.skipped.map((skip) => skip.blocking)).toEqual([true]); - expect(backward.source).toContain(`"17.6.1.173"`); + expect(plan.updates).toEqual([]); + expect(plan.source).toBe(fixture); + expect(plan.skipped).toEqual([ + { + alias: "postgrest", + reason: "S3 copy of v16.3-r0 darwin-arm64 is stale; run the slim-services mirror backfill", + blocking: true, + }, + ]); + }); + + test("a Dependabot bump older than the catalog pin is a non-blocking, unchanged skip", async () => { + const ahead = fixture.replace( + 'placeholderPin("postgrest", "v16.2")', + 'placeholderPin("postgrest", "v16.5")', + ); + const io: RevisionIo = { + listReleaseTags: async () => { + throw new Error("should not be reached for an older bump"); + }, + fetchChecksums: async () => undefined, + imageDigest: async () => undefined, + s3Sha256: async () => undefined, + }; + + const plan = await planArtifactCatalogUpdate({ + baseDockerfile: "FROM postgrest/postgrest:v16.2 AS postgrest\n", + dockerfile: "FROM postgrest/postgrest:v16.3 AS postgrest\n", + catalog: ahead, + io, + }); + + expect(plan.updates).toEqual([]); + expect(plan.source).toBe(ahead); + expect(plan.skipped).toEqual([ + { + alias: "postgrest", + reason: "postgrest v16.3 is older than the catalog pin v16.5.", + blocking: false, + }, + ]); + expect(plan.skipped.some((skip) => skip.blocking)).toBe(false); + }); + + test("refuses a tag that would escape the catalog string", async () => { await expect( planArtifactCatalogUpdate({ dockerfile: 'FROM supabase/gotrue:v1" AS gotrue\n', catalog: fixture, - publication: async () => published(), + io: { + listReleaseTags: async () => [], + fetchChecksums: async () => undefined, + imageDigest: async () => undefined, + s3Sha256: async () => undefined, + }, }), ).rejects.toThrow(InvalidPayloadError); }); }); +describe("refreshCatalogPin", () => { + test("refreshes a service to the highest revision of its currently pinned upstream", async () => { + const digests = nativeDigests("g"); + const io: RevisionIo = { + listReleaseTags: async () => ["analytics-v1.50.9-r0", "analytics-v1.50.9-r1"], + fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r1", digests), + imageDigest: async () => digest("h"), + s3Sha256: matchingS3("analytics", "v1.50.9-r1", digests), + }; + + const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); + + expect(result.update).toEqual({ + service: "analytics", + version: "v1.50.9", + revision: 1, + previousVersion: "v1.50.9", + target: "default", + }); + expect(result.source).toContain("revision: 1"); + expect(result.source).toContain( + `image: "ghcr.io/supabase/cli/analytics:v1.50.9-r1@${digest("h")}"`, + ); + }); +}); + describe("against the real catalog", () => { test("every slim Dockerfile alias is a catalog entry", async () => { const dockerfile = await Bun.file("apps/cli/src/shared/services/Dockerfile").text(); const catalog = await Bun.file(CATALOG_PATH).text(); - const plan = await planArtifactCatalogUpdate({ - dockerfile, - catalog, - publication: async () => published(`sha256:${"a".repeat(64)}`), + + const pins = parseDockerfileServiceImages(dockerfile) + .filter((from) => !isOrioleImage(from.image)) + .map((from) => slimCatalogPin(from.alias, from.image)) + .filter((pin): pin is NonNullable => pin !== undefined); + + const resolved = pins.map((pin) => { + const releaseVersion = `${pin.version}-r0`; + const digests = nativeDigests(pin.service); + return { + service: pin.service, + releaseVersion, + checksums: checksumsFor(pin.service, releaseVersion, digests), + digests, + }; }); - expect( - plan.skipped.filter((skip) => skip.blocking && !skip.reason.includes("older than")), - ).toEqual([]); + const io: RevisionIo = { + listReleaseTags: async () => + resolved.map((entry) => `${entry.service}-${entry.releaseVersion}`), + fetchChecksums: async (service, releaseVersion) => + resolved.find( + (entry) => entry.service === service && entry.releaseVersion === releaseVersion, + )?.checksums, + imageDigest: async () => digest("real"), + s3Sha256: async (url) => { + for (const entry of resolved) { + for (const target of NATIVE_TARGETS) { + const files = nativeFileNames(entry.service, entry.releaseVersion, target); + if (url === nativeObjectUrl(entry.service, entry.releaseVersion, files.archive)) { + return entry.digests[target].archive; + } + if (url === nativeObjectUrl(entry.service, entry.releaseVersion, files.manifest)) { + return entry.digests[target].manifest; + } + } + } + return undefined; + }, + }; + + const plan = await planArtifactCatalogUpdate({ dockerfile, catalog, io }); + + expect(plan.skipped.filter((skip) => skip.blocking)).toEqual([]); }); }); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index 838f2c5b75..51a5e10025 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -1,10 +1,19 @@ /** - * Pins `packages/stack/src/Artifacts.ts` to the slim workloads named by - * `apps/cli/src/shared/services/Dockerfile`. Native archive URLs are derived - * from service + version. An entry that already carries a digest keeps one: - * the published `ghcr.io/supabase/cli/:` manifest digest. + * Pins `packages/stack/src/Artifacts.ts` to committed `supabase/slim-services` + * revisions (`-r`). Every entry is pinned by content: the GHCR + * image digest, plus an archive and manifest sha256 per native target. * - * Run: `bun .github/scripts/sync-artifacts-catalog.ts [base-dockerfile]` + * Dependabot mode (used by CI): rewrites every catalog entry whose slim + * service moved between `dockerfile` and `base-dockerfile`, to the highest + * committed revision of its new upstream version. + * + * bun .github/scripts/sync-artifacts-catalog.ts [base-dockerfile] + * + * Manual mode: refreshes one catalog entry to the highest committed revision + * of a given upstream version (or of its currently pinned upstream version, + * when `--upstream` is omitted). + * + * bun .github/scripts/sync-artifacts-catalog.ts --service [--upstream ] */ import { parseDockerfileServiceImages } from "../../apps/cli/src/shared/services/parse-dockerfile-service-images.ts"; @@ -14,15 +23,24 @@ import { InvalidPayloadError, SOURCE_REGISTRY, VERSION_PATTERN, + checksumFor, escapeRegExp, + nativeFileNames, + nativeObjectUrl, } from "./slim-mirror-payload.ts"; export const CATALOG_PATH = "packages/stack/src/Artifacts.ts"; const DOCKERFILE_PATH = "apps/cli/src/shared/services/Dockerfile"; -const NATIVE_RELEASES = "https://api.github.com/repos/supabase/slim-services/releases/tags"; const SLIM_IMAGE_PREFIX = `${SOURCE_REGISTRY}/`; +/** The three native targets the catalog pins per revision. Kept self-contained; see module docs. */ +const NATIVE_TARGETS = ["darwin-arm64", "linux-amd64", "linux-arm64"] as const; +type NativeTargetName = (typeof NATIVE_TARGETS)[number]; + +const RELEASES_API = "https://api.github.com/repos/supabase/slim-services/releases"; +const RELEASE_DOWNLOAD_BASE = "https://github.com/supabase/slim-services/releases/download"; + /** Leading numeric component, `v` stripped. Only postgres carries more than one line. */ function releaseLine(version: string): string { const withoutPrefix = version.replace(/^[vV]/, ""); @@ -30,7 +48,10 @@ function releaseLine(version: string): string { return separator === -1 ? withoutPrefix : withoutPrefix.slice(0, separator); } -/** True when every numeric prefix of `next` is older than `current`. Equal prefixes are not older. */ +/** + * True when every numeric prefix of `next` is older than `current`. Equal prefixes are not + * older. Compares upstream versions only; revisions never enter this comparison. + */ function isOlderRelease(next: string, current: string): boolean { const nextParts = next.replace(/^[vV]/, "").split("."); const currentParts = current.replace(/^[vV]/, "").split("."); @@ -52,6 +73,7 @@ function leadingInteger(part: string): number | undefined { export interface CatalogPinUpdate { readonly service: string; readonly version: string; + readonly revision: number; readonly previousVersion: string; readonly target: "default" | "additional"; } @@ -59,7 +81,7 @@ export interface CatalogPinUpdate { export interface SkippedCatalogPin { readonly alias: string; readonly reason: string; - /** OrioleDB has no slim image and must not fail the other pins. */ + /** OrioleDB and an older Dependabot bump are the non-blocking skips. */ readonly blocking: boolean; } @@ -69,75 +91,254 @@ export interface CatalogPlan { readonly skipped: ReadonlyArray; } -export type ReleasePublication = - | { readonly status: "published"; readonly digest?: string } - | { readonly status: "missing" | "lookup-failed" }; +/** Content pin for one resolved `-r` revision, ready to serialize into the catalog. */ +interface ResolvedPin { + readonly upstreamVersion: string; + readonly revision: number; + readonly image: string; + readonly natives: Readonly< + Record + >; +} -/** `definition("", "", ""`. */ -function defaultEntryPattern(service: string): RegExp { - const s = escapeRegExp(service); - return new RegExp( - `(definition\\(\\s*"${s}",\\s*")([^"]+)("\\s*,\\s*")(${escapeRegExp(SLIM_IMAGE_PREFIX)}${s}:[^"]+)(")`, - ); +export type RevisionResolution = + | { readonly status: "resolved"; readonly pin: ResolvedPin } + | { + readonly status: "missing" | "incomplete" | "stale" | "lookup-failed"; + readonly message: string; + }; + +/** + * Network ports the resolver needs: the release list (for revision allocation), a release's + * `SHA256SUMS` body, the GHCR manifest digest, and a byte source's sha256. Tests stub these + * directly, the same way the previous `publication` callback was stubbed. + */ +export interface RevisionIo { + readonly listReleaseTags: () => Promise>; + readonly fetchChecksums: (service: string, releaseVersion: string) => Promise; + readonly imageDigest: (service: string, releaseVersion: string) => Promise; + readonly s3Sha256: (url: string) => Promise; +} + +function desiredImage(service: string, releaseVersion: string, digest: string): string { + if (!DIGEST_PATTERN.test(digest)) { + throw new InvalidPayloadError(`missing slim digest for ${service}:${releaseVersion}`); + } + return `${SLIM_IMAGE_PREFIX}${service}:${releaseVersion}@${digest}`; } -/** Additional release entries: `"": ""`. */ -function additionalEntryPattern(service: string, version?: string): RegExp { - const s = escapeRegExp(service); - const key = version === undefined ? `[^"]+` : escapeRegExp(version); - return new RegExp( - `"(${key})"(\\s*:\\s*)"(${escapeRegExp(SLIM_IMAGE_PREFIX)}${s}:[^"]+)"`, - version === undefined ? "g" : "", +/** + * Resolves `service`'s highest committed `-r` revision, pinned by content: the + * GHCR manifest digest, and every native target's archive and manifest sha256, cross-checked + * against the S3 mirror copy. See module docs for the five-step protocol. + */ +export async function resolveRevisionPin( + service: string, + upstream: string, + io: RevisionIo, +): Promise { + const tagPattern = new RegExp( + `^${escapeRegExp(service)}-${escapeRegExp(upstream)}-r(0|[1-9][0-9]*)$`, ); + let highest: number | undefined; + for (const tag of await io.listReleaseTags()) { + const match = tagPattern.exec(tag); + if (match === null) continue; + const revision = Number(match[1]); + if (highest === undefined || revision > highest) highest = revision; + } + if (highest === undefined) { + return { + status: "missing", + message: `${service}:${upstream} has no published slim-services revision.`, + }; + } + + const releaseVersion = `${upstream}-r${highest}`; + const checksums = await io.fetchChecksums(service, releaseVersion); + if (checksums === undefined) { + return { + status: "incomplete", + message: `${service}-${releaseVersion} has no SHA256SUMS asset.`, + }; + } + + const natives: Record = {}; + for (const target of NATIVE_TARGETS) { + const files = nativeFileNames(service, releaseVersion, target); + const archive = checksumFor(checksums, files.archive); + const manifest = checksumFor(checksums, files.manifest); + if (archive === undefined || manifest === undefined) { + return { + status: "incomplete", + message: `${service}-${releaseVersion} SHA256SUMS has no line for ${ + archive === undefined ? files.archive : files.manifest + }.`, + }; + } + natives[target] = { archive, manifest }; + } + + const digest = await io.imageDigest(service, releaseVersion); + if (digest === undefined || !DIGEST_PATTERN.test(digest)) { + return { + status: "lookup-failed", + message: `${SLIM_IMAGE_PREFIX}${service}:${releaseVersion} has no published manifest.`, + }; + } + + for (const target of NATIVE_TARGETS) { + const files = nativeFileNames(service, releaseVersion, target); + for (const part of ["archive", "manifest"] as const) { + const url = nativeObjectUrl(service, releaseVersion, files[part]); + const actual = await io.s3Sha256(url); + if (actual === undefined || actual !== natives[target]?.[part]) { + return { + status: "stale", + message: `S3 copy of ${releaseVersion} ${target} is stale; run the slim-services mirror backfill`, + }; + } + } + } + + return { + status: "resolved", + pin: { + upstreamVersion: upstream, + revision: highest, + image: desiredImage(service, releaseVersion, digest), + natives: natives as Record, + }, + }; } -function imageHasDigest(image: string): boolean { - return /@sha256:[0-9a-f]{64}$/.test(image); +/** Serializes a resolved pin into the object literal `Artifacts.ts` embeds, in catalog order. */ +function serializePin(pin: ResolvedPin): string { + const natives = NATIVE_TARGETS.map((target) => { + const native = pin.natives[target]; + return `"${target}": { archive: "${native.archive}", manifest: "${native.manifest}" }`; + }).join(", "); + return `{ upstreamVersion: "${pin.upstreamVersion}", revision: ${pin.revision}, image: "${pin.image}", natives: { ${natives} } }`; } -function desiredImage( - service: string, - version: string, - currentImage: string, - digest: string, -): string { - const tagged = `${SLIM_IMAGE_PREFIX}${service}:${version}`; - if (!imageHasDigest(currentImage)) return tagged; - if (!DIGEST_PATTERN.test(digest)) { - throw new InvalidPayloadError(`missing slim digest for ${service}:${version}`); +/** Matches `Artifacts.ts`'s emitted pin literal, in the exact shape `serializePin` writes. */ +function pinLiteralSource(): string { + const nativeEntries = NATIVE_TARGETS.map( + (target) => + `"${target}":\\s*\\{\\s*archive:\\s*"([0-9a-f]{64})"\\s*,\\s*manifest:\\s*"([0-9a-f]{64})"\\s*\\},?`, + ).join("\\s*"); + return `\\{\\s*upstreamVersion:\\s*"([^"]+)"\\s*,\\s*revision:\\s*(\\d+)\\s*,\\s*image:\\s*"([^"]+)"\\s*,\\s*natives:\\s*\\{\\s*${nativeEntries}\\s*\\},?\\s*\\}`; +} + +/** `placeholderPin("", "")`, B1's stand-in for an unresolved pin. */ +function placeholderCallSource(): string { + return `placeholderPin\\(\\s*"[a-z0-9-]+"\\s*,\\s*"([^"]+)"\\s*\\)`; +} + +interface PinSpan { + readonly start: number; + readonly end: number; + readonly version: string; +} + +/** Matches a pin expression (placeholder call or resolved literal) starting exactly at `index`. */ +function matchPinAt(source: string, index: number): PinSpan | undefined { + const placeholder = new RegExp(placeholderCallSource(), "y"); + placeholder.lastIndex = index; + const placeholderMatch = placeholder.exec(source); + if (placeholderMatch !== null) { + return { + start: index, + end: index + placeholderMatch[0].length, + version: placeholderMatch[1] ?? "", + }; + } + const literal = new RegExp(pinLiteralSource(), "y"); + literal.lastIndex = index; + const literalMatch = literal.exec(source); + if (literalMatch !== null) { + return { start: index, end: index + literalMatch[0].length, version: literalMatch[1] ?? "" }; } - return `${tagged}@${digest}`; + return undefined; +} + +/** Index right after the matching `)` for the `(` at `afterOpenParenIndex - 1`. */ +function scanToMatchingParen(source: string, afterOpenParenIndex: number): number { + let depth = 1; + let index = afterOpenParenIndex; + while (index < source.length && depth > 0) { + const ch = source[index]; + if (ch === '"' || ch === "'" || ch === "`") { + const quote = ch; + index++; + while (index < source.length && source[index] !== quote) { + if (source[index] === "\\") index++; + index++; + } + } else if (ch === "(") depth++; + else if (ch === ")") depth--; + index++; + } + return index; } type SelectedEntry = - | { readonly kind: "default" | "additional"; readonly version: string; readonly image: string } + | { readonly kind: "default" | "additional"; readonly version: string; readonly span: PinSpan } | { readonly kind: "unmodelled-service" } | { readonly kind: "unmodelled-release-line"; readonly known: ReadonlyArray }; -function selectEntry(source: string, service: string, version: string): SelectedEntry { - const defaultMatch = defaultEntryPattern(service).exec(source); - if (defaultMatch === null) return { kind: "unmodelled-service" }; +/** + * Locates `service`'s pin expression in `source`: the `definition("", , ...)` + * default pin, and, when `version` is given, whichever pin (default or additional) sits on its + * release line. With no `version`, returns the default pin unconditionally. + */ +function selectEntry(source: string, service: string, version: string | undefined): SelectedEntry { + const prefix = new RegExp(`definition\\(\\s*"${escapeRegExp(service)}"\\s*,\\s*`); + const prefixMatch = prefix.exec(source); + if (prefixMatch === null) return { kind: "unmodelled-service" }; + const pinIndex = prefixMatch.index + prefixMatch[0].length; + const defaultPin = matchPinAt(source, pinIndex); + if (defaultPin === undefined) return { kind: "unmodelled-service" }; + + if (version === undefined) { + return { kind: "default", version: defaultPin.version, span: defaultPin }; + } + + const openParenIndex = prefixMatch.index + prefixMatch[0].indexOf("("); + const callEnd = scanToMatchingParen(source, openParenIndex + 1); + + const additional: Array<{ version: string; span: PinSpan }> = []; + const keyPattern = /"([^"]+)"\s*:\s*/g; + keyPattern.lastIndex = defaultPin.end; + for ( + let keyMatch = keyPattern.exec(source); + keyMatch !== null; + keyMatch = keyPattern.exec(source) + ) { + if (keyMatch.index >= callEnd) break; + const valueStart = keyMatch.index + keyMatch[0].length; + const pin = matchPinAt(source, valueStart); + if (pin === undefined) { + keyPattern.lastIndex = valueStart; + continue; + } + additional.push({ version: pin.version, span: pin }); + keyPattern.lastIndex = pin.end; + } - const currentDefaultVersion = defaultMatch[2] ?? ""; - const currentDefaultImage = defaultMatch[4] ?? ""; - const additional = [...source.matchAll(additionalEntryPattern(service))].map((match) => ({ - version: match[1] ?? "", - image: match[3] ?? "", - })); const bumpsDefault = - additional.length === 0 || releaseLine(version) === releaseLine(currentDefaultVersion); + additional.length === 0 || releaseLine(version) === releaseLine(defaultPin.version); if (bumpsDefault) { - return { kind: "default", version: currentDefaultVersion, image: currentDefaultImage }; + return { kind: "default", version: defaultPin.version, span: defaultPin }; } - const sameLine = additional.find((entry) => releaseLine(entry.version) === releaseLine(version)); if (sameLine === undefined) { return { kind: "unmodelled-release-line", - known: [currentDefaultVersion, ...additional.map((entry) => entry.version)], + known: [defaultPin.version, ...additional.map((entry) => entry.version)], }; } - return { kind: "additional", version: sameLine.version, image: sameLine.image }; + return { kind: "additional", version: sameLine.version, span: sameLine.span }; } function skipReason(alias: string, version: string, entry: SelectedEntry): string | undefined { @@ -159,65 +360,35 @@ function slimVersions(dockerfile: string): ReadonlyMap { return versions; } -type PinResult = - | { - readonly kind: "updated"; - readonly source: string; - readonly previousVersion: string; - readonly target: "default" | "additional"; - } - | { readonly kind: "unchanged" }; +const normalizeText = (text: string): string => text.replace(/\s+/g, " ").trim(); -function pinService( +/** Writes `pin` over `entry`'s span in `source`, or returns `source` unchanged when it already matches. */ +function writePin( source: string, - service: string, - version: string, - digest: string | undefined, -): PinResult { - const entry = selectEntry(source, service, version); - if (entry.kind !== "default" && entry.kind !== "additional") { - throw new InvalidPayloadError(`${service} ${version} is not a catalog entry.`); - } - if (imageHasDigest(entry.image) && (digest === undefined || !DIGEST_PATTERN.test(digest))) { - throw new InvalidPayloadError(`missing slim digest for ${service}:${version}`); - } - const desired = desiredImage(service, version, entry.image, digest ?? ""); - if (entry.version === version && entry.image === desired) return { kind: "unchanged" }; - - if (entry.kind === "default") { - return { - kind: "updated", - source: source.replace( - defaultEntryPattern(service), - (_full, prefix: string, _version: string, mid: string, _image: string, suffix: string) => - `${prefix}${version}${mid}${desired}${suffix}`, - ), - previousVersion: entry.version, - target: "default", - }; - } - + entry: Extract, + pin: ResolvedPin, +): { readonly source: string; readonly changed: boolean } { + const desired = serializePin(pin); + const current = source.slice(entry.span.start, entry.span.end); + if (normalizeText(current) === normalizeText(desired)) return { source, changed: false }; return { - kind: "updated", - source: source.replace( - additionalEntryPattern(service, entry.version), - (_full, _key: string, separator: string) => `"${version}"${separator}"${desired}"`, - ), - previousVersion: entry.version, - target: "additional", + source: source.slice(0, entry.span.start) + desired + source.slice(entry.span.end), + changed: true, }; } /** - * Rewrites `catalog` from Dockerfile tags that differ from `baseDockerfile`. - * With no base, every slim tag is in scope. A pin that cannot be applied is - * reported in `skipped`. OrioleDB is the only non-blocking skip. + * Rewrites `catalog` from Dockerfile tags that differ from `baseDockerfile`, pinning each to its + * upstream's highest committed slim-services revision. With no base, every slim tag is in scope. + * A pin that cannot be applied is reported in `skipped`. OrioleDB and a Dependabot bump older + * than the current catalog pin are the non-blocking skips; the catalog is allowed to lead the + * Dockerfile until Dependabot catches up. */ export async function planArtifactCatalogUpdate(input: { readonly dockerfile: string; readonly baseDockerfile?: string; readonly catalog: string; - readonly publication: (service: string, version: string) => Promise; + readonly io: RevisionIo; }): Promise { let source = input.catalog; const updates: CatalogPinUpdate[] = []; @@ -254,112 +425,201 @@ export async function planArtifactCatalogUpdate(input: { skipped.push({ alias: from.alias, reason: `${pin.service} ${pin.version} is older than the catalog pin ${entry.version}.`, - blocking: true, + blocking: false, }); continue; } - if (entry.version === pin.version && !imageHasDigest(entry.image)) continue; - const release = await input.publication(pin.service, pin.version); - if (release.status !== "published") { - skipped.push({ - alias: from.alias, - reason: - release.status === "missing" - ? `${pin.service}:${pin.version} has no published slim image and native release.` - : `${pin.service}:${pin.version} publication check failed.`, - blocking: true, - }); - continue; - } - const digest = imageHasDigest(entry.image) ? release.digest : undefined; - if (imageHasDigest(entry.image) && (digest === undefined || !DIGEST_PATTERN.test(digest))) { - skipped.push({ - alias: from.alias, - reason: `${pin.service}:${pin.version} has no published slim manifest.`, - blocking: true, - }); + const resolution = await resolveRevisionPin(pin.service, pin.version, input.io); + if (resolution.status !== "resolved") { + skipped.push({ alias: from.alias, reason: resolution.message, blocking: true }); continue; } - const result = pinService(source, pin.service, pin.version, digest); - if (result.kind === "unchanged") continue; - source = result.source; + const written = writePin(source, entry, resolution.pin); + if (!written.changed) continue; + source = written.source; updates.push({ service: pin.service, version: pin.version, - previousVersion: result.previousVersion, - target: result.target, + revision: resolution.pin.revision, + previousVersion: entry.version, + target: entry.kind, }); } return { source, updates, skipped }; } -type Probe = "published" | "missing" | "lookup-failed"; +export interface CatalogRefreshResult { + readonly source: string; + readonly update?: CatalogPinUpdate; +} -async function probeManifest(reference: string): Promise<{ probe: Probe; digest?: string }> { +/** + * Refreshes one catalog entry to the highest committed revision of `upstream` (or, when omitted, + * of its currently pinned upstream version). Pure aside from `io`: callers own reading and + * writing `Artifacts.ts`. This is what the manual CLI mode calls, and what a later hotfix pickup + * workflow can call directly with `{service, upstream_version}` from its dispatch payload. + */ +export async function refreshCatalogPin(input: { + readonly catalog: string; + readonly service: string; + readonly upstream?: string; + readonly io: RevisionIo; +}): Promise { + const entry = selectEntry(input.catalog, input.service, input.upstream); + if (entry.kind === "unmodelled-service") { + throw new InvalidPayloadError(`${CATALOG_PATH} has no slim entry for ${input.service}.`); + } + if (entry.kind === "unmodelled-release-line") { + throw new InvalidPayloadError( + `${input.service} ${input.upstream ?? ""} is not on a release line ${CATALOG_PATH} carries (${entry.known.join(", ")}).`, + ); + } + const upstream = input.upstream ?? entry.version; + const resolution = await resolveRevisionPin(input.service, upstream, input.io); + if (resolution.status !== "resolved") { + throw new InvalidPayloadError(resolution.message); + } + const written = writePin(input.catalog, entry, resolution.pin); + if (!written.changed) return { source: input.catalog }; + return { + source: written.source, + update: { + service: input.service, + version: upstream, + revision: resolution.pin.revision, + previousVersion: entry.version, + target: entry.kind, + }, + }; +} + +function githubHeaders(): Record { + const headers: Record = { + Accept: "application/vnd.github+json", + "User-Agent": "supabase-cli-catalog-sync", + }; + const token = process.env.GITHUB_TOKEN; + if (token !== undefined && token !== "") headers.Authorization = `Bearer ${token}`; + return headers; +} + +async function listReleaseTags(): Promise> { + const tags: string[] = []; + for (let page = 1; ; page++) { + const response = await fetch(`${RELEASES_API}?per_page=100&page=${page}`, { + headers: githubHeaders(), + }); + if (!response.ok) { + throw new InvalidPayloadError( + `slim-services releases list failed (HTTP ${response.status}).`, + ); + } + const batch: unknown = await response.json(); + if (!Array.isArray(batch)) { + throw new InvalidPayloadError("Malformed slim-services releases response."); + } + for (const item of batch) { + const tagName = (item as { tag_name?: unknown } | null)?.tag_name; + if (typeof tagName === "string") tags.push(tagName); + } + if (batch.length < 100) break; + } + return tags; +} + +async function fetchChecksums( + service: string, + releaseVersion: string, +): Promise { + const response = await fetch(`${RELEASE_DOWNLOAD_BASE}/${service}-${releaseVersion}/SHA256SUMS`); + if (response.status === 404) return undefined; + if (!response.ok) { + throw new InvalidPayloadError( + `SHA256SUMS download failed for ${service}-${releaseVersion} (HTTP ${response.status}).`, + ); + } + return response.text(); +} + +async function imageDigest(service: string, releaseVersion: string): Promise { + const reference = `${SLIM_IMAGE_PREFIX}${service}:${releaseVersion}`; const proc = Bun.spawn(["regctl", "manifest", "head", reference], { stdout: "pipe", stderr: "pipe", }); - const [stdout, stderr, exit] = await Promise.all([ + const [stdout, , exit] = await Promise.all([ new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited, ]); const digest = stdout.trim(); - if (exit === 0 && DIGEST_PATTERN.test(digest)) return { probe: "published", digest }; - const detail = stderr.toLowerCase(); - if ( - detail.includes("manifest unknown") || - detail.includes("name unknown") || - detail.includes("not found") || - detail.includes("404") - ) { - return { probe: "missing" }; + return exit === 0 && DIGEST_PATTERN.test(digest) ? digest : undefined; +} + +async function s3Sha256(url: string): Promise { + const response = await fetch(url); + if (!response.ok) return undefined; + const hasher = new Bun.CryptoHasher("sha256"); + hasher.update(new Uint8Array(await response.arrayBuffer())); + return hasher.digest("hex"); +} + +function defaultRevisionIo(): RevisionIo { + return { listReleaseTags, fetchChecksums, imageDigest, s3Sha256 }; +} + +function parseFlags(argv: ReadonlyArray): ReadonlyMap { + const flags = new Map(); + for (let index = 0; index < argv.length; index++) { + const arg = argv[index]; + if (arg === undefined || !arg.startsWith("--")) { + throw new InvalidPayloadError(`unexpected argument '${arg ?? ""}'`); + } + const key = arg.slice(2); + const value = argv[index + 1]; + if (value === undefined || value.startsWith("--")) { + throw new InvalidPayloadError(`missing value for --${key}`); + } + flags.set(key, value); + index++; } - const message = stderr.trim(); - console.log( - `::warning ::${reference} publication check failed${message === "" ? "" : `: ${message}`}`, - ); - return { probe: "lookup-failed" }; + return flags; } -async function probeNativeRelease(service: string, version: string): Promise { - const tag = `${service}-${version}`; - const headers: Record = { - Accept: "application/vnd.github+json", - "User-Agent": "supabase-cli-catalog-sync", - }; - const token = process.env.GITHUB_TOKEN; - if (token !== undefined && token !== "") headers.Authorization = `Bearer ${token}`; - try { - const response = await fetch(`${NATIVE_RELEASES}/${encodeURIComponent(tag)}`, { headers }); - if (response.status === 200) return "published"; - if (response.status === 404) return "missing"; - console.log(`::warning ::${tag} native release check returned HTTP ${response.status}.`); - return "lookup-failed"; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.log(`::warning ::${tag} native release check failed: ${message}`); - return "lookup-failed"; - } -} - -async function lookupPublication(service: string, version: string): Promise { - const reference = `${SLIM_IMAGE_PREFIX}${service}:${version}`; - const [manifest, native] = await Promise.all([ - probeManifest(reference), - probeNativeRelease(service, version), - ]); - if (manifest.probe === "lookup-failed" || native === "lookup-failed") { - return { status: "lookup-failed" }; +async function runManual(argv: ReadonlyArray): Promise { + const flags = parseFlags(argv); + const service = flags.get("service"); + if (service === undefined) { + throw new InvalidPayloadError( + "Usage: sync-artifacts-catalog.ts --service [--upstream ]", + ); } - if (manifest.probe === "missing" || native === "missing") return { status: "missing" }; - return { status: "published", digest: manifest.digest }; + const catalogPath = CATALOG_PATH; + const catalog = await Bun.file(catalogPath).text(); + const result = await refreshCatalogPin({ + catalog, + service, + upstream: flags.get("upstream"), + io: defaultRevisionIo(), + }); + if (result.update === undefined) { + console.log(`${service} already pins the highest committed revision.`); + return; + } + await Bun.write(catalogPath, result.source); + console.log( + `Pinned ${service} ${result.update.target} ${result.update.previousVersion} -> ${result.update.version} r${result.update.revision}.`, + ); } async function main(argv: ReadonlyArray): Promise { + if (argv[0]?.startsWith("--") === true) { + await runManual(argv); + return; + } + const [dockerfilePath = DOCKERFILE_PATH, catalogPath = CATALOG_PATH, baseDockerfilePath] = argv; const dockerfile = await Bun.file(dockerfilePath).text(); const catalog = await Bun.file(catalogPath).text(); @@ -369,7 +629,7 @@ async function main(argv: ReadonlyArray): Promise { dockerfile, baseDockerfile, catalog, - publication: lookupPublication, + io: defaultRevisionIo(), }); for (const skip of plan.skipped) { console.log(`::warning ::Left ${skip.alias} unchanged: ${skip.reason}`); @@ -385,7 +645,7 @@ async function main(argv: ReadonlyArray): Promise { } for (const update of plan.updates) { console.log( - `Pinned ${update.service} ${update.target} ${update.previousVersion} -> ${update.version}.`, + `Pinned ${update.service} ${update.target} ${update.previousVersion} -> ${update.version} r${update.revision}.`, ); } } From 0ea358f1c227ba6857e90d565cff7cd1d2b7b9e0 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 29 Sep 2026 10:10:40 +0200 Subject: [PATCH 03/45] fix(ci): parse formatter-wrapped catalog pins --- .../scripts/sync-artifacts-catalog.test.ts | 161 ++++++++++++++++++ .github/scripts/sync-artifacts-catalog.ts | 92 +++++----- 2 files changed, 208 insertions(+), 45 deletions(-) diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index 2b40d7e975..f309e1c8fd 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -1,4 +1,7 @@ import { describe, expect, test } from "bun:test"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { parseDockerfileServiceImages } from "../../apps/cli/src/shared/services/parse-dockerfile-service-images.ts"; import { isOrioleImage, slimCatalogPin } from "../../apps/cli/src/shared/services/slim-images.ts"; @@ -46,6 +49,28 @@ function matchingS3( return async (url) => byUrl.get(url); } +/** + * Runs the repo's pinned `oxfmt` binary over `source`, the way `sync-artifacts-catalog.yml` + * formats the catalog after every write, so parsing tests exercise real formatter output + * (line-wrapping, trailing commas) instead of a hand-written single-line literal. + */ +async function formatWithOxfmt(source: string): Promise { + const dir = await mkdtemp(join(tmpdir(), "sync-artifacts-catalog-")); + const filePath = join(dir, "Artifacts.ts"); + try { + await writeFile(filePath, source); + const proc = Bun.spawn(["node_modules/.bin/oxfmt", "--config", ".oxfmtrc.json", filePath], { + stdout: "pipe", + stderr: "pipe", + }); + const [stderr, exitCode] = await Promise.all([new Response(proc.stderr).text(), proc.exited]); + if (exitCode !== 0) throw new Error(`oxfmt failed: ${stderr}`); + return await readFile(filePath, "utf8"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + const fixture = `const workloadCatalog = { database: definition( "postgres", @@ -268,6 +293,102 @@ describe("refreshCatalogPin", () => { `image: "ghcr.io/supabase/cli/analytics:v1.50.9-r1@${digest("h")}"`, ); }); + + test("a resolved pin survives real formatting and can be refreshed again", async () => { + const first = nativeDigests("i"); + const firstIo: RevisionIo = { + listReleaseTags: async () => ["postgrest-v16.2-r0"], + fetchChecksums: async () => checksumsFor("postgrest", "v16.2-r0", first), + imageDigest: async () => digest("j"), + s3Sha256: matchingS3("postgrest", "v16.2-r0", first), + }; + const written = await refreshCatalogPin({ + catalog: fixture, + service: "postgrest", + io: firstIo, + }); + expect(written.update?.revision).toBe(0); + + const formatted = await formatWithOxfmt(written.source); + // The formatter actually wrapped the literal onto several lines with trailing commas; + // otherwise this test would not be exercising what it claims to. + expect(formatted.split("\n").length).toBeGreaterThan(written.source.split("\n").length); + + const second = nativeDigests("k"); + const secondIo: RevisionIo = { + listReleaseTags: async () => ["postgrest-v16.2-r0", "postgrest-v16.2-r1"], + fetchChecksums: async () => checksumsFor("postgrest", "v16.2-r1", second), + imageDigest: async () => digest("l"), + s3Sha256: matchingS3("postgrest", "v16.2-r1", second), + }; + const refreshed = await refreshCatalogPin({ + catalog: formatted, + service: "postgrest", + io: secondIo, + }); + + expect(refreshed.update).toEqual({ + service: "postgrest", + version: "v16.2", + revision: 1, + previousVersion: "v16.2", + target: "default", + }); + expect(refreshed.source).toContain("revision: 1"); + expect(refreshed.source).toContain(second["darwin-arm64"].archive); + }); + + test("refreshes the Postgres 15 additional pin, including after formatting", async () => { + const digests = nativeDigests("m"); + const io: RevisionIo = { + listReleaseTags: async () => ["postgres-15.14.1.168-r0"], + fetchChecksums: async () => checksumsFor("postgres", "15.14.1.168-r0", digests), + imageDigest: async () => digest("n"), + s3Sha256: matchingS3("postgres", "15.14.1.168-r0", digests), + }; + + const written = await refreshCatalogPin({ + catalog: fixture, + service: "postgres", + upstream: "15.14.1.168", + io, + }); + expect(written.update).toEqual({ + service: "postgres", + version: "15.14.1.168", + revision: 0, + previousVersion: "15.14.1.168", + target: "additional", + }); + // The default (17.x) postgres line is untouched. + expect(written.source).toContain('placeholderPin("postgres", "17.6.1.168")'); + + const formatted = await formatWithOxfmt(written.source); + const nextDigests = nativeDigests("o"); + const nextIo: RevisionIo = { + listReleaseTags: async () => ["postgres-15.14.1.168-r0", "postgres-15.14.1.168-r1"], + fetchChecksums: async () => checksumsFor("postgres", "15.14.1.168-r1", nextDigests), + imageDigest: async () => digest("p"), + s3Sha256: matchingS3("postgres", "15.14.1.168-r1", nextDigests), + }; + + const refreshed = await refreshCatalogPin({ + catalog: formatted, + service: "postgres", + upstream: "15.14.1.168", + io: nextIo, + }); + + expect(refreshed.update).toEqual({ + service: "postgres", + version: "15.14.1.168", + revision: 1, + previousVersion: "15.14.1.168", + target: "additional", + }); + expect(refreshed.source).toContain('placeholderPin("postgres", "17.6.1.168")'); + expect(refreshed.source).toContain(nextDigests["linux-arm64"].manifest); + }); }); describe("against the real catalog", () => { @@ -319,4 +440,44 @@ describe("against the real catalog", () => { expect(plan.skipped.filter((skip) => skip.blocking)).toEqual([]); }); + + test("a real catalog entry survives real formatting and can be refreshed again", async () => { + const catalog = await Bun.file(CATALOG_PATH).text(); + + const first = nativeDigests("q"); + const firstIo: RevisionIo = { + listReleaseTags: async () => ["auth-v2.196.0-r0"], + fetchChecksums: async () => checksumsFor("auth", "v2.196.0-r0", first), + imageDigest: async () => digest("r"), + s3Sha256: matchingS3("auth", "v2.196.0-r0", first), + }; + const written = await refreshCatalogPin({ catalog, service: "auth", io: firstIo }); + expect(written.update?.revision).toBe(0); + + const formatted = await formatWithOxfmt(written.source); + expect(formatted).toContain('upstreamVersion: "v2.196.0"'); + + const second = nativeDigests("s"); + const secondIo: RevisionIo = { + listReleaseTags: async () => ["auth-v2.196.0-r0", "auth-v2.196.0-r1"], + fetchChecksums: async () => checksumsFor("auth", "v2.196.0-r1", second), + imageDigest: async () => digest("t"), + s3Sha256: matchingS3("auth", "v2.196.0-r1", second), + }; + const refreshed = await refreshCatalogPin({ + catalog: formatted, + service: "auth", + io: secondIo, + }); + + expect(refreshed.update).toEqual({ + service: "auth", + version: "v2.196.0", + revision: 1, + previousVersion: "v2.196.0", + target: "default", + }); + expect(refreshed.source).toContain("revision: 1"); + expect(refreshed.source).toContain(second["darwin-arm64"].manifest); + }); }); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index 51a5e10025..f363df9248 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -221,52 +221,21 @@ function serializePin(pin: ResolvedPin): string { return `{ upstreamVersion: "${pin.upstreamVersion}", revision: ${pin.revision}, image: "${pin.image}", natives: { ${natives} } }`; } -/** Matches `Artifacts.ts`'s emitted pin literal, in the exact shape `serializePin` writes. */ -function pinLiteralSource(): string { - const nativeEntries = NATIVE_TARGETS.map( - (target) => - `"${target}":\\s*\\{\\s*archive:\\s*"([0-9a-f]{64})"\\s*,\\s*manifest:\\s*"([0-9a-f]{64})"\\s*\\},?`, - ).join("\\s*"); - return `\\{\\s*upstreamVersion:\\s*"([^"]+)"\\s*,\\s*revision:\\s*(\\d+)\\s*,\\s*image:\\s*"([^"]+)"\\s*,\\s*natives:\\s*\\{\\s*${nativeEntries}\\s*\\},?\\s*\\}`; -} - -/** `placeholderPin("", "")`, B1's stand-in for an unresolved pin. */ -function placeholderCallSource(): string { - return `placeholderPin\\(\\s*"[a-z0-9-]+"\\s*,\\s*"([^"]+)"\\s*\\)`; -} - interface PinSpan { readonly start: number; readonly end: number; readonly version: string; } -/** Matches a pin expression (placeholder call or resolved literal) starting exactly at `index`. */ -function matchPinAt(source: string, index: number): PinSpan | undefined { - const placeholder = new RegExp(placeholderCallSource(), "y"); - placeholder.lastIndex = index; - const placeholderMatch = placeholder.exec(source); - if (placeholderMatch !== null) { - return { - start: index, - end: index + placeholderMatch[0].length, - version: placeholderMatch[1] ?? "", - }; - } - const literal = new RegExp(pinLiteralSource(), "y"); - literal.lastIndex = index; - const literalMatch = literal.exec(source); - if (literalMatch !== null) { - return { start: index, end: index + literalMatch[0].length, version: literalMatch[1] ?? "" }; - } - return undefined; -} - -/** Index right after the matching `)` for the `(` at `afterOpenParenIndex - 1`. */ -function scanToMatchingParen(source: string, afterOpenParenIndex: number): number { - let depth = 1; - let index = afterOpenParenIndex; - while (index < source.length && depth > 0) { +/** + * Index right after the matching close-bracket for the open-bracket character at `openIndex` + * (which must itself be `open`). Skips string contents, so a formatter's line-wrapping, trailing + * commas, or reordered properties never confuse the boundary — only bracket balance matters. + */ +function scanBalanced(source: string, openIndex: number, open: string, close: string): number { + let depth = 0; + let index = openIndex; + for (; index < source.length; index++) { const ch = source[index]; if (ch === '"' || ch === "'" || ch === "`") { const quote = ch; @@ -275,11 +244,44 @@ function scanToMatchingParen(source: string, afterOpenParenIndex: number): numbe if (source[index] === "\\") index++; index++; } - } else if (ch === "(") depth++; - else if (ch === ")") depth--; - index++; + continue; + } + if (ch === open) depth++; + else if (ch === close) { + depth--; + if (depth === 0) return index + 1; + } } - return index; + throw new InvalidPayloadError(`unterminated '${open}' while parsing ${CATALOG_PATH}`); +} + +const PLACEHOLDER_PREFIX = /placeholderPin\s*\(/y; +/** Loosely finds `placeholderPin`'s second (version) argument anywhere inside its call text. */ +const PLACEHOLDER_VERSION = /"[a-z0-9-]+"\s*,\s*"([^"]+)"/; +/** Loosely finds `upstreamVersion` anywhere inside a resolved pin literal's text. */ +const PIN_UPSTREAM_VERSION = /upstreamVersion:\s*"([^"]+)"/; + +/** + * Matches a pin expression (B1's `placeholderPin(...)` call, or a resolved `ArtifactPin` object + * literal) starting exactly at `index`. The span is found by bracket balance, then the version is + * pulled out with a loose field search — so a formatter's whitespace, line breaks, trailing + * commas, or property order never break matching, only the two literal shapes themselves would. + */ +function matchPinAt(source: string, index: number): PinSpan | undefined { + PLACEHOLDER_PREFIX.lastIndex = index; + const placeholderPrefix = PLACEHOLDER_PREFIX.exec(source); + if (placeholderPrefix !== null) { + const openParen = index + placeholderPrefix[0].length - 1; + const end = scanBalanced(source, openParen, "(", ")"); + const version = PLACEHOLDER_VERSION.exec(source.slice(index, end))?.[1]; + return version === undefined ? undefined : { start: index, end, version }; + } + if (source[index] === "{") { + const end = scanBalanced(source, index, "{", "}"); + const version = PIN_UPSTREAM_VERSION.exec(source.slice(index, end))?.[1]; + return version === undefined ? undefined : { start: index, end, version }; + } + return undefined; } type SelectedEntry = @@ -305,7 +307,7 @@ function selectEntry(source: string, service: string, version: string | undefine } const openParenIndex = prefixMatch.index + prefixMatch[0].indexOf("("); - const callEnd = scanToMatchingParen(source, openParenIndex + 1); + const callEnd = scanBalanced(source, openParenIndex, "(", ")"); const additional: Array<{ version: string; span: PinSpan }> = []; const keyPattern = /"([^"]+)"\s*:\s*/g; From 95d03405e1dcc09717d9eabe71a408cef133b31b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 29 Sep 2026 10:19:57 +0200 Subject: [PATCH 04/45] feat(ci): open catalog PRs for slim hotfix revisions --- .../scripts/sync-artifacts-catalog.test.ts | 81 ++++++++ .github/scripts/sync-artifacts-catalog.ts | 139 ++++++++++++-- .github/workflows/slim-release-published.yml | 176 ++++++++++++++++++ 3 files changed, 377 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/slim-release-published.yml diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index f309e1c8fd..6893cf5e01 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -8,6 +8,7 @@ import { isOrioleImage, slimCatalogPin } from "../../apps/cli/src/shared/service import { InvalidPayloadError, nativeFileNames, nativeObjectUrl } from "./slim-mirror-payload.ts"; import { CATALOG_PATH, + findHotfixMatches, planArtifactCatalogUpdate, refreshCatalogPin, type RevisionIo, @@ -89,6 +90,86 @@ const fixture = `const workloadCatalog = { }; `; +/** A catalog whose Postgres 15 additional pin is already resolved (not a `placeholderPin`), at `r3`. */ +const resolvedFixture = fixture.replace( + '{ "15.14.1.168": placeholderPin("postgres", "15.14.1.168") },', + '{ "15.14.1.168": { upstreamVersion: "15.14.1.168", revision: 3, image: "ghcr.io/supabase/cli/postgres:15.14.1.168-r3@sha256:0000000000000000000000000000000000000000000000000000000000000", natives: {} } },', +); + +describe("findHotfixMatches", () => { + test("selects the Postgres 15 entry for a higher revision of the same upstream", () => { + const matches = findHotfixMatches(resolvedFixture, { + service: "postgres", + upstream_version: "15.14.1.168", + revision: 4, + release_version: "15.14.1.168-r4", + }); + + expect(matches).toEqual([{ target: "additional", currentRevision: 3 }]); + }); + + test("ignores an equal or lower revision of the same upstream", () => { + for (const revision of [3, 2]) { + const matches = findHotfixMatches(resolvedFixture, { + service: "postgres", + upstream_version: "15.14.1.168", + revision, + release_version: `15.14.1.168-r${revision}`, + }); + expect(matches).toEqual([]); + } + }); + + test("ignores a higher revision of a different upstream version", () => { + const matches = findHotfixMatches(resolvedFixture, { + service: "postgres", + upstream_version: "15.14.1.999", + revision: 4, + release_version: "15.14.1.999-r4", + }); + + expect(matches).toEqual([]); + }); + + test("matches the default pin too, once it is resolved", () => { + const withResolvedDefault = resolvedFixture.replace( + 'placeholderPin("postgres", "17.6.1.168")', + '{ upstreamVersion: "17.6.1.168", revision: 1, image: "ghcr.io/supabase/cli/postgres:17.6.1.168-r1@sha256:1111111111111111111111111111111111111111111111111111111111111", natives: {} }', + ); + const matches = findHotfixMatches(withResolvedDefault, { + service: "postgres", + upstream_version: "17.6.1.168", + revision: 2, + release_version: "17.6.1.168-r2", + }); + + expect(matches).toEqual([{ target: "default", currentRevision: 1 }]); + }); + + test("treats an unresolved placeholderPin as always eligible for a hotfix", () => { + const matches = findHotfixMatches(fixture, { + service: "postgrest", + upstream_version: "v16.2", + revision: 0, + release_version: "v16.2-r0", + }); + + expect(matches).toEqual([{ target: "default", currentRevision: -1 }]); + }); + + test("survives real oxfmt formatting", async () => { + const formatted = await formatWithOxfmt(resolvedFixture); + const matches = findHotfixMatches(formatted, { + service: "postgres", + upstream_version: "15.14.1.168", + revision: 4, + release_version: "15.14.1.168-r4", + }); + + expect(matches).toEqual([{ target: "additional", currentRevision: 3 }]); + }); +}); + describe("planArtifactCatalogUpdate", () => { test("pins to the highest committed revision, writing every native target's digests", async () => { const digests = nativeDigests("c"); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index f363df9248..80a9c9a96d 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -14,6 +14,13 @@ * when `--upstream` is omitted). * * bun .github/scripts/sync-artifacts-catalog.ts --service [--upstream ] + * + * Hotfix-matches mode (used by the `slim-release-published` dispatch + * workflow): prints, as JSON, which of a service's catalog pins a hotfix + * revision should refresh. + * + * bun .github/scripts/sync-artifacts-catalog.ts hotfix-matches --service \ + * --upstream --revision */ import { parseDockerfileServiceImages } from "../../apps/cli/src/shared/services/parse-dockerfile-service-images.ts"; @@ -260,6 +267,8 @@ const PLACEHOLDER_PREFIX = /placeholderPin\s*\(/y; const PLACEHOLDER_VERSION = /"[a-z0-9-]+"\s*,\s*"([^"]+)"/; /** Loosely finds `upstreamVersion` anywhere inside a resolved pin literal's text. */ const PIN_UPSTREAM_VERSION = /upstreamVersion:\s*"([^"]+)"/; +/** Loosely finds `revision` anywhere inside a resolved pin literal's text. Absent for a `placeholderPin`. */ +const PIN_REVISION = /revision:\s*(\d+)/; /** * Matches a pin expression (B1's `placeholderPin(...)` call, or a resolved `ArtifactPin` object @@ -284,32 +293,28 @@ function matchPinAt(source: string, index: number): PinSpan | undefined { return undefined; } -type SelectedEntry = - | { readonly kind: "default" | "additional"; readonly version: string; readonly span: PinSpan } - | { readonly kind: "unmodelled-service" } - | { readonly kind: "unmodelled-release-line"; readonly known: ReadonlyArray }; +interface ServicePins { + readonly defaultPin: PinSpan; + readonly additional: ReadonlyArray; +} /** - * Locates `service`'s pin expression in `source`: the `definition("", , ...)` - * default pin, and, when `version` is given, whichever pin (default or additional) sits on its - * release line. With no `version`, returns the default pin unconditionally. + * Finds every pin expression `service` carries in `source`: the `definition("", , + * ...)` default pin, plus any additional (release-line-keyed) pins in its trailing object + * argument. Both `selectEntry` and `findHotfixMatches` build on this single traversal. */ -function selectEntry(source: string, service: string, version: string | undefined): SelectedEntry { +function collectServicePins(source: string, service: string): ServicePins | undefined { const prefix = new RegExp(`definition\\(\\s*"${escapeRegExp(service)}"\\s*,\\s*`); const prefixMatch = prefix.exec(source); - if (prefixMatch === null) return { kind: "unmodelled-service" }; + if (prefixMatch === null) return undefined; const pinIndex = prefixMatch.index + prefixMatch[0].length; const defaultPin = matchPinAt(source, pinIndex); - if (defaultPin === undefined) return { kind: "unmodelled-service" }; - - if (version === undefined) { - return { kind: "default", version: defaultPin.version, span: defaultPin }; - } + if (defaultPin === undefined) return undefined; const openParenIndex = prefixMatch.index + prefixMatch[0].indexOf("("); const callEnd = scanBalanced(source, openParenIndex, "(", ")"); - const additional: Array<{ version: string; span: PinSpan }> = []; + const additional: PinSpan[] = []; const keyPattern = /"([^"]+)"\s*:\s*/g; keyPattern.lastIndex = defaultPin.end; for ( @@ -324,23 +329,91 @@ function selectEntry(source: string, service: string, version: string | undefine keyPattern.lastIndex = valueStart; continue; } - additional.push({ version: pin.version, span: pin }); + additional.push(pin); keyPattern.lastIndex = pin.end; } + return { defaultPin, additional }; +} + +type SelectedEntry = + | { readonly kind: "default" | "additional"; readonly version: string; readonly span: PinSpan } + | { readonly kind: "unmodelled-service" } + | { readonly kind: "unmodelled-release-line"; readonly known: ReadonlyArray }; + +/** + * Locates `service`'s pin expression in `source`: the `definition("", , ...)` + * default pin, and, when `version` is given, whichever pin (default or additional) sits on its + * release line. With no `version`, returns the default pin unconditionally. + */ +function selectEntry(source: string, service: string, version: string | undefined): SelectedEntry { + const pins = collectServicePins(source, service); + if (pins === undefined) return { kind: "unmodelled-service" }; + const { defaultPin, additional } = pins; + + if (version === undefined) { + return { kind: "default", version: defaultPin.version, span: defaultPin }; + } + const bumpsDefault = additional.length === 0 || releaseLine(version) === releaseLine(defaultPin.version); if (bumpsDefault) { return { kind: "default", version: defaultPin.version, span: defaultPin }; } - const sameLine = additional.find((entry) => releaseLine(entry.version) === releaseLine(version)); + const sameLine = additional.find((pin) => releaseLine(pin.version) === releaseLine(version)); if (sameLine === undefined) { return { kind: "unmodelled-release-line", - known: [defaultPin.version, ...additional.map((entry) => entry.version)], + known: [defaultPin.version, ...additional.map((pin) => pin.version)], }; } - return { kind: "additional", version: sameLine.version, span: sameLine.span }; + return { kind: "additional", version: sameLine.version, span: sameLine }; +} + +/** The `{service, upstream_version, revision, release_version}` payload a `slim-release-published` dispatch carries. */ +export interface SlimReleasePublishedPayload { + readonly service: string; + readonly upstream_version: string; + readonly revision: number; + readonly release_version: string; +} + +export interface HotfixMatch { + /** Which of `service`'s catalog pins (default, or one of the additional release lines) matched. */ + readonly target: "default" | "additional"; + /** The revision currently pinned; `-1` when the entry is still an unresolved `placeholderPin`. */ + readonly currentRevision: number; +} + +/** + * Every pin `payload.service` carries in `catalog` (default or additional) whose + * `upstreamVersion` equals `payload.upstream_version` and whose currently pinned revision is + * lower than `payload.revision` — i.e. the entries a `slim-release-published` hotfix dispatch + * should refresh. Pure: callers own running the actual refresh and PR flow. + */ +export function findHotfixMatches( + catalog: string, + payload: SlimReleasePublishedPayload, +): ReadonlyArray { + const pins = collectServicePins(catalog, payload.service); + if (pins === undefined) return []; + + const candidates: Array<{ target: "default" | "additional"; span: PinSpan }> = [ + { target: "default", span: pins.defaultPin }, + ...pins.additional.map((span) => ({ target: "additional" as const, span })), + ]; + + const matches: HotfixMatch[] = []; + for (const candidate of candidates) { + if (candidate.span.version !== payload.upstream_version) continue; + const text = catalog.slice(candidate.span.start, candidate.span.end); + const revisionMatch = PIN_REVISION.exec(text); + const currentRevision = revisionMatch === null ? -1 : Number(revisionMatch[1]); + if (currentRevision < payload.revision) { + matches.push({ target: candidate.target, currentRevision }); + } + } + return matches; } function skipReason(alias: string, version: string, entry: SelectedEntry): string | undefined { @@ -616,7 +689,35 @@ async function runManual(argv: ReadonlyArray): Promise { ); } +async function runHotfixMatches(argv: ReadonlyArray): Promise { + const flags = parseFlags(argv); + const service = flags.get("service"); + const upstream = flags.get("upstream"); + const revisionFlag = flags.get("revision"); + if (service === undefined || upstream === undefined || revisionFlag === undefined) { + throw new InvalidPayloadError( + "Usage: sync-artifacts-catalog.ts hotfix-matches --service --upstream --revision ", + ); + } + if (!/^(0|[1-9][0-9]*)$/.test(revisionFlag)) { + throw new InvalidPayloadError(`invalid --revision '${revisionFlag}'`); + } + const revision = Number(revisionFlag); + const catalog = await Bun.file(CATALOG_PATH).text(); + const matches = findHotfixMatches(catalog, { + service, + upstream_version: upstream, + revision, + release_version: `${upstream}-r${revision}`, + }); + console.log(JSON.stringify(matches)); +} + async function main(argv: ReadonlyArray): Promise { + if (argv[0] === "hotfix-matches") { + await runHotfixMatches(argv.slice(1)); + return; + } if (argv[0]?.startsWith("--") === true) { await runManual(argv); return; diff --git a/.github/workflows/slim-release-published.yml b/.github/workflows/slim-release-published.yml new file mode 100644 index 0000000000..90261fe6b4 --- /dev/null +++ b/.github/workflows/slim-release-published.yml @@ -0,0 +1,176 @@ +name: Slim Release Published + +# supabase/slim-services sends this dispatch after it mints an immutable release +# `--r`. When that revision is a hotfix of an upstream +# version the workload catalog (packages/stack/src/Artifacts.ts) already pins, this refreshes the +# pin and opens a pull request. +# +# The payload arrives with whatever authority holds the dispatch token, so it is treated as +# untrusted: fields are pattern- and shape-checked before use, and never interpolated directly +# into a `run:` script (only passed through `env:`). The matching logic itself lives in +# `.github/scripts/sync-artifacts-catalog.ts`'s `findHotfixMatches`, which is unit-tested; this +# workflow only wires it to a PR. + +on: + repository_dispatch: + types: + - slim-release-published + +permissions: + contents: read + +concurrency: + group: slim-release-published-${{ github.event.client_payload.service }}-${{ github.event.client_payload.upstream_version }} + cancel-in-progress: false + +jobs: + pickup: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: .bun-version + + - name: Validate payload + id: validate + env: + SERVICE: ${{ github.event.client_payload.service }} + UPSTREAM_VERSION: ${{ github.event.client_payload.upstream_version }} + REVISION: ${{ github.event.client_payload.revision }} + RELEASE_VERSION: ${{ github.event.client_payload.release_version }} + run: | + set -euo pipefail + if ! [[ "$SERVICE" =~ ^[a-z0-9-]+$ ]]; then + echo "::error ::invalid service '${SERVICE}'" + exit 1 + fi + if [ -z "$UPSTREAM_VERSION" ]; then + echo "::error ::missing upstream_version" + exit 1 + fi + if ! [[ "$REVISION" =~ ^(0|[1-9][0-9]*)$ ]]; then + echo "::error ::invalid revision '${REVISION}'" + exit 1 + fi + expected="${UPSTREAM_VERSION}-r${REVISION}" + if [ "$RELEASE_VERSION" != "$expected" ]; then + echo "::error ::release_version '${RELEASE_VERSION}' does not match derived '${expected}'" + exit 1 + fi + { + echo "service=${SERVICE}" + echo "upstream_version=${UPSTREAM_VERSION}" + echo "revision=${REVISION}" + echo "release_version=${RELEASE_VERSION}" + } >>"$GITHUB_OUTPUT" + + - name: Find hotfix matches + id: matches + env: + SERVICE: ${{ steps.validate.outputs.service }} + UPSTREAM_VERSION: ${{ steps.validate.outputs.upstream_version }} + REVISION: ${{ steps.validate.outputs.revision }} + run: | + set -euo pipefail + result="$(bun .github/scripts/sync-artifacts-catalog.ts hotfix-matches \ + --service "$SERVICE" --upstream "$UPSTREAM_VERSION" --revision "$REVISION")" + count="$(bun -e 'console.log(JSON.parse(await Bun.stdin.text()).length)' <<<"$result")" + echo "count=${count}" >>"$GITHUB_OUTPUT" + + - name: No matching catalog pin + if: steps.matches.outputs.count == '0' + env: + SERVICE: ${{ steps.validate.outputs.service }} + RELEASE_VERSION: ${{ steps.validate.outputs.release_version }} + run: echo "No catalog pin for ${SERVICE} needs ${RELEASE_VERSION}; nothing to do." + + - name: Install toolchains + if: steps.matches.outputs.count != '0' + uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0 + with: + version: 2026.9.0 + + - name: Install regctl + if: steps.matches.outputs.count != '0' + run: | + set -euo pipefail + install -d "${RUNNER_TEMP}/regctl-bin" + curl -fsSLo "${RUNNER_TEMP}/regctl-bin/regctl" \ + https://github.com/regclient/regclient/releases/download/v0.11.5/regctl-linux-amd64 + echo "c93aa7638749f5aaac1a8e01787321889c78f0101809bb2880343478d0ba0467 ${RUNNER_TEMP}/regctl-bin/regctl" | sha256sum -c - + chmod +x "${RUNNER_TEMP}/regctl-bin/regctl" + echo "${RUNNER_TEMP}/regctl-bin" >>"$GITHUB_PATH" + "${RUNNER_TEMP}/regctl-bin/regctl" version + + - name: Generate token + if: steps.matches.outputs.count != '0' + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.GH_APP_CLIENT_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + + - name: Pin the catalog to the hotfix revision + if: steps.matches.outputs.count != '0' + env: + SERVICE: ${{ steps.validate.outputs.service }} + UPSTREAM_VERSION: ${{ steps.validate.outputs.upstream_version }} + run: bun .github/scripts/sync-artifacts-catalog.ts --service "$SERVICE" --upstream "$UPSTREAM_VERSION" + + - name: Format catalog + if: steps.matches.outputs.count != '0' + run: bun x oxfmt@0.65.0 --config .oxfmtrc.json packages/stack/src/Artifacts.ts + + - name: Open pull request + if: steps.matches.outputs.count != '0' + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} + SERVICE: ${{ steps.validate.outputs.service }} + UPSTREAM_VERSION: ${{ steps.validate.outputs.upstream_version }} + REVISION: ${{ steps.validate.outputs.revision }} + RELEASE_VERSION: ${{ steps.validate.outputs.release_version }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + set -euo pipefail + if git diff --quiet -- packages/stack/src/Artifacts.ts; then + echo "Workload catalog already pins ${SERVICE} ${RELEASE_VERSION}." + exit 0 + fi + + manual_command="bun .github/scripts/sync-artifacts-catalog.ts --service ${SERVICE} --upstream ${UPSTREAM_VERSION}" + branch="slim-hotfix/${SERVICE}-${RELEASE_VERSION}" + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -b "$branch" + git add -- packages/stack/src/Artifacts.ts + git commit -m "chore(stack): pin ${SERVICE} ${RELEASE_VERSION}" + git remote set-url origin "https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + + if ! git push origin "HEAD:refs/heads/${branch}"; then + echo "::error ::Failed to push ${branch}. Run manually: ${manual_command}, then open a pull request by hand." + exit 1 + fi + + body="$(printf 'supabase/slim-services published a hotfix revision `%s` for `%s`. This pins the workload catalog to it.\n\nhttps://github.com/supabase/slim-services/releases/tag/%s-%s\n' \ + "$RELEASE_VERSION" "$SERVICE" "$SERVICE" "$RELEASE_VERSION")" + + if ! gh pr create \ + --title "chore(stack): pin ${SERVICE} ${RELEASE_VERSION}" \ + --body "$body" \ + --assignee jgoux \ + --head "$branch" \ + --base "$DEFAULT_BRANCH"; then + echo "::error ::Pushed ${branch} but failed to open the pull request. Run manually: ${manual_command}, then open a pull request from ${branch} against ${DEFAULT_BRANCH} by hand." + exit 1 + fi From 1b97247cbb30dc26545c758e48a59a0f22ef0d59 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 29 Sep 2026 10:28:21 +0200 Subject: [PATCH 05/45] fix(ci): validate slim hotfix payload versions --- .../scripts/sync-artifacts-catalog.test.ts | 57 ++++++++++++ .github/scripts/sync-artifacts-catalog.ts | 90 +++++++++++++++++++ .github/workflows/slim-release-published.yml | 30 ++----- 3 files changed, 156 insertions(+), 21 deletions(-) diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index 6893cf5e01..7d264629f0 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -11,6 +11,7 @@ import { findHotfixMatches, planArtifactCatalogUpdate, refreshCatalogPin, + validateSlimReleasePublishedPayload, type RevisionIo, } from "./sync-artifacts-catalog.ts"; @@ -170,6 +171,62 @@ describe("findHotfixMatches", () => { }); }); +describe("validateSlimReleasePublishedPayload", () => { + test("rejects a newline injected into upstream_version", () => { + expect(() => + validateSlimReleasePublishedPayload({ + service: "postgres", + upstream_version: "15.14.1.168\nrevision=999\ninjected=yes", + revision: "0", + release_version: "15.14.1.168\nrevision=999\ninjected=yes-r0", + }), + ).toThrow(InvalidPayloadError); + }); + + test("accepts a Studio-style calendar-versioned upstream", () => { + const payload = validateSlimReleasePublishedPayload({ + service: "studio", + upstream_version: "2026.09.04-sha-5a67366", + revision: "1", + release_version: "2026.09.04-sha-5a67366-r1", + }); + + expect(payload).toEqual({ + service: "studio", + upstream_version: "2026.09.04-sha-5a67366", + revision: 1, + release_version: "2026.09.04-sha-5a67366-r1", + }); + }); + + test("accepts a Postgres four-part upstream version", () => { + const payload = validateSlimReleasePublishedPayload({ + service: "postgres", + upstream_version: "15.14.1.168", + revision: "3", + release_version: "15.14.1.168-r3", + }); + + expect(payload).toEqual({ + service: "postgres", + upstream_version: "15.14.1.168", + revision: 3, + release_version: "15.14.1.168-r3", + }); + }); + + test("rejects a release_version that does not match the derived value", () => { + expect(() => + validateSlimReleasePublishedPayload({ + service: "postgres", + upstream_version: "15.14.1.168", + revision: "3", + release_version: "15.14.1.168-r4", + }), + ).toThrow(InvalidPayloadError); + }); +}); + describe("planArtifactCatalogUpdate", () => { test("pins to the highest committed revision, writing every native target's digests", async () => { const digests = nativeDigests("c"); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index 80a9c9a96d..488a2a0d55 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -21,6 +21,14 @@ * * bun .github/scripts/sync-artifacts-catalog.ts hotfix-matches --service \ * --upstream --revision + * + * Validate-payload mode (used by the same workflow, before anything else): + * checks an untrusted `slim-release-published` dispatch payload against + * anchored charsets and prints it back as `key=value` lines, so a value that + * fails validation is never written to `$GITHUB_OUTPUT` in the first place. + * + * bun .github/scripts/sync-artifacts-catalog.ts validate-payload --service \ + * --upstream --revision --release */ import { parseDockerfileServiceImages } from "../../apps/cli/src/shared/services/parse-dockerfile-service-images.ts"; @@ -403,6 +411,11 @@ export function findHotfixMatches( ...pins.additional.map((span) => ({ target: "additional" as const, span })), ]; + // A service's default pin and every additional pin live on distinct release lines + // (`selectEntry`'s `releaseLine` split), so at most one candidate's `upstreamVersion` can equal + // `payload.upstream_version`: this can return more than one entry only if the catalog itself + // duplicates an upstream version across pins, which `selectEntry` would already treat as + // ambiguous. const matches: HotfixMatch[] = []; for (const candidate of candidates) { if (candidate.span.version !== payload.upstream_version) continue; @@ -416,6 +429,51 @@ export function findHotfixMatches( return matches; } +/** Every upstream tag format the catalog carries, including Studio's `2026.09.04-sha-5a67366`. */ +const UPSTREAM_VERSION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +const SERVICE_NAME_PATTERN = /^[a-z0-9-]+$/; +const RELEASE_VERSION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*-r(0|[1-9][0-9]*)$/; + +/** + * Validates an untrusted `slim-release-published` dispatch payload before any of its fields are + * written anywhere (a `$GITHUB_OUTPUT` line, a branch name, a PR title, …). Every field is checked + * against an anchored charset — `upstream_version` and `release_version` included, not just + * `service` and `revision` — so a value carrying a newline or shell/Actions metacharacter is + * rejected here instead of being echoed downstream. + */ +export function validateSlimReleasePublishedPayload(input: { + readonly service: string; + readonly upstream_version: string; + readonly revision: string; + readonly release_version: string; +}): SlimReleasePublishedPayload { + if (!SERVICE_NAME_PATTERN.test(input.service)) { + throw new InvalidPayloadError(`invalid service: '${input.service}'`); + } + if (!UPSTREAM_VERSION_PATTERN.test(input.upstream_version)) { + throw new InvalidPayloadError(`invalid upstream_version: '${input.upstream_version}'`); + } + if (!/^(0|[1-9][0-9]*)$/.test(input.revision)) { + throw new InvalidPayloadError(`invalid revision: '${input.revision}'`); + } + if (!RELEASE_VERSION_PATTERN.test(input.release_version)) { + throw new InvalidPayloadError(`invalid release_version: '${input.release_version}'`); + } + const revision = Number(input.revision); + const expected = `${input.upstream_version}-r${revision}`; + if (input.release_version !== expected) { + throw new InvalidPayloadError( + `release_version '${input.release_version}' does not match derived '${expected}'`, + ); + } + return { + service: input.service, + upstream_version: input.upstream_version, + revision, + release_version: input.release_version, + }; +} + function skipReason(alias: string, version: string, entry: SelectedEntry): string | undefined { if (entry.kind === "unmodelled-service") { return `${CATALOG_PATH} has no slim entry for ${alias}.`; @@ -713,7 +771,39 @@ async function runHotfixMatches(argv: ReadonlyArray): Promise { console.log(JSON.stringify(matches)); } +async function runValidatePayload(argv: ReadonlyArray): Promise { + const flags = parseFlags(argv); + const service = flags.get("service"); + const upstream = flags.get("upstream"); + const revision = flags.get("revision"); + const release = flags.get("release"); + if ( + service === undefined || + upstream === undefined || + revision === undefined || + release === undefined + ) { + throw new InvalidPayloadError( + "Usage: sync-artifacts-catalog.ts validate-payload --service --upstream --revision --release ", + ); + } + const payload = validateSlimReleasePublishedPayload({ + service, + upstream_version: upstream, + revision, + release_version: release, + }); + console.log(`service=${payload.service}`); + console.log(`upstream_version=${payload.upstream_version}`); + console.log(`revision=${payload.revision}`); + console.log(`release_version=${payload.release_version}`); +} + async function main(argv: ReadonlyArray): Promise { + if (argv[0] === "validate-payload") { + await runValidatePayload(argv.slice(1)); + return; + } if (argv[0] === "hotfix-matches") { await runHotfixMatches(argv.slice(1)); return; diff --git a/.github/workflows/slim-release-published.yml b/.github/workflows/slim-release-published.yml index 90261fe6b4..25f2a3582c 100644 --- a/.github/workflows/slim-release-published.yml +++ b/.github/workflows/slim-release-published.yml @@ -45,31 +45,19 @@ jobs: UPSTREAM_VERSION: ${{ github.event.client_payload.upstream_version }} REVISION: ${{ github.event.client_payload.revision }} RELEASE_VERSION: ${{ github.event.client_payload.release_version }} + # Delegates to `validateSlimReleasePublishedPayload` (sync-artifacts-catalog.ts) so every + # field — including `upstream_version` and `release_version`, not just `service` and + # `revision` — is checked against an anchored charset before it is ever written to + # `$GITHUB_OUTPUT`, a branch name, or a PR title. A value that fails validation makes the + # script exit non-zero without printing anything. run: | set -euo pipefail - if ! [[ "$SERVICE" =~ ^[a-z0-9-]+$ ]]; then - echo "::error ::invalid service '${SERVICE}'" + if ! output="$(bun .github/scripts/sync-artifacts-catalog.ts validate-payload \ + --service "$SERVICE" --upstream "$UPSTREAM_VERSION" --revision "$REVISION" --release "$RELEASE_VERSION")"; then + echo "$output" exit 1 fi - if [ -z "$UPSTREAM_VERSION" ]; then - echo "::error ::missing upstream_version" - exit 1 - fi - if ! [[ "$REVISION" =~ ^(0|[1-9][0-9]*)$ ]]; then - echo "::error ::invalid revision '${REVISION}'" - exit 1 - fi - expected="${UPSTREAM_VERSION}-r${REVISION}" - if [ "$RELEASE_VERSION" != "$expected" ]; then - echo "::error ::release_version '${RELEASE_VERSION}' does not match derived '${expected}'" - exit 1 - fi - { - echo "service=${SERVICE}" - echo "upstream_version=${UPSTREAM_VERSION}" - echo "revision=${REVISION}" - echo "release_version=${RELEASE_VERSION}" - } >>"$GITHUB_OUTPUT" + echo "$output" >>"$GITHUB_OUTPUT" - name: Find hotfix matches id: matches From cc82c1797e92a8715e6c7d1445927968d6df72fb Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 29 Sep 2026 10:57:51 +0200 Subject: [PATCH 06/45] feat(cli): resolve slim images from pinned catalog revisions --- .../db-bootstrap/pinned-image.unit.test.ts | 27 +- .../command-internal/db-image.unit.test.ts | 14 +- .../edge-runtime-image.unit.test.ts | 4 +- .../start/start.services.unit.test.ts | 8 +- .../start/start.slim-images.e2e.test.ts | 5 +- .../functions/functions.shared.unit.test.ts | 3 +- .../src/shared/services/services.shared.ts | 16 +- .../services/services.shared.unit.test.ts | 87 +++++-- apps/cli/src/shared/services/slim-images.ts | 54 ++-- .../shared/services/slim-images.unit.test.ts | 240 +++++++++++------- 10 files changed, 309 insertions(+), 149 deletions(-) diff --git a/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts b/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts index 408f9adefa..1ecbd27145 100644 --- a/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts +++ b/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts @@ -30,9 +30,11 @@ describe("resolvePinnedImage", () => { it("resolves slim images when the flag is on and the pin is current", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); - expect(resolvePinnedImage("gotrue", "auth", {})).toBe(toSlimImage("gotrue", currentAuth)); + expect(resolvePinnedImage("gotrue", "auth", {})).toBe( + toSlimImage("gotrue", currentAuth) ?? currentAuth, + ); expect(resolvePinnedImage("gotrue", "auth", { auth: currentAuthTag })).toBe( - toSlimImage("gotrue", currentAuth), + toSlimImage("gotrue", currentAuth) ?? currentAuth, ); }); @@ -52,15 +54,18 @@ describe("resolvePinnedImage", () => { it("normalizes a current pooler pin onto the slim tag scheme", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); expect(resolvePinnedImage("supavisor", "pooler", { pooler: currentPoolerTag })).toBe( - toSlimImage("supavisor", currentPooler), + toSlimImage("supavisor", currentPooler) ?? currentPooler, + ); + // The opposite `v`-prefix variant of the same pin still counts as current (`pinMatchesCurrentImage` + // normalizes both before comparing), so it slim-translates the same way, or — with no catalog + // match — keeps this variant's own tag rather than the default Dockerfile one. + const altPoolerTag = currentPoolerTag.startsWith("v") + ? currentPoolerTag.slice(1) + : `v${currentPoolerTag}`; + const altPooler = currentPooler.replace(/:[^:]+$/, `:${altPoolerTag}`); + expect(resolvePinnedImage("supavisor", "pooler", { pooler: altPoolerTag })).toBe( + toSlimImage("supavisor", altPooler) ?? altPooler, ); - expect( - resolvePinnedImage("supavisor", "pooler", { - pooler: currentPoolerTag.startsWith("v") - ? currentPoolerTag.slice(1) - : `v${currentPoolerTag}`, - }), - ).toBe(toSlimImage("supavisor", currentPooler)); }); it("keeps a historical postgres pin on docker.io", () => { @@ -73,7 +78,7 @@ describe("resolvePinnedImage", () => { "supabase/postgres:17.4.1.1", ); expect(resolvePinnedImage("pg", "postgres", { postgres: currentPostgresTag })).toBe( - toSlimImage("pg", currentPostgres), + toSlimImage("pg", currentPostgres) ?? currentPostgres, ); }); }); diff --git a/apps/cli/src/command-internal/db-image.unit.test.ts b/apps/cli/src/command-internal/db-image.unit.test.ts index 72946b7efa..abbf38809d 100644 --- a/apps/cli/src/command-internal/db-image.unit.test.ts +++ b/apps/cli/src/command-internal/db-image.unit.test.ts @@ -118,11 +118,15 @@ describe("resolveDbImage", () => { const dir = withTemp(); return Effect.gen(function* () { expect(yield* resolve(dir, 15)).toEqual({ - image: toSlimImage("pg", POSTGRES_FALLBACK_IMAGE_PG15_SLIM), + image: + toSlimImage("pg", POSTGRES_FALLBACK_IMAGE_PG15_SLIM) ?? + POSTGRES_FALLBACK_IMAGE_PG15_SLIM, configImage: POSTGRES_FALLBACK_IMAGE_PG15_SLIM, }); expect(yield* resolve(dir, 13)).toEqual({ - image: toSlimImage("pg", POSTGRES_FALLBACK_IMAGE_PG15_SLIM), + image: + toSlimImage("pg", POSTGRES_FALLBACK_IMAGE_PG15_SLIM) ?? + POSTGRES_FALLBACK_IMAGE_PG15_SLIM, configImage: POSTGRES_FALLBACK_IMAGE_PG15_SLIM, }); rmSync(dir, { recursive: true, force: true }); @@ -148,7 +152,9 @@ describe("resolveDbImage", () => { writePin(dir, pg15SlimTag); return Effect.gen(function* () { expect(yield* resolve(dir, 15)).toEqual({ - image: toSlimImage("pg", POSTGRES_FALLBACK_IMAGE_PG15_SLIM), + image: + toSlimImage("pg", POSTGRES_FALLBACK_IMAGE_PG15_SLIM) ?? + POSTGRES_FALLBACK_IMAGE_PG15_SLIM, configImage: POSTGRES_FALLBACK_IMAGE_PG15_SLIM, }); rmSync(dir, { recursive: true, force: true }); @@ -174,7 +180,7 @@ describe("resolveDbImage", () => { writePin(dir, currentPostgresTag); return Effect.gen(function* () { expect(yield* resolve(dir, 17)).toEqual({ - image: toSlimImage("pg", currentPostgres), + image: toSlimImage("pg", currentPostgres) ?? currentPostgres, configImage: currentPostgres, }); rmSync(dir, { recursive: true, force: true }); diff --git a/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts b/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts index 458531b686..1ebb3f3326 100644 --- a/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts +++ b/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts @@ -93,7 +93,9 @@ describe("resolveEdgeRuntimeImage", () => { return resolve(dir, 2).pipe( Effect.tap((image) => Effect.sync(() => { - expect(image).toBe(toSlimImage("edgeruntime", currentEdgeRuntime)); + expect(image).toBe( + toSlimImage("edgeruntime", currentEdgeRuntime) ?? currentEdgeRuntime, + ); rmSync(dir, { recursive: true, force: true }); }), ), diff --git a/apps/cli/src/commands/start/start.services.unit.test.ts b/apps/cli/src/commands/start/start.services.unit.test.ts index 73f6921233..ea2a7e0535 100644 --- a/apps/cli/src/commands/start/start.services.unit.test.ts +++ b/apps/cli/src/commands/start/start.services.unit.test.ts @@ -249,11 +249,11 @@ describe("resolveStartImagePlan under SUPABASE_USE_SLIM_IMAGES", () => { it("plans slim images when the flag is on, keeping unmapped services on docker.io", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); - expect(imageFor("gotrue")).toBe(toSlimImage("gotrue", currentGotrue)); - expect(imageFor("logflare")).toBe(toSlimImage("logflare", currentLogflare)); - expect(imageFor("vector")).toBe(toSlimImage("vector", currentVector)); + expect(imageFor("gotrue")).toBe(toSlimImage("gotrue", currentGotrue) ?? currentGotrue); + expect(imageFor("logflare")).toBe(toSlimImage("logflare", currentLogflare) ?? currentLogflare); + expect(imageFor("vector")).toBe(toSlimImage("vector", currentVector) ?? currentVector); expect(imageFor("supavisor", { pooler: currentPoolerTag })).toBe( - toSlimImage("supavisor", currentPooler), + toSlimImage("supavisor", currentPooler) ?? currentPooler, ); expect(imageFor("supavisor", { pooler: "2.0.0" })).toBe("supabase/supavisor:2.0.0"); expect(imageFor("kong")).toBe("library/kong:2.8.1"); diff --git a/apps/cli/src/commands/start/start.slim-images.e2e.test.ts b/apps/cli/src/commands/start/start.slim-images.e2e.test.ts index 65dae6a808..80f9ff9a8a 100644 --- a/apps/cli/src/commands/start/start.slim-images.e2e.test.ts +++ b/apps/cli/src/commands/start/start.slim-images.e2e.test.ts @@ -75,7 +75,7 @@ const WGET_PROBE_ALIASES = [ function latestImagesToPull(): ReadonlyArray { return [...new Set([...PULL_ALIASES, ...WGET_PROBE_ALIASES])].map((alias) => - toSlimImage(alias, dockerfileServiceImageRaw(alias)), + expectedSlimImage(alias), ); } @@ -93,7 +93,8 @@ const containerImage = Effect.fnUntraced(function* (name: string) { }); function expectedSlimImage(alias: string): string { - return toSlimImage(alias, dockerfileServiceImageRaw(alias)); + const raw = dockerfileServiceImageRaw(alias); + return toSlimImage(alias, raw) ?? raw; } const containerHealthcheckTest = Effect.fnUntraced(function* (name: string) { diff --git a/apps/cli/src/shared/functions/functions.shared.unit.test.ts b/apps/cli/src/shared/functions/functions.shared.unit.test.ts index 5eb8806232..2836cd12b9 100644 --- a/apps/cli/src/shared/functions/functions.shared.unit.test.ts +++ b/apps/cli/src/shared/functions/functions.shared.unit.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { Effect } from "effect"; import { dockerfileServiceImageRaw } from "../services/dockerfile-images.ts"; +import { toSlimImage } from "../services/slim-images.ts"; import { DENO1_EDGE_RUNTIME_VERSION, edgeRuntimeImage, @@ -26,7 +27,7 @@ describe("edgeRuntimeImage", () => { it("rewrites the current Dockerfile tag onto the slim ghcr.io image when the flag is on", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); expect(edgeRuntimeImage(currentEdgeRuntimeTag)).toBe( - `ghcr.io/supabase/cli/edge-runtime:${currentEdgeRuntimeTag}`, + toSlimImage("edgeruntime", rawEdgeRuntimeImage) ?? rawEdgeRuntimeImage, ); }); diff --git a/apps/cli/src/shared/services/services.shared.ts b/apps/cli/src/shared/services/services.shared.ts index 1496757576..a6a07dd1cc 100644 --- a/apps/cli/src/shared/services/services.shared.ts +++ b/apps/cli/src/shared/services/services.shared.ts @@ -211,17 +211,27 @@ export interface ServiceVersionRow { readonly remote: string; } +/** A release tag's `-r` suffix, matching the slim-services revision grammar. */ +const RELEASE_REVISION_SUFFIX = /^(?.+)-r(?:0|[1-9][0-9]*)$/; + +/** Strips a slim release tag's `-r` suffix, if any, back to its upstream version. */ +function upstreamVersionFromTag(tag: string): string { + return RELEASE_REVISION_SUFFIX.exec(tag)?.groups?.upstream ?? tag; +} + function toServiceVersionRow( service: ServiceImageSpec, remote: Partial> = {}, ): ServiceVersionRow { - const tagSeparator = service.image.lastIndexOf(":"); + // `@`-aware: a slim catalog pin's image carries a `@sha256:…` digest after the tag. + const withoutDigest = service.image.split("@")[0] ?? service.image; + const tagSeparator = withoutDigest.lastIndexOf(":"); if (tagSeparator === -1) { throw new Error(`Invalid service image entry: ${service.image}`); } - const name = service.image.slice(0, tagSeparator); - const local = service.image.slice(tagSeparator + 1); + const name = withoutDigest.slice(0, tagSeparator); + const local = upstreamVersionFromTag(withoutDigest.slice(tagSeparator + 1)); return { name, diff --git a/apps/cli/src/shared/services/services.shared.unit.test.ts b/apps/cli/src/shared/services/services.shared.unit.test.ts index c22fb49b29..72f9e38c0e 100644 --- a/apps/cli/src/shared/services/services.shared.unit.test.ts +++ b/apps/cli/src/shared/services/services.shared.unit.test.ts @@ -6,12 +6,43 @@ import { fetchLinkedServiceVersions, listLocalServiceVersions, localServiceImagesFromDockerfile, + mergeRemoteServiceVersions, parseDockerfileServiceImages, postgresImageForDbMajorVersion, renderServicesTable, renderServicesWarning, } from "./services.shared.ts"; +// Only `auth` is pinned in this fixture catalog, at the current Dockerfile-independent version +// `v2.197.0-r0`, with a realistic (non-placeholder) fixture digest built to the real +// `ArtifactPin`/`NativePin` shape from `@supabase/stack/internal/artifacts`. Every other service +// is deliberately absent, so `toSlimImage` falls through to the upstream image for them — the +// expected state until Dependabot catches the Dockerfile up. `vi.mock` factories are hoisted +// above every other top-level statement, so the fixture is inlined rather than referencing an +// outer const. +vi.mock("@supabase/stack/internal/artifacts", () => { + const digest = "260e94edb8d402555791146fcf70b8e90efdc6a81877a04e5aa26f0f416a5dd7"; + const nativePin = { archive: digest, manifest: digest }; + return { + catalogPins: () => [ + { + service: "auth", + sourceService: "auth", + pin: { + upstreamVersion: "v2.197.0", + revision: 0, + image: `ghcr.io/supabase/cli/auth:v2.197.0-r0@sha256:${digest}`, + natives: { + "darwin-arm64": nativePin, + "linux-amd64": nativePin, + "linux-arm64": nativePin, + }, + }, + }, + ], + }; +}); + const ACCESS_TOKEN = Redacted.make(`sbp_${"a".repeat(40)}`); const PROJECT_REF = "abcdefghijklmnopqrst"; @@ -85,20 +116,24 @@ describe("services shared", () => { expect(postgresImageForDbMajorVersion(15)).toBe("supabase/postgres:15.14.1.167"); }); - test("lists slim images when SUPABASE_USE_SLIM_IMAGES is set", () => { + test("slim-translates a version override that matches a catalog pin", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); - expect(listLocalServiceVersions().map((row) => row.name)).toEqual([ - "ghcr.io/supabase/cli/postgres", - "ghcr.io/supabase/cli/auth", - "ghcr.io/supabase/cli/postgrest", - "ghcr.io/supabase/cli/realtime", - "ghcr.io/supabase/cli/storage", - "ghcr.io/supabase/cli/edge-runtime", - "ghcr.io/supabase/cli/studio", - "ghcr.io/supabase/cli/pgmeta", - "ghcr.io/supabase/cli/analytics", - "ghcr.io/supabase/cli/pooler", - ]); + expect(listLocalServiceVersions({ serviceVersions: { auth: "v2.197.0" } })).toContainEqual({ + name: "ghcr.io/supabase/cli/auth", + // The catalog's release version (`v2.197.0-r0`) is a Dockerfile/manifest tag; the row shows + // the upstream version so a `supabase services` mismatch check compares upstream to upstream. + local: "v2.197.0", + remote: "", + }); + }); + + test("keeps a version override that isn't in the catalog on docker.io", () => { + vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); + expect(listLocalServiceVersions({ serviceVersions: { storage: "v1.70.3" } })).toContainEqual({ + name: "supabase/storage-api", + local: "v1.70.3", + remote: "", + }); }); test("keeps historical pins on docker.io when slimCurrentPinOnly is set", () => { @@ -126,15 +161,6 @@ describe("services shared", () => { ).toContainEqual({ name: "supabase/gotrue", local: "v2.151.0", remote: "" }); }); - test("slim-translates catalog version overrides that are not the Dockerfile pin", () => { - vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); - expect(listLocalServiceVersions({ serviceVersions: { storage: "v1.70.3" } })).toContainEqual({ - name: "ghcr.io/supabase/cli/storage", - local: "v1.70.3", - remote: "", - }); - }); - // Explicit overrides keep their registry; a serviceVersions pin still rewrites the tag. test("leaves explicit image overrides on docker.io when SUPABASE_USE_SLIM_IMAGES is set", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); @@ -513,4 +539,21 @@ describe("services shared", () => { ]), ).toContain("supabase/postgres:17.6.1.132 => 17.6.1.200"); }); + + test("compares upstream versions for a slim catalog pin, not the release tag", () => { + vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); + const rows = mergeRemoteServiceVersions( + { auth: "v2.197.0" }, + { serviceVersions: { auth: "v2.197.0" } }, + ); + + expect(rows).toContainEqual({ + name: "ghcr.io/supabase/cli/auth", + local: "v2.197.0", + remote: "v2.197.0", + }); + // The pinned image's release tag (`v2.197.0-r0@sha256:…`) never surfaces as a mismatch + // against the upstream-only remote version. + expect(renderServicesWarning(rows)).toBeUndefined(); + }); }); diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index af0635fc11..a1fce543a6 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -1,3 +1,5 @@ +import { catalogPins } from "@supabase/stack/internal/artifacts"; + const SLIM_IMAGES_ENV = "SUPABASE_USE_SLIM_IMAGES"; const SLIM_IMAGE_PREFIX = "ghcr.io/supabase/cli/"; @@ -94,35 +96,55 @@ export function slimCatalogPin(alias: string, image: string): SlimCatalogPin | u } /** - * Rewrites a docker.io image reference to its `ghcr.io/supabase/cli` slim - * equivalent, keeping the pin's version. This helper owns tag normalization - * (`v`-prefixing, `tagPrefix`), so pins that differ only in prefix between the - * two registries (`supavisor`, `logflare`) land on the right slim tag. Vector's - * docker.io tags carry an `-alpine` variant suffix that the slim build does - * not publish, so the strip is scoped to `vector` only — an `-alpine`-suffixed - * pin on any other service is a real tag, not a variant marker. + * Looks up the pinned catalog image (with its published `@sha256` digest) for + * `service` whose `upstreamVersion` equals `version`. Reads the same catalog + * `apps/cli`'s stack-independent clients use, keyed by the slim-services + * `sourceService` name (which matches this module's `SlimServiceName`). */ -export function toSlimImage(alias: string, image: string): string { +function catalogImageFor(service: SlimServiceName, upstreamVersion: string): string | undefined { + for (const entry of catalogPins()) { + if (entry.sourceService === service && entry.pin.upstreamVersion === upstreamVersion) { + return entry.pin.image; + } + } + return undefined; +} + +/** + * Resolves the catalog's pinned slim image (repository, release version and + * digest) whose `upstreamVersion` normalizes to `image`'s tag for `alias`. + * This owns tag normalization (`v`-prefixing, `tagPrefix`, vector's `-alpine` + * strip) via {@link slimCatalogPin}, so pins that differ only in prefix + * between the two registries (`supavisor`, `logflare`) still match. Returns + * `undefined` when no catalog entry's `upstreamVersion` matches — including + * when Dependabot has bumped the Dockerfile ahead of the catalog — so callers + * keep the upstream (non-slim) image instead of guessing a slim tag. + */ +export function toSlimImage(alias: string, image: string): string | undefined { const pin = slimCatalogPin(alias, image); if (pin === undefined) { - return image; + return undefined; } - return `${SLIM_IMAGE_PREFIX}${pin.service}:${pin.version}`; + return catalogImageFor(pin.service, pin.version); } /** `toSlimImage` behind the feature flag; a no-op while the flag is off. */ export function slimImageForAlias(alias: string, image: string): string { - return slimImagesEnabled() ? toSlimImage(alias, image) : image; + return slimImagesEnabled() ? (toSlimImage(alias, image) ?? image) : image; } +/** The tag portion of `image`, ignoring any `@sha256:…` digest suffix. */ export function imageTag(image: string): string | undefined { - const tagSeparator = image.lastIndexOf(":"); - return tagSeparator === -1 ? undefined : image.slice(tagSeparator + 1); + const withoutDigest = image.split("@")[0] ?? image; + const tagSeparator = withoutDigest.lastIndexOf(":"); + return tagSeparator === -1 ? undefined : withoutDigest.slice(tagSeparator + 1); } +/** Replaces `image`'s tag with `tag`, dropping any `@sha256:…` digest — a new tag invalidates it. */ function replaceImageTag(image: string, tag: string): string { - const tagSeparator = image.lastIndexOf(":"); - return tagSeparator === -1 ? image : `${image.slice(0, tagSeparator + 1)}${tag}`; + const withoutDigest = image.split("@")[0] ?? image; + const tagSeparator = withoutDigest.lastIndexOf(":"); + return tagSeparator === -1 ? image : `${withoutDigest.slice(0, tagSeparator + 1)}${tag}`; } /** @@ -163,7 +185,7 @@ export function slimImageForCurrentPin( if (trimmed.length > 0 && !pinMatchesCurrentImage(alias, trimmed, currentRawImage)) { return tagged; } - return toSlimImage(alias, tagged); + return toSlimImage(alias, tagged) ?? tagged; } /** Slim images are published only under this prefix; single home for the check. */ diff --git a/apps/cli/src/shared/services/slim-images.unit.test.ts b/apps/cli/src/shared/services/slim-images.unit.test.ts index 4e082a3b7c..da6303d63f 100644 --- a/apps/cli/src/shared/services/slim-images.unit.test.ts +++ b/apps/cli/src/shared/services/slim-images.unit.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { dockerfileServiceImageRaw } from "./dockerfile-images.ts"; import { + imageTag, pinMatchesCurrentImage, slimCatalogPin, slimImageForAlias, @@ -11,97 +12,168 @@ import { usesSlimImageRuntime, } from "./slim-images.ts"; +// Only `auth` is pinned in this fixture catalog, at `v2.197.0-r0`, with a realistic +// (non-placeholder) fixture digest built to the real `ArtifactPin`/`NativePin` shape from +// `@supabase/stack/internal/artifacts`. Every other service is deliberately absent, so +// `toSlimImage` falls through to the upstream image for them — the expected state until +// Dependabot catches the Dockerfile up. `vi.mock` factories are hoisted above every other +// top-level statement, so the fixture is inlined rather than referencing an outer const. +vi.mock("@supabase/stack/internal/artifacts", () => { + const digest = "d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c"; + const nativePin = { archive: digest, manifest: digest }; + return { + catalogPins: () => [ + { + service: "auth", + sourceService: "auth", + pin: { + upstreamVersion: "v2.197.0", + revision: 0, + image: `ghcr.io/supabase/cli/auth:v2.197.0-r0@sha256:${digest}`, + natives: { + "darwin-arm64": nativePin, + "linux-amd64": nativePin, + "linux-arm64": nativePin, + }, + }, + }, + ], + }; +}); + +const AUTH_FIXTURE_PIN_IMAGE = + "ghcr.io/supabase/cli/auth:v2.197.0-r0@sha256:d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c"; + afterEach(() => { vi.unstubAllEnvs(); }); -describe("toSlimImage", () => { +describe("slimCatalogPin", () => { it.each([ - ["pg", "ghcr.io/supabase/cli/postgres"], - ["gotrue", "ghcr.io/supabase/cli/auth"], - ["postgrest", "ghcr.io/supabase/cli/postgrest"], - ["realtime", "ghcr.io/supabase/cli/realtime"], - ["storage", "ghcr.io/supabase/cli/storage"], - ["edgeruntime", "ghcr.io/supabase/cli/edge-runtime"], - ["studio", "ghcr.io/supabase/cli/studio"], - ["pgmeta", "ghcr.io/supabase/cli/pgmeta"], - ["logflare", "ghcr.io/supabase/cli/analytics"], - ["supavisor", "ghcr.io/supabase/cli/pooler"], - ["vector", "ghcr.io/supabase/cli/vector"], - ["imgproxy", "ghcr.io/supabase/cli/imgproxy"], - ["mailpit", "ghcr.io/supabase/cli/mailpit"], - ])("maps the %s manifest pin onto %s", (alias, repository) => { - const translated = toSlimImage(alias, dockerfileServiceImageRaw(alias)); - expect(translated.slice(0, translated.lastIndexOf(":"))).toBe(repository); - }); - - it("keeps a non-current pin instead of the catalog default", () => { - expect(toSlimImage("pg", "supabase/postgres:17.6.1.164")).toBe( - "ghcr.io/supabase/cli/postgres:17.6.1.164", - ); - expect(toSlimImage("studio", "supabase/studio:2026.08.17-sha-0c1da8f")).toBe( - "ghcr.io/supabase/cli/studio:2026.08.17-sha-0c1da8f", - ); + ["pg", "postgres"], + ["gotrue", "auth"], + ["postgrest", "postgrest"], + ["realtime", "realtime"], + ["storage", "storage"], + ["edgeruntime", "edge-runtime"], + ["studio", "studio"], + ["pgmeta", "pgmeta"], + ["logflare", "analytics"], + ["supavisor", "pooler"], + ["vector", "vector"], + ["imgproxy", "imgproxy"], + ["mailpit", "mailpit"], + ])("maps the %s alias onto the %s slim service", (alias, service) => { + const pin = slimCatalogPin(alias, dockerfileServiceImageRaw(alias)); + expect(pin?.service).toBe(service); + }); + + it("keeps a non-current pin's version verbatim", () => { + expect(slimCatalogPin("pg", "supabase/postgres:17.6.1.164")).toEqual({ + service: "postgres", + version: "17.6.1.164", + }); + expect(slimCatalogPin("studio", "supabase/studio:2026.08.17-sha-0c1da8f")).toEqual({ + service: "studio", + version: "2026.08.17-sha-0c1da8f", + }); }); - // Fixed pins, not manifest pins: dependabot bumps the manifest, so spelling - // out a current pin here would fail on every bump. The `it.each` above covers - // the part that must track it (the repository each alias maps to). it("keeps a single v on pins already prefixed on docker.io", () => { - expect(toSlimImage("realtime", "supabase/realtime:v2.130.0")).toBe( - "ghcr.io/supabase/cli/realtime:v2.130.0", - ); - expect(toSlimImage("storage", "supabase/storage-api:v1.72.1")).toBe( - "ghcr.io/supabase/cli/storage:v1.72.1", - ); - expect(toSlimImage("gotrue", "supabase/gotrue:V2.196.0")).toBe( - "ghcr.io/supabase/cli/auth:v2.196.0", - ); + expect(slimCatalogPin("realtime", "supabase/realtime:v2.130.0")).toEqual({ + service: "realtime", + version: "v2.130.0", + }); + expect(slimCatalogPin("storage", "supabase/storage-api:v1.72.1")).toEqual({ + service: "storage", + version: "v1.72.1", + }); + expect(slimCatalogPin("gotrue", "supabase/gotrue:V2.196.0")).toEqual({ + service: "auth", + version: "v2.196.0", + }); }); it("v-prefixes pins whose slim tag scheme differs from docker.io's", () => { - expect(toSlimImage("supavisor", "supabase/supavisor:2.9.10")).toBe( - "ghcr.io/supabase/cli/pooler:v2.9.10", - ); - expect(toSlimImage("logflare", "supabase/logflare:1.50.4")).toBe( - "ghcr.io/supabase/cli/analytics:v1.50.4", - ); - expect(toSlimImage("pgmeta", "supabase/postgres-meta:v0.98.0")).toBe( - "ghcr.io/supabase/cli/pgmeta:v0.98.0", - ); - }); - - it("keeps OrioleDB tags on docker.io", () => { - expect(toSlimImage("pg", "supabase/postgres:16.0.0.1-orioledb")).toBe( - "supabase/postgres:16.0.0.1-orioledb", - ); - expect(toSlimImage("pg", "supabase/postgres:orioledb-15.1.0.55")).toBe( - "supabase/postgres:orioledb-15.1.0.55", - ); + expect(slimCatalogPin("supavisor", "supabase/supavisor:2.9.10")).toEqual({ + service: "pooler", + version: "v2.9.10", + }); + expect(slimCatalogPin("logflare", "supabase/logflare:1.50.4")).toEqual({ + service: "analytics", + version: "v1.50.4", + }); + expect(slimCatalogPin("pgmeta", "supabase/postgres-meta:v0.98.0")).toEqual({ + service: "pgmeta", + version: "v0.98.0", + }); + }); + + it("excludes OrioleDB tags", () => { expect(slimCatalogPin("pg", "supabase/postgres:16.0.0.1-orioledb")).toBeUndefined(); + expect(slimCatalogPin("pg", "supabase/postgres:orioledb-15.1.0.55")).toBeUndefined(); }); it("strips vector's docker.io -alpine variant suffix", () => { - expect(toSlimImage("vector", "timberio/vector:0.53.0-alpine")).toBe( - "ghcr.io/supabase/cli/vector:0.53.0", - ); + expect(slimCatalogPin("vector", "timberio/vector:0.53.0-alpine")).toEqual({ + service: "vector", + version: "0.53.0", + }); }); it("does not strip -alpine from a non-vector service's tag", () => { - expect(toSlimImage("studio", "supabase/studio:2026.08.17-alpine")).toBe( - "ghcr.io/supabase/cli/studio:2026.08.17-alpine", - ); + expect(slimCatalogPin("studio", "supabase/studio:2026.08.17-alpine")).toEqual({ + service: "studio", + version: "2026.08.17-alpine", + }); }); - it("passes through aliases with no slim build", () => { + it("is absent for aliases with no slim build", () => { for (const alias of ["kong", "differ", "migra", "pgprove"]) { - const image = dockerfileServiceImageRaw(alias); - expect(toSlimImage(alias, image)).toBe(image); + expect(slimCatalogPin(alias, dockerfileServiceImageRaw(alias))).toBeUndefined(); } }); - it("passes through an untagged reference", () => { - expect(toSlimImage("pg", "supabase/postgres")).toBe("supabase/postgres"); + it("is absent for an untagged reference", () => { + expect(slimCatalogPin("pg", "supabase/postgres")).toBeUndefined(); + }); +}); + +describe("toSlimImage", () => { + it("returns the catalog's pinned image (with its digest) when the tag matches a catalog upstream version", () => { + expect(toSlimImage("gotrue", "supabase/gotrue:v2.197.0")).toBe(AUTH_FIXTURE_PIN_IMAGE); + // The alias-normalization prefix (V -> v) still applies before the catalog match. + expect(toSlimImage("gotrue", "supabase/gotrue:V2.197.0")).toBe(AUTH_FIXTURE_PIN_IMAGE); + }); + + it("returns undefined, keeping the upstream image, when the tag isn't in the catalog", () => { + expect(toSlimImage("gotrue", "supabase/gotrue:v2.100.0")).toBeUndefined(); + // `pg` has no entry at all in this fixture catalog. + expect(toSlimImage("pg", "supabase/postgres:17.6.1.164")).toBeUndefined(); + }); + + it("returns undefined for aliases and tags slimCatalogPin already excludes", () => { + expect(toSlimImage("pg", "supabase/postgres:16.0.0.1-orioledb")).toBeUndefined(); + expect(toSlimImage("kong", dockerfileServiceImageRaw("kong"))).toBeUndefined(); + expect(toSlimImage("pg", "supabase/postgres")).toBeUndefined(); + }); +}); + +describe("imageTag", () => { + it("returns the release tag, ignoring an @sha256 digest", () => { + expect( + imageTag( + "ghcr.io/supabase/cli/auth:v2.197.0-r0@sha256:d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c", + ), + ).toBe("v2.197.0-r0"); + }); + + it("returns the tag on a plain (digest-less) reference", () => { + expect(imageTag("supabase/gotrue:v2.197.0")).toBe("v2.197.0"); + }); + + it("returns undefined on an untagged reference", () => { + expect(imageTag("supabase/postgres")).toBeUndefined(); }); }); @@ -123,15 +195,20 @@ describe("slimImagesEnabled", () => { describe("slimImageForAlias", () => { it("is a no-op while the flag is off", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", ""); - expect(slimImageForAlias("pg", "supabase/postgres:17.6.1.165")).toBe( - "supabase/postgres:17.6.1.165", + expect(slimImageForAlias("gotrue", "supabase/gotrue:v2.197.0")).toBe( + "supabase/gotrue:v2.197.0", ); }); - it("translates when the flag is on", () => { + it("translates when the flag is on and the tag matches the catalog", () => { + vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); + expect(slimImageForAlias("gotrue", "supabase/gotrue:v2.197.0")).toBe(AUTH_FIXTURE_PIN_IMAGE); + }); + + it("keeps the upstream image when the flag is on but the tag isn't in the catalog", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); expect(slimImageForAlias("pg", "supabase/postgres:17.6.1.165")).toBe( - "ghcr.io/supabase/cli/postgres:17.6.1.165", + "supabase/postgres:17.6.1.165", ); }); }); @@ -161,24 +238,17 @@ describe("pinMatchesCurrentImage", () => { }); describe("slimImageForCurrentPin", () => { - it("slim-translates the current pin and leaves a historical pin on docker.io", () => { + it("slim-translates the current pin using the catalog's pinned digest", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); - const current = dockerfileServiceImageRaw("storage"); - const currentTag = current.split(":")[1] ?? ""; - expect(slimImageForCurrentPin("storage", current)).toBe(toSlimImage("storage", current)); - expect(slimImageForCurrentPin("storage", current, currentTag)).toBe( - toSlimImage("storage", current), - ); - expect(slimImageForCurrentPin("storage", current, "v1.67.0")).toBe( - "supabase/storage-api:v1.67.0", - ); + const current = "supabase/gotrue:v2.197.0"; + expect(slimImageForCurrentPin("gotrue", current)).toBe(AUTH_FIXTURE_PIN_IMAGE); + expect(slimImageForCurrentPin("gotrue", current, "v2.197.0")).toBe(AUTH_FIXTURE_PIN_IMAGE); + expect(slimImageForCurrentPin("gotrue", current, "v1.67.0")).toBe("supabase/gotrue:v1.67.0"); }); it("is a no-op while the flag is off", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", ""); - const current = dockerfileServiceImageRaw("storage"); - expect(slimImageForCurrentPin("storage", current, "v1.67.0")).toBe( - "supabase/storage-api:v1.67.0", - ); + const current = "supabase/gotrue:v2.197.0"; + expect(slimImageForCurrentPin("gotrue", current, "v1.67.0")).toBe("supabase/gotrue:v1.67.0"); }); }); From 7bccafd60948ef0f535e7f8047586bc0719d96fe Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 29 Sep 2026 10:57:56 +0200 Subject: [PATCH 07/45] docs(cli): rewrite the slim artifact mirrors ADR for immutable revisions --- docs/adr/0026-slim-artifact-mirrors.md | 136 ++++++++++++++++++++----- infra/cli-artifacts/README.md | 19 +++- 2 files changed, 127 insertions(+), 28 deletions(-) diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index cdcc4fb4ca..b478613500 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -1,48 +1,138 @@ # 0026. Slim image and native artifact mirrors -**Status**: proposed -**Date**: 2026-09-17 +**Status**: accepted +**Date**: 2026-09-29 ## Problem Statement -Slim service images publish to `ghcr.io/supabase/cli/:`. Native archives publish to GitHub Releases. A broken build must be replaceable under the **same** upstream version string (`force=true`); bumping the tag is not an option. +slim-services used to publish each artifact under the exact upstream version string, so the +only way to fix bad packaging was `force=true`: overwrite the GitHub release assets +(`--clobber`), move the GHCR/ECR tags, and overwrite the S3 objects. That broke already-released +CLIs pinned to that version, and the CLI itself resolved artifacts inconsistently: some catalog +images were pinned by `@sha256` and some were tag-only, native archives had no pin at all, and +`manifest.json` was not integrity-checked despite supplying `entrypoint`/`cmd`. The runtime +checksum came from mutable sources — the release `SHA256SUMS` or a GHCR `:-native-` +tag — so one CLI version could run different bytes on different machines. -Some consumers cannot reach GitHub release assets or a single registry blob CDN. This ADR covers publishing those copies and the local stack's fail-through. +Some consumers also cannot reach GitHub release assets or a single registry blob CDN. This ADR +covers the immutable publish scheme, the CLI's content pins, and the local stack's fail-through +across mirrors. ## Decision -Publish each slim **image** to GHCR and AWS ECR Public (`public.ecr.aws/supabase/cli/`), digest-preserving, via `mirror-slim-image.yml` and `.github/scripts/mirror-slim-image.ts`. Publish each slim **native** archive as an OCI artifact on the same two repositories under `:version-native-` (not `:version-linux-*`, which are image platform tags). GitHub Releases remain the human HTTPS copy and `--clobber` on force. - -`force=true` always requests the ECR copy, including when the image digest is unchanged (natives may have moved). The **image** destination digest gates `publish-release` once the dispatch token exists. Native ECR copy is best-effort (`continue-on-error`) and must not fail that release. Native tags ride in a separate `natives[]` payload field. This mirror does not open a pull request to pin `packages/stack/src/Artifacts.ts`. A Dependabot Dockerfile bump commits that catalog pin onto the same pull request, so the docker.io tag, the slim image, and the native archive stay on one version. A PostgreSQL release line that is not the Dockerfile `pg` tag, and a same-version digest republish, stay put until that Dockerfile line changes. Do not prune untagged GHCR or ECR manifests: already-shipped CLIs still pin old image digests until that pull request merges. Natives follow the moved tag immediately. - -ECR Public tags are always mutable; `ecr-public create-repository` has no immutability flag. Reuse the existing `PROD_AWS_ROLE` in `supabase/cli`. - -A public S3 bucket on `*.amazonaws.com` holds native archives for hosts that cannot reach GitHub Releases ([infra/cli-artifacts](../../infra/cli-artifacts/README.md)). It is not a checksum authority. The stack tries the GitHub Release first and falls back to that bucket. The expected checksum comes from the release `SHA256SUMS` or, when that is blocked, from the archive layer of the `:version-native-` artifact on GHCR. An archive from either host is accepted only when it matches. - -The container runtime pulls a catalog image from GHCR first and falls back to the same reference on ECR Public. A failing image fallback still reports the primary's original error. When every native checksum source or archive mirror fails, the error names each source with its failure. +### Immutable revisions + +Every slim-services publish carries a release version `R = -r`, where `U` is the upstream +tag exactly as the service's `tag_pattern` matches it and `N` is a revision starting at `0`. +Revision `N` of `U` is taken exactly when the GitHub release `-U-rN` exists — the GitHub +release is the commit point. Allocation is `max(taken) + 1`, computed from the full paginated +release list before any push. Once a release exists, `gh release create` is create-only: nothing +ever writes to that `-rN` again, so **there is no same-version overwrite**. A hotfix publishes a +new `-rN`; it never touches a previously published revision's bytes. `R` is never fed to a +semver library — ordering is the service's upstream version key, then revision, numerically. + +**Legacy tags** (without `-rN`) are frozen: nothing parses, audits, rewrites, or unpublishes +them, so CLIs pinned to a legacy tag keep working. + +### Content pins, not runtime checksums + +The CLI's stack catalog (`packages/stack/src/Artifacts.ts`) pins every artifact by content: + +- the slim image, as `ghcr.io/supabase/cli/:@sha256:`; +- each target's archive sha256 and manifest sha256 (`ArtifactPin.natives`). + +`SlimServicesSource` hash-checks the manifest against its pin before parsing it (its `version` +field must equal `R`) and hash-checks the archive against its pin while it streams. There is no +runtime checksum authority: **GitHub Releases, the S3 mirror, GHCR, and ECR Public are byte +mirrors only** for both images and native archives. None of them is consulted for the expected +hash — that comes only from the pin already committed to the catalog. + +### Sync and the S3-staleness check + +`.github/scripts/sync-artifacts-catalog.ts` writes those pins. On a Dependabot upstream bump to +`U`, it picks the highest committed revision `N` of `-U-rN`, reads that release's +`SHA256SUMS` for the archive and manifest sha256 per target, and resolves the image digest with +`regctl manifest head`. Before writing the pin, it downloads each target's S3 archive and +manifest and hashes them against those same release sums; a mismatch fails the sync instead of +pinning bytes that don't match GitHub. This exists because `publish-release` does not wait for +the ECR/S3 mirror to finish, so a freshly committed revision's S3 copy can briefly lag or hold +bytes from an earlier failed attempt. Hosts that reach GitHub are unaffected — GitHub is the +primary mirror — but a host that can only reach S3 would otherwise fail verification with no +fallback. A mismatch here means "run the mirror backfill", not "the CLI is broken". A Dependabot +bump to a version _older_ than the catalog's current pin is a non-blocking skip: the catalog is +allowed to lead the Dockerfile, and the skip still prevents a downgrade. + +### Hotfix pickup + +`mirror-ecr` runs before `publish-release`, so the mirror dispatch fires before a revision is +committed and can't trigger a catalog update. Instead, the last step of slim-services' +`publish-release` sends a `repository_dispatch` (`slim-release-published`) to the CLI repo with +`{service, upstream_version, revision}`. A CLI workflow handles it: when `upstream_version` +matches any pinned upstream version for that service and `revision` is newer than the pinned +one, it runs the sync for that entry and opens a pull request. The fallback, if that dispatch or +the PR step fails, is a documented manual `bun .github/scripts/sync-artifacts-catalog.ts` +invocation — the release itself is already committed by then, so a failure here means "notify +the CLI by hand", not "republish". + +### Registry and bucket mirrors + +Publish each slim **image** to GHCR and AWS ECR Public (`public.ecr.aws/supabase/cli/`), +digest-preserving, via `mirror-slim-image.yml` and `.github/scripts/mirror-slim-image.ts`. +Publish each slim **native** archive as an OCI artifact on the same two repositories under +`:R-native-`. A public S3 bucket on `*.amazonaws.com` +([infra/cli-artifacts](../../infra/cli-artifacts/README.md)) holds native archives for hosts that +cannot reach GitHub Releases. The container runtime pulls a catalog image from GHCR first and +falls back to the same reference on ECR Public. A failing image fallback still reports the +primary's original error. When every native archive mirror fails to serve the pinned bytes, the +error names each mirror with its failure. + +ECR Public tags are always mutable; `ecr-public create-repository` has no immutability flag. +Mirror backfills of a committed revision copy the immutable GHCR bytes by digest, so re-running +one is idempotent. Reuse the existing `PROD_AWS_ROLE` in `supabase/cli`. ## Follow-up -- Native ECR copy is best-effort; a daily mirror audit should report native drift. That audit is not defined in this repository yet. +- Native ECR copy is best-effort; a daily mirror audit should report native drift and treat the + committed GHCR digest as the source of truth for backfills. That audit is not defined in this + repository yet. +- Unpublishing legacy (non-revision) artifacts is planned as a separate task; they stay published + and frozen until then. ## Rationale -ECR Public mirroring already exists for other CLI images and needs no new vendor. Native OCI on those same repos reuses `regctl` and that role. Extracting the workflow into bun scripts lets the copy, digest check, and native payload validation run in CI and locally. +ECR Public mirroring already exists for other CLI images and needs no new vendor. Native OCI on +those same repos reuses `regctl` and that role. Making the GitHub release the single commit +point, rather than a destination-specific immutability guard on each mirror, keeps the +invariant in one place: once `-U-rN` exists, every downstream copy of it is either correct +or considered stale and backfilled — never rewritten in place. ## Consequences -- Same-version overwrite works on GHCR, GitHub Releases, and ECR Public. -- Images come from the GHCR catalog pin with ECR Public as fallback, and natives from GitHub Releases with the S3 bucket as fallback. -- A GitHub native can be live while the ECR native is stale. -- Old CLI releases keep pulling the previous image digest. +- Hotfixing packaging never touches previously published bytes; it always publishes a new + revision. +- The CLI verifies every artifact it downloads against a pin already committed to its own + catalog — no destination is trusted to supply the expected hash at runtime. +- A same-version republish is no longer possible; every fix is a new, allocatable revision. +- A revision's ECR/S3 copy can briefly lag GitHub after publish; the sync's S3-staleness check + and the mirror backfill are what keep them converging. +- Old CLI releases keep pulling their originally pinned image digest and native bytes forever. ## Alternatives considered -1. **Google Artifact Registry / GCS** — blob host `storage.googleapis.com` is on at least one major sandbox Trusted list. Rejected for this cut: new company-wide vendor. -2. **Public S3 bucket** — HTTPS GET on `*.amazonaws.com` can work without CloudFront. Adopted for native archives. The bucket is not a checksum authority, and image pulls stay on GHCR and ECR Public. +1. **Google Artifact Registry / GCS** — blob host `storage.googleapis.com` is on at least one + major sandbox Trusted list. Rejected for this cut: new company-wide vendor. +2. **Public S3 bucket** — HTTPS GET on `*.amazonaws.com` can work without CloudFront. Adopted for + native archives, as a byte mirror only; the CLI's checksum authority is its own catalog pin, + verified against the release's `SHA256SUMS` once at sync time, not read at runtime. 3. **npm packages** — allowlisted widely, but a published version cannot be replaced. -4. **Docker Hub `supabase/cli-*`** — deferred; blob CDN is also off some default allowlists, and names collide with upstream `supabase/`. -5. **Unpin slim images** — would make old CLIs see a moved tag. Rejected: conflicts with ADR 0017. +4. **Docker Hub `supabase/cli-*`** — deferred; blob CDN is also off some default allowlists, and + names collide with upstream `supabase/`. +5. **Same-version overwrite (`force=true` clobber)** — the original model. Rejected: it broke + already-released CLIs pinned to that version and made a republish silently move bytes out + from under them. Replaced by immutable `-rN` revisions. +6. **A destination-specific immutability guard (for example, an S3 conditional write)** — + rejected in favor of making the GitHub release the single commit point; every other + destination is just a copy that either matches it or is stale. ## Related diff --git a/infra/cli-artifacts/README.md b/infra/cli-artifacts/README.md index 9413175da1..07f0b01708 100644 --- a/infra/cli-artifacts/README.md +++ b/infra/cli-artifacts/README.md @@ -18,13 +18,22 @@ agent sandboxes that allow `*.amazonaws.com`. Objects are addressed as ``` -https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com///--.tar.zst -https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com///--.manifest.json -https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com///--.SHA256SUMS +https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com///--.tar.zst +https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com///--.manifest.json +https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com///--.SHA256SUMS ``` -which keeps the GitHub release asset names, so the `SHA256SUMS` lines match the archive name -without rewriting. +where `` is the immutable `-r` slim-services revision, which keeps +the GitHub release asset names so the `SHA256SUMS` lines match the archive name without +rewriting. + +This bucket, like GHCR and ECR, is a byte mirror only — it is never a checksum authority. The +CLI does not fetch `SHA256SUMS` (from here or from GitHub) at runtime: it pins each target's +archive and manifest sha256 directly in `packages/stack/src/Artifacts.ts`, written by +`.github/scripts/sync-artifacts-catalog.ts` after that script has already verified this bucket's +copy against the release's committed `SHA256SUMS` (see +[ADR 0026](../../docs/adr/0026-slim-artifact-mirrors.md)). A stale object here fails that sync, +not a CLI download. ## How it deploys From 2eb829d9bfe0063ad1c38e41a455053a32ccf3bf Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Tue, 29 Sep 2026 11:08:25 +0200 Subject: [PATCH 08/45] fix(cli): share the digest-aware image tag helper --- .../src/shared/services/services.shared.ts | 19 ++++++++--------- .../services/services.shared.unit.test.ts | 21 +++++++++++++++++++ apps/cli/src/shared/services/slim-images.ts | 2 +- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/apps/cli/src/shared/services/services.shared.ts b/apps/cli/src/shared/services/services.shared.ts index a6a07dd1cc..c0726d778f 100644 --- a/apps/cli/src/shared/services/services.shared.ts +++ b/apps/cli/src/shared/services/services.shared.ts @@ -14,7 +14,12 @@ import { parseDockerfileServiceImages, type DockerfileImageSpec, } from "./dockerfile-images.ts"; -import { slimImageForAlias, slimImageForCurrentPin, slimImagesEnabled } from "./slim-images.ts"; +import { + replaceImageTag, + slimImageForAlias, + slimImageForCurrentPin, + slimImagesEnabled, +} from "./slim-images.ts"; export { parseDockerfileServiceImages } from "./dockerfile-images.ts"; @@ -132,14 +137,6 @@ export function postgresImageForDbMajorVersion(majorVersion: number): string | u } } -function replaceImageTag(image: string, tag: string): string { - const index = image.lastIndexOf(":"); - if (index === -1) { - return image; - } - return `${image.slice(0, index + 1)}${tag.trim()}`; -} - /** Applies that service's image-tag prefix when the version does not already start with it. */ export function tagForServiceVersion(service: LocalServiceVersionName, version: string): string { const trimmed = version.trim(); @@ -178,9 +175,11 @@ function localServiceImagesForOptions( if (version === undefined || version.trim().length === 0) { return baseImage === service.image ? service : { ...service, image: baseImage }; } + // `slim-images.ts`'s `replaceImageTag` doesn't trim (its own callers already do), so this + // path — the only one that skips `tagForServiceVersion`'s trim — trims here. const pin = normalizeVersionTags ? tagForServiceVersion(service.localService, version) - : version; + : version.trim(); if (override === undefined && slim) { return { ...service, diff --git a/apps/cli/src/shared/services/services.shared.unit.test.ts b/apps/cli/src/shared/services/services.shared.unit.test.ts index 72f9e38c0e..d41982a1a3 100644 --- a/apps/cli/src/shared/services/services.shared.unit.test.ts +++ b/apps/cli/src/shared/services/services.shared.unit.test.ts @@ -181,6 +181,27 @@ describe("services shared", () => { ); }); + // A digest-carrying override paired with a serviceVersions pin exercises the same + // `replaceImageTag` used for the plain-tag case above; it must drop the stale digest + // rather than splice the new tag into it (`…-r0@sha256:`). + test("rewrites the tag on a digest-carrying image override, dropping the stale digest", () => { + vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); + const rows = listLocalServiceVersions({ + imageOverrides: { + postgres: + "ghcr.io/supabase/cli/postgres:17.6.1.173-r0@sha256:24e96b8d5daf90f67a62b5593d3008446744007e0a57e302d269c02a4e459e8f", + }, + normalizeVersionTags: false, + serviceVersions: { postgres: "17.6.1.200" }, + }); + + expect(rows).toContainEqual({ + name: "ghcr.io/supabase/cli/postgres", + local: "17.6.1.200", + remote: "", + }); + }); + test("can preserve raw local service version overrides", () => { expect( listLocalServiceVersions({ diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index a1fce543a6..f0f2d5d75b 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -141,7 +141,7 @@ export function imageTag(image: string): string | undefined { } /** Replaces `image`'s tag with `tag`, dropping any `@sha256:…` digest — a new tag invalidates it. */ -function replaceImageTag(image: string, tag: string): string { +export function replaceImageTag(image: string, tag: string): string { const withoutDigest = image.split("@")[0] ?? image; const tagSeparator = withoutDigest.lastIndexOf(":"); return tagSeparator === -1 ? image : `${withoutDigest.slice(0, tagSeparator + 1)}${tag}`; From 3341d7ce084682fc50ef37f0ca2ac6e4545f6718 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 12:13:37 +0200 Subject: [PATCH 09/45] fix(ci): support moving a catalog pin to a new upstream version Manual sync mode located an additional (release-line-keyed) pin's span by content, but only ever rewrote the pin literal's own upstreamVersion field, leaving the entry's string key behind on the old upstream version. That desyncs resolveArtifact/postgresVersion lookups, which key off the object's literal keys. Track each additional pin's key span and rewrite it alongside the pin when the upstream version moves. --- .../scripts/sync-artifacts-catalog.test.ts | 76 ++++++++++++++++--- .github/scripts/sync-artifacts-catalog.ts | 33 ++++++-- 2 files changed, 93 insertions(+), 16 deletions(-) diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index 7d264629f0..599c09d529 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -527,6 +527,60 @@ describe("refreshCatalogPin", () => { expect(refreshed.source).toContain('placeholderPin("postgres", "17.6.1.168")'); expect(refreshed.source).toContain(nextDigests["linux-arm64"].manifest); }); + + test("moves the Postgres 15 additional pin to a new upstream version, updating its key", async () => { + const digests = nativeDigests("u"); + const io: RevisionIo = { + listReleaseTags: async () => ["postgres-15.19.0.002-r0"], + fetchChecksums: async () => checksumsFor("postgres", "15.19.0.002-r0", digests), + imageDigest: async () => digest("v"), + s3Sha256: matchingS3("postgres", "15.19.0.002-r0", digests), + }; + + const written = await refreshCatalogPin({ + catalog: fixture, + service: "postgres", + upstream: "15.19.0.002", + io, + }); + expect(written.update).toEqual({ + service: "postgres", + version: "15.19.0.002", + revision: 0, + previousVersion: "15.14.1.168", + target: "additional", + }); + // The old key is gone; the new key matches the resolved pin's `upstreamVersion`. + expect(written.source).not.toContain('"15.14.1.168"'); + expect(written.source).toContain('"15.19.0.002": { upstreamVersion: "15.19.0.002"'); + // The default (17.x) postgres line is untouched. + expect(written.source).toContain('placeholderPin("postgres", "17.6.1.168")'); + + // The renamed key resolves the entry again after real formatting, e.g. for a later hotfix. + const formatted = await formatWithOxfmt(written.source); + const nextDigests = nativeDigests("w"); + const nextIo: RevisionIo = { + listReleaseTags: async () => ["postgres-15.19.0.002-r0", "postgres-15.19.0.002-r1"], + fetchChecksums: async () => checksumsFor("postgres", "15.19.0.002-r1", nextDigests), + imageDigest: async () => digest("x"), + s3Sha256: matchingS3("postgres", "15.19.0.002-r1", nextDigests), + }; + const refreshed = await refreshCatalogPin({ + catalog: formatted, + service: "postgres", + upstream: "15.19.0.002", + io: nextIo, + }); + + expect(refreshed.update).toEqual({ + service: "postgres", + version: "15.19.0.002", + revision: 1, + previousVersion: "15.19.0.002", + target: "additional", + }); + expect(refreshed.source).toContain(nextDigests["linux-arm64"].manifest); + }); }); describe("against the real catalog", () => { @@ -581,26 +635,30 @@ describe("against the real catalog", () => { test("a real catalog entry survives real formatting and can be refreshed again", async () => { const catalog = await Bun.file(CATALOG_PATH).text(); + const pinnedVersion = /definition\(\s*"auth",\s*\{\s*upstreamVersion:\s*"([^"]+)"/.exec( + catalog, + )?.[1]; + if (pinnedVersion === undefined) throw new Error("auth pin not found in the real catalog"); const first = nativeDigests("q"); const firstIo: RevisionIo = { - listReleaseTags: async () => ["auth-v2.196.0-r0"], - fetchChecksums: async () => checksumsFor("auth", "v2.196.0-r0", first), + listReleaseTags: async () => [`auth-${pinnedVersion}-r0`], + fetchChecksums: async () => checksumsFor("auth", `${pinnedVersion}-r0`, first), imageDigest: async () => digest("r"), - s3Sha256: matchingS3("auth", "v2.196.0-r0", first), + s3Sha256: matchingS3("auth", `${pinnedVersion}-r0`, first), }; const written = await refreshCatalogPin({ catalog, service: "auth", io: firstIo }); expect(written.update?.revision).toBe(0); const formatted = await formatWithOxfmt(written.source); - expect(formatted).toContain('upstreamVersion: "v2.196.0"'); + expect(formatted).toContain(`upstreamVersion: "${pinnedVersion}"`); const second = nativeDigests("s"); const secondIo: RevisionIo = { - listReleaseTags: async () => ["auth-v2.196.0-r0", "auth-v2.196.0-r1"], - fetchChecksums: async () => checksumsFor("auth", "v2.196.0-r1", second), + listReleaseTags: async () => [`auth-${pinnedVersion}-r0`, `auth-${pinnedVersion}-r1`], + fetchChecksums: async () => checksumsFor("auth", `${pinnedVersion}-r1`, second), imageDigest: async () => digest("t"), - s3Sha256: matchingS3("auth", "v2.196.0-r1", second), + s3Sha256: matchingS3("auth", `${pinnedVersion}-r1`, second), }; const refreshed = await refreshCatalogPin({ catalog: formatted, @@ -610,9 +668,9 @@ describe("against the real catalog", () => { expect(refreshed.update).toEqual({ service: "auth", - version: "v2.196.0", + version: pinnedVersion, revision: 1, - previousVersion: "v2.196.0", + previousVersion: pinnedVersion, target: "default", }); expect(refreshed.source).toContain("revision: 1"); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index 488a2a0d55..5654547f90 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -240,6 +240,11 @@ interface PinSpan { readonly start: number; readonly end: number; readonly version: string; + /** + * Span of an additional (release-line-keyed) pin's own string key, content only (no quotes). + * Absent for the default pin, which carries no separate key to keep in sync. + */ + readonly key?: { readonly start: number; readonly end: number }; } /** @@ -337,7 +342,9 @@ function collectServicePins(source: string, service: string): ServicePins | unde keyPattern.lastIndex = valueStart; continue; } - additional.push(pin); + const keyText = keyMatch[1] ?? ""; + const keyStart = keyMatch.index + keyMatch[0].indexOf(keyText); + additional.push({ ...pin, key: { start: keyStart, end: keyStart + keyText.length } }); keyPattern.lastIndex = pin.end; } @@ -495,7 +502,12 @@ function slimVersions(dockerfile: string): ReadonlyMap { const normalizeText = (text: string): string => text.replace(/\s+/g, " ").trim(); -/** Writes `pin` over `entry`'s span in `source`, or returns `source` unchanged when it already matches. */ +/** + * Writes `pin` over `entry`'s span in `source`, or leaves it unchanged when it already matches. + * An additional (release-line-keyed) pin also gets its own string key rewritten to `pin`'s + * `upstreamVersion` when it moves to a different upstream version, so the key an additional pin + * is looked up by never drifts from the `upstreamVersion` its resolved pin literal carries. + */ function writePin( source: string, entry: Extract, @@ -503,11 +515,18 @@ function writePin( ): { readonly source: string; readonly changed: boolean } { const desired = serializePin(pin); const current = source.slice(entry.span.start, entry.span.end); - if (normalizeText(current) === normalizeText(desired)) return { source, changed: false }; - return { - source: source.slice(0, entry.span.start) + desired + source.slice(entry.span.end), - changed: true, - }; + let next = source; + let changed = false; + if (normalizeText(current) !== normalizeText(desired)) { + next = next.slice(0, entry.span.start) + desired + next.slice(entry.span.end); + changed = true; + } + const key = entry.span.key; + if (key !== undefined && source.slice(key.start, key.end) !== pin.upstreamVersion) { + next = next.slice(0, key.start) + pin.upstreamVersion + next.slice(key.end); + changed = true; + } + return { source: next, changed }; } /** From 4eb4e52afaa9e5249e3c76d3a2422265bf080287 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 12:13:42 +0200 Subject: [PATCH 10/45] feat(stack): pin the artifact catalog to the newest slim-services revisions slim-services has published immutable -r0 releases for every service. Replace every placeholder pin in Artifacts.ts with the real content pin (image digest, archive and manifest sha256 per native target) at the newest -r0, using the sync script's manual mode so each pin's S3 staleness check actually ran. Drop PLACEHOLDER_PINS/placeholderPin now that nothing needs them, and turn the "catalog has no placeholder pins" test from it.fails into a normal passing assertion. --- packages/stack/src/Artifacts.ts | 364 ++++++++++++++++-- .../slim-services.integration.test.ts | 11 +- 2 files changed, 329 insertions(+), 46 deletions(-) diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 57e597cd8e..0bd6497587 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -94,59 +94,347 @@ const definition = ( const SLIM_IMAGE_GHCR_REGISTRY = "ghcr.io/supabase/cli/"; -/** Stand-in digest of every catalog pin that no published revision backs yet. */ -export const PLACEHOLDER_PINS = "0".repeat(64); - -const placeholderPin = (repository: string, upstreamVersion: string): ArtifactPin => { - const native = { archive: PLACEHOLDER_PINS, manifest: PLACEHOLDER_PINS }; - return { - upstreamVersion, - revision: 0, - image: `${SLIM_IMAGE_GHCR_REGISTRY}${repository}:${upstreamVersion}-r0@sha256:${PLACEHOLDER_PINS}`, - natives: { "darwin-arm64": native, "linux-amd64": native, "linux-arm64": native }, - }; -}; - const definitions: Readonly> = { database: definition( "postgres", - placeholderPin("postgres", "17.6.1.173"), + { + upstreamVersion: "17.11.0.002", + revision: 0, + image: + "ghcr.io/supabase/cli/postgres:17.11.0.002-r0@sha256:5a551a204a41267c4f78a2e5b34657f5c6fdd39eddccd50262466481b95739cc", + natives: { + "darwin-arm64": { + archive: "10410e402c77210a0543539d9bafcabd0c04923d8e796928cbd2a053cf7b4f4f", + manifest: "6c1f236324f02baf472152aafed68884cb115b3979fbf1c49209717485d36ac2", + }, + "linux-amd64": { + archive: "4a4410791bdeeda2e08fda400039b4319bb26d74e08e68951f38abc9dacd7c26", + manifest: "cd76249031db380831d773acea6fea8a54104d5ca97bb18db34d05e7bd124e5f", + }, + "linux-arm64": { + archive: "1d54954b2441990643f25a825e4e46ec50ecf538a63baa8e413e353e7faeb939", + manifest: "8dda36ffdb7b5b501873ff41c265bb061e8eb9aeed0e4372c5ad9af59110776b", + }, + }, + }, "bin/supabase-postgres-start", ["bin/supabase-postgres-start", "bin/pg_dump", "bin/pg_dumpall", "bin/pg_prove", "bin/psql"], - { "15.14.1.173": placeholderPin("postgres", "15.14.1.173") }, + { + "15.19.0.002": { + upstreamVersion: "15.19.0.002", + revision: 0, + image: + "ghcr.io/supabase/cli/postgres:15.19.0.002-r0@sha256:a3f343f19323497a5766fa4e86a51dba495a8eca354cc128d879b4632c6bd017", + natives: { + "darwin-arm64": { + archive: "59b05ca76db9d807803840264d885a5ff435e552f294d7f22ca831fac1ffb4a3", + manifest: "b7956520ee15a8265635cabb318490f89c4737706f9349246da9921a17fbac07", + }, + "linux-amd64": { + archive: "bdf565e0b866ca7b2494b56b502963ef57d54ab1c99eac6855f760caaffda712", + manifest: "e1ecda082f1c414fea6356d0399eaf76d22a25b96808961a945aacb242f904c9", + }, + "linux-arm64": { + archive: "c7420f0eb0938397d90906265630359313936cc576035cb31c2bdef6451565c6", + manifest: "207feac260bcbc5607decca9e14e0422accc4a0ae1f7d24b8fe0992d284cc43c", + }, + }, + }, + }, ), - rest: definition("postgrest", placeholderPin("postgrest", "v16.2"), "bin/postgrest"), - auth: definition("auth", placeholderPin("auth", "v2.196.0"), "bin/auth"), - realtime: definition("realtime", placeholderPin("realtime", "v2.134.5"), "bin/server", [ + rest: definition( + "postgrest", + { + upstreamVersion: "v16.4", + revision: 0, + image: + "ghcr.io/supabase/cli/postgrest:v16.4-r0@sha256:63a8d4acfdeb107b6568f4582759c78072100ef07951a7fbe58c9a51241138a7", + natives: { + "darwin-arm64": { + archive: "a1f5449d739404cbd042ec9dbabadea6dcf810db636e44c7176b859d5e78ab07", + manifest: "943cbeb7a3cf2f9dd0406152a178d814e81221d0061626b04ab2b042209d5e03", + }, + "linux-amd64": { + archive: "d9170378062dba1188c25fb05cd08a3397bb62afd63151812f831eed4fab1051", + manifest: "a3f3f29c4c4c5f7b004b507152f29316bc79df052652fb00ad5ad9bdd6f7be29", + }, + "linux-arm64": { + archive: "c1a99eddaac0c005b2520960d258c7fc79945ba0703eff3a2515aa119fac4477", + manifest: "7ced80e3f814d9748763983de74706c6e04a897f8c024366af156c8093cf150f", + }, + }, + }, + "bin/postgrest", + ), + auth: definition( + "auth", + { + upstreamVersion: "v2.197.0", + revision: 0, + image: + "ghcr.io/supabase/cli/auth:v2.197.0-r0@sha256:7eb303831d170865840dbb3f9018b48561c988704ad1d9a2bafabb4e93e92da5", + natives: { + "darwin-arm64": { + archive: "eb365c4aea1e1cd04998ed16cd49b1b90e6a097ce633e5ee55f8bbbebdd349c7", + manifest: "178f748da1c886b07945a67a839d3a17fa546e01215a90329ded1a634831deb6", + }, + "linux-amd64": { + archive: "bd8f59ed1a817014afe71288c2275ad0396630f016f8cb5ab5129388860ec49b", + manifest: "ccad45ee54edf7506b055b85dcebee6db8c5b9ec57ac3c6d30baf38bb81b1deb", + }, + "linux-arm64": { + archive: "9598a8dd5a08707f65795505211c479b686c54e3dadbe64b3e1989855215cf0c", + manifest: "4f5f3a6284f0b161cf49a201fe551470d5b6e0e2b37dedb4dacdd0e42abaf3b1", + }, + }, + }, + "bin/auth", + ), + realtime: definition( + "realtime", + { + upstreamVersion: "v2.140.3", + revision: 0, + image: + "ghcr.io/supabase/cli/realtime:v2.140.3-r0@sha256:af7b883647770351eef150cc16e1f12d7fe4b4920324e2e40487961ad3b22789", + natives: { + "darwin-arm64": { + archive: "e015ced15ed2110c2a0123b6e75edb2aeb12ac6bdbacffdfb296262a4215f97e", + manifest: "a833a92e7b2e34b6bfcf41c4c934587f85e96eb11acb9cce14852b8ffec312d6", + }, + "linux-amd64": { + archive: "d528a255b2cb20df25ab9440e29f026e2fb3c993e4ed7b0707e037d0cb3880d9", + manifest: "fc5445d4b28d41691091c8cdccde7cff0dbd06af2bf56e70401ec6a049fb8757", + }, + "linux-arm64": { + archive: "bb3b810540dbba2f9a94eb974799384cbffe67756cd91ebb3b8254d02d890e2e", + manifest: "1a33f5620a39a64b49645c25878a892122d695f8f1a26d3548fb5332d6e0d88b", + }, + }, + }, "bin/server", - "bin/prepare", - ]), - storage: definition("storage", placeholderPin("storage", "v1.73.0"), "bin/storage", [ + ["bin/server", "bin/prepare"], + ), + storage: definition( + "storage", + { + upstreamVersion: "v1.79.26", + revision: 0, + image: + "ghcr.io/supabase/cli/storage:v1.79.26-r0@sha256:277bac6513c4b6d5e45dc80a5a98021a407d83852393fd368585e51e0d11e41b", + natives: { + "darwin-arm64": { + archive: "5039b176ed7fce641c9bb94a6cfb232450775680c7bb268f50910b52b4bd5047", + manifest: "26e6ab22229cfc3a1ae297e007529f2fe1e7588dd0885c1ad14e6b5922dd1624", + }, + "linux-amd64": { + archive: "70040e165f94557144fdd4a4f6005c39300cb3c0071f5432b78a1c48bad1ec81", + manifest: "4d6e8f14356d2380b8455e5aa22ba70f1570e7530b3960ee582c8a9ff19ccfe3", + }, + "linux-arm64": { + archive: "34aadf86e58c773829ea4b2d7573ce971456248b4e40d925d5c7a046d4202276", + manifest: "c48ae83317e17158798cc59b78966b7717b18954925fb0b479fbc3584df301d7", + }, + }, + }, "bin/storage", - "bin/prepare", - ]), - imgproxy: definition("imgproxy", placeholderPin("imgproxy", "v3.8.0"), "bin/imgproxy"), + ["bin/storage", "bin/prepare"], + ), + imgproxy: definition( + "imgproxy", + { + upstreamVersion: "v3.26.0", + revision: 0, + image: + "ghcr.io/supabase/cli/imgproxy:v3.26.0-r0@sha256:5871582cf6c5140d3b50b21e0d1aa236fdf46ee6133e3bfec1baf63545e01cf0", + natives: { + "darwin-arm64": { + archive: "1f82084b056bf4806bf492bca0b6f963068d07a6776a2a122b61da940732683b", + manifest: "53b5c4f4f069db88c3919571dbee12af2373e5c90db291cf5d7456aa74e17ecb", + }, + "linux-amd64": { + archive: "7865edf054b8217c2daf9d42bce6bebe34f83df5c4183fbbe6495a3603dcd4a8", + manifest: "5ae7f22a6fbdc55f41ffc3334747a3475295676699ac1d2c372bc5fe2ad239bb", + }, + "linux-arm64": { + archive: "4528fd7877c9df3052cd6ec06c8cafd7e92977dd26490fb532964c19faec3b0a", + manifest: "5420e6d9b3661060b2dedfe157c18b56f22011c43cc3127031b4bbc65e936b59", + }, + }, + }, + "bin/imgproxy", + ), functions: definition( "edge-runtime", - placeholderPin("edge-runtime", "v1.77.1"), + { + upstreamVersion: "v1.77.1", + revision: 0, + image: + "ghcr.io/supabase/cli/edge-runtime:v1.77.1-r0@sha256:98582ce39914bba6ac856a5bf6ed3c1584ce4616687ca359c8acd5f59dd9ff59", + natives: { + "darwin-arm64": { + archive: "e52fdbedccf258fd9a427b19bd34569ca210c1ca257a85fc23f66b4e18c1d6a1", + manifest: "ef2590d8119fe1c2c877746ef72f58ed4d935ce23647b893fd5d36892b0496e0", + }, + "linux-amd64": { + archive: "e524630f0743319d77535f93748a894520fd52cf9c8fa8c2adb22e08ddb39419", + manifest: "3ff18dc0667373dfc108da7d206dbfbf844f2596a804319343d5793298416e9c", + }, + "linux-arm64": { + archive: "c0fe416e506087afd75771beeeb6cfd070d4d6056293a940e3bf453637ba7a5f", + manifest: "703581ebe6213c484955313fbec3ea6ad5f20a86f0c48eb0e29d1bbe9d1c286b", + }, + }, + }, "bin/edge-runtime", ), - studio: definition("studio", placeholderPin("studio", "2026.09.04-sha-5a67366"), "bin/studio"), - pgmeta: definition("pgmeta", placeholderPin("pgmeta", "v0.99.0"), "bin/pgmeta"), - mail: definition("mailpit", placeholderPin("mailpit", "v1.30.2"), "bin/mailpit"), - analytics: definition("analytics", placeholderPin("analytics", "v1.50.9"), "bin/logflare", [ + studio: definition( + "studio", + { + upstreamVersion: "2026.09.28-sha-5e59b60", + revision: 0, + image: + "ghcr.io/supabase/cli/studio:2026.09.28-sha-5e59b60-r0@sha256:4cf4f70978bb0866d1644b43cb0d23acc4b4ef7a898592043ad0d13cad8059be", + natives: { + "darwin-arm64": { + archive: "37f420f6af3d5ee7dab884e8c39fe2c3bcddcee0d90e53d1e075a05ae3cb4b73", + manifest: "b5d6209c4c28e594cc8b0ab961105465418f0021ba50a3fd92067ff5720ae820", + }, + "linux-amd64": { + archive: "19a9903732b71fba04ce342e4d13b83bf7579bce806981cf8f2b491ad792fc6e", + manifest: "5a4f1e316ad84ff058263601b9b144177ab0cd18443e826354551aec88013199", + }, + "linux-arm64": { + archive: "ce69544c9ec5dbae50e5da731770bc199e12f0f1d63b22ec7a43b6a0ba00cb4c", + manifest: "c27374ccc51cc6419f9bbd14f095f1a16db4bda369928effcc15a6366dcb3d1f", + }, + }, + }, + "bin/studio", + ), + pgmeta: definition( + "pgmeta", + { + upstreamVersion: "v0.99.0", + revision: 0, + image: + "ghcr.io/supabase/cli/pgmeta:v0.99.0-r0@sha256:c19f6bba3ab66fcf30c8737d2361ec9f8e12a4fa8a071481f53366dc13e83340", + natives: { + "darwin-arm64": { + archive: "337f8cc6a23d93f3f9cfeed12de2a86d686ce6f4346ff9b334b5dfdcba7e890f", + manifest: "6b35ee42d334138562444842ed0662b97b3cbde504caa11e97407b137bc8b2ca", + }, + "linux-amd64": { + archive: "43156ba28901710bf02a333dc04c4b30754eb6a2334ff1d890a4a3ea55c02f22", + manifest: "dbd8a7eac705b6c6df16826f22f3317906842be8498ce44fa1229ae6f16273ad", + }, + "linux-arm64": { + archive: "a8565d6e550efa8a8481c7d542b612e7a6d50668321fcd2e668b2ee9bcb28ed7", + manifest: "1b67627c5ccde0f94a4997ab223a0ac072c70e9bf1bb86d3e4d0df42e90b4059", + }, + }, + }, + "bin/pgmeta", + ), + mail: definition( + "mailpit", + { + upstreamVersion: "v1.31.3", + revision: 0, + image: + "ghcr.io/supabase/cli/mailpit:v1.31.3-r0@sha256:ed9b00c609e77e99c79b93f1178255ebc271868920f2c69a8d166bd5634ed10d", + natives: { + "darwin-arm64": { + archive: "d460a6a55693a2a321ad83ebb83417b8078324751a744f982bb3512da0632712", + manifest: "43177dc88b6f625bec6d98096e39c10f4cd10956081df01b5b7cebeeae47766a", + }, + "linux-amd64": { + archive: "57e39cb39b2288313e26abe9ae9300f6af9332abd1f76f4d67be533741b2c942", + manifest: "36595519f40e406b17a0ce6e21f153400f217ae24b0365f0a5c495ff315b9f82", + }, + "linux-arm64": { + archive: "805b09d9008e9be2c6c9230e668bd2c6ab446eeae26389e91732f9e20b2c3f5f", + manifest: "973b76700a5a865243bd5a5c36664121ed99c538d58633921cc8fd7c675e655c", + }, + }, + }, + "bin/mailpit", + ), + analytics: definition( + "analytics", + { + upstreamVersion: "v1.50.15", + revision: 0, + image: + "ghcr.io/supabase/cli/analytics:v1.50.15-r0@sha256:cea1595bafa6ab32df4854d407261ddef6e35394e300ae29770ba48c49c8dc7e", + natives: { + "darwin-arm64": { + archive: "336c78c501aff270b0fcd2a42229b76d9f2feafa646f49a59c3de29358e3de17", + manifest: "d0e1a0c3fa0a4ebfcebbe490f290699424e3b8c92a542f50111ff88d6e47b8b5", + }, + "linux-amd64": { + archive: "a699ce1522dac43989987a9d152648ed0506fb887ffedff49c148da02cbbc7e7", + manifest: "ba8c55eeb06be91ad582156854babdc6a06d7a26fa6956bc1271eee4cae9d536", + }, + "linux-arm64": { + archive: "bb977b9cd0623e1b1ece9875377fdf29577f96991d2e5c72a38a1b664c6a7538", + manifest: "1739bbd4afb838e62b6191c4de8aa39685df9f5d83e1b7cc303bd096237ba08c", + }, + }, + }, "bin/logflare", - "bin/prepare", - ]), - vector: definition("vector", placeholderPin("vector", "0.53.0"), "bin/vector", [ + ["bin/logflare", "bin/prepare"], + ), + vector: definition( + "vector", + { + upstreamVersion: "0.58.0", + revision: 0, + image: + "ghcr.io/supabase/cli/vector:0.58.0-r0@sha256:5dcf67db0ee378caa87f3395cb9484ebe3e97bb0334d119f2ac33116e00c5773", + natives: { + "darwin-arm64": { + archive: "567245cf9a7d54eee45ecf74e1c9a61ca6d02cdca103edc7b32b005e54f4e632", + manifest: "a973a763b00599858ceae8f304714fb99f785860112e9e0812ef0df8f81dd2ee", + }, + "linux-amd64": { + archive: "697f4fae35be3026474695bef16336f6fcfd429ce8cfba884c595896e96c30ec", + manifest: "8989b8b061f08bd7653e9ae71c6ee0e5cf01a0b10f2d358fe3dbc671ec5b63e1", + }, + "linux-arm64": { + archive: "c66b8ad0a0dd0fdcb3e4ee36bae8040b7b23b44ec2b4023565ca335884268c1d", + manifest: "992467ec68a99a6413468b9311294abb7a1f86b7857ad37f1f3bd9e6aecc9dbb", + }, + }, + }, "bin/vector", - "share/doc/vector/config/vector.yaml", - ]), - pooler: definition("pooler", placeholderPin("pooler", "v2.9.12"), "bin/server", [ + ["bin/vector", "share/doc/vector/config/vector.yaml"], + ), + pooler: definition( + "pooler", + { + upstreamVersion: "v2.9.13", + revision: 0, + image: + "ghcr.io/supabase/cli/pooler:v2.9.13-r0@sha256:3a32b56d03675ed24e84408afcbb12fa52b7ec6e7741f913770fea5b66c2ddd3", + natives: { + "darwin-arm64": { + archive: "c05285be2a945a29d5be491661926f8c88d976540d49ddbcab10aa7276b29934", + manifest: "0fafa8dcf601ff3cea7326e82f7a5191ffe03e3e4edd0d2b155e80e8c9f53d78", + }, + "linux-amd64": { + archive: "d94e36f44267b4159fa55e1aea39320a68789b6af07f9cc87c27327ed6a00602", + manifest: "4f181d1d25da91f72ca37c22cdc46278a424a4edef6bdd2ea2c30f463c629a5a", + }, + "linux-arm64": { + archive: "124f3e6c95e6ba985239e013cab76c5be28fbd92cb4986b687dd5e827f0525ca", + manifest: "c6cdc298bc3a00d082a1c9c34f4131f13437df51fb0bf9a0ce857b0a2fd66460", + }, + }, + }, "bin/server", - "bin/prepare", - "bin/provision-tenant", - ]), + ["bin/server", "bin/prepare", "bin/provision-tenant"], + ), }; const targetForPlatform = (platform: { diff --git a/packages/stack/src/preparation/slim-services.integration.test.ts b/packages/stack/src/preparation/slim-services.integration.test.ts index d4e043f415..c321aae3c1 100644 --- a/packages/stack/src/preparation/slim-services.integration.test.ts +++ b/packages/stack/src/preparation/slim-services.integration.test.ts @@ -17,12 +17,7 @@ import { import { createServer, type Server } from "node:http"; import { zstdCompress } from "node:zlib"; import { FetchHttpClient } from "effect/unstable/http"; -import { - catalogPins, - PLACEHOLDER_PINS, - prepareNativeArtifact, - resolveArtifact, -} from "../Artifacts.ts"; +import { catalogPins, prepareNativeArtifact, resolveArtifact } from "../Artifacts.ts"; import { makeArtifactStore, type ArtifactRequest, type ArtifactSource } from "./ArtifactStore.ts"; import { digestHex } from "./Integrity.ts"; import { @@ -867,11 +862,11 @@ describe("native artifact catalog", () => { ), ); - it.fails("catalog has no placeholder pins", () => { + it("catalog has no placeholder pins", () => { const digests = catalogPins().flatMap(({ pin }) => [ pin.image.slice(pin.image.lastIndexOf(":") + 1), ...Object.values(pin.natives).flatMap((native) => [native.archive, native.manifest]), ]); - expect(digests).not.toContain(PLACEHOLDER_PINS); + expect(digests).not.toContain("0".repeat(64)); }); }); From 823d77fac7c2c4e004286537e98b8ba483fbfa95 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 12:13:47 +0200 Subject: [PATCH 11/45] fix(stack): allow vector's own env-var config interpolation Vector 0.58 (up from 0.53) disabled \${VAR} interpolation in config files by default; the API config's address, sourced from our own VECTOR_API_ADDRESS env var, was left as a literal unexpanded string, so vector rejected it with "api.address: invalid socket address syntax" and exited before it was ready. Pass --dangerously-allow-env-var-interpolation: the interpolated value is stack-controlled, not user-supplied config. --- packages/stack/src/services/Vector.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/stack/src/services/Vector.ts b/packages/stack/src/services/Vector.ts index da59642e4b..23314bcafb 100644 --- a/packages/stack/src/services/Vector.ts +++ b/packages/stack/src/services/Vector.ts @@ -99,11 +99,14 @@ const makeSpec = ( }), ), args: (creation, _endpoints, context) => - Effect.succeed( - context.container + Effect.succeed([ + // Vector 0.58 disabled `${VAR}` config interpolation by default; the API address is our + // own `VECTOR_API_ADDRESS` env var, not user-supplied config, so opting back in is safe. + "--dangerously-allow-env-var-interpolation", + ...(context.container ? ["--config", containerPipelinePath, "--config", containerApiPath] - : ["--config", pipelinePath(creation), "--config", apiConfigPath], - ), + : ["--config", pipelinePath(creation), "--config", apiConfigPath]), + ]), mounts: (creation) => Effect.succeed([ { source: pipelinePath(creation), target: containerPipelinePath, readOnly: true }, From 49b5533946836fe7590aa50929f4b29e2eaf7ffc Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 12:13:57 +0200 Subject: [PATCH 12/45] test: update fixtures pinned to the previous catalog versions Several stack and CLI integration tests hardcoded upstream versions (or, for auth, derived the real catalog's pinned version rather than hardcoding it) that matched the old placeholder catalog. Now that the catalog resolves real -r0 pins at newer upstream versions, update those fixtures so the tests exercise real artifact resolution again instead of failing on "Unsupported artifact version" or a stale saved-stack-binding/readiness-marker mismatch. --- .../commands/db/start/start.integration.test.ts | 2 +- .../services/services.integration.test.ts | 16 ++++++++-------- ...mandRunner.initialization.integration.test.ts | 6 +++--- .../DatabaseSnapshot.integration.test.ts | 4 ++-- .../services/ProcessRecipe.integration.test.ts | 4 ++-- .../DockerDatabaseStorage.integration.test.ts | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index 2ede82a5ec..cf59bbd77c 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -1577,7 +1577,7 @@ describe("db start stack backend", () => { const databaseCreation: Extract = { service: "database", config: { - version: "17.6.1.173", + version: "17.11.0.002", databasePassword: Redacted.make("secret"), jwtSecret: Redacted.make("secret"), jwtExpiry: 3600, diff --git a/apps/cli/src/commands/services/services.integration.test.ts b/apps/cli/src/commands/services/services.integration.test.ts index c22e86f41a..26720f44d4 100644 --- a/apps/cli/src/commands/services/services.integration.test.ts +++ b/apps/cli/src/commands/services/services.integration.test.ts @@ -333,13 +333,13 @@ describe("services", () => { expect(out.stderrText).toBe(""); expect(rows).toHaveLength(13); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "15.14.1.173" }), + expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "15.19.0.002" }), ); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/mailpit", local: "v1.30.2" }), + expect.objectContaining({ name: "ghcr.io/supabase/cli/mailpit", local: "v1.31.3" }), ); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/vector", local: "0.53.0" }), + expect.objectContaining({ name: "ghcr.io/supabase/cli/vector", local: "0.58.0" }), ); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); @@ -359,7 +359,7 @@ describe("services", () => { const rows = yield* decodeServiceRows(out.stdoutText); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "15.14.1.173" }), + expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "15.19.0.002" }), ); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); @@ -377,10 +377,10 @@ describe("services", () => { const rows = yield* decodeServiceRows(out.stdoutText); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "17.6.1.173" }), + expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "17.11.0.002" }), ); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/auth", local: "v2.196.0" }), + expect.objectContaining({ name: "ghcr.io/supabase/cli/auth", local: "v2.197.0" }), ); expect(out.stderrText).toContain("unsupported PostgreSQL major version: 16"); expect(out.stderrText).toContain("using default stack catalog versions"); @@ -396,7 +396,7 @@ describe("services", () => { const rows = yield* decodeServiceRows(out.stdoutText); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "17.6.1.173" }), + expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "17.11.0.002" }), ); expect(out.stderrText).toMatch(/^failed to read config:/); expect(out.stderrText).toContain("using default stack catalog versions"); @@ -580,7 +580,7 @@ major_version = 15 expect(stackRows).toContainEqual( expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", - local: "17.6.1.173", + local: "17.11.0.002", remote: "17.6.1.200", }), ); diff --git a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts index ea12cd3348..bd97dffa11 100644 --- a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts +++ b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts @@ -96,7 +96,7 @@ it.live.skipIf(target === undefined || process.platform === "win32")( const error = yield* runner .run({ command: initialization.auth({ - version: "v2.196.0", + version: "v2.197.0", databaseUrl: "postgresql://invalid:invalid@127.0.0.1:1/postgres", }), credentials, @@ -139,7 +139,7 @@ it.live.skipIf(target === undefined || process.platform === "win32")( const command = yield* runner .run({ command: initialization.auth({ - version: "v2.196.0", + version: "v2.197.0", databaseUrl: "postgresql://invalid:invalid@127.0.0.1:1/postgres", }), credentials, @@ -197,7 +197,7 @@ it.effect.skipIf(target === undefined || process.platform === "win32")( const command = yield* runner .run({ command: initialization.auth({ - version: "v2.196.0", + version: "v2.197.0", databaseUrl: "postgresql://invalid:invalid@127.0.0.1:1/postgres", }), credentials, diff --git a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts index 4d3bbc131e..4c36d7e624 100644 --- a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts +++ b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts @@ -10,7 +10,7 @@ import { shellQuote } from "../storage/DockerSnapshotBackend.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { makeDatabaseSnapshots, type SnapshotScope } from "./DatabaseSnapshot.ts"; -const version = "17.6.1.173"; +const version = "17.11.0.002"; class ShellError extends Data.TaggedError("ShellError")<{ readonly message: string }> {} @@ -511,7 +511,7 @@ for (const { name, make } of engines) const target = yield* engine.instance("target"); yield* shell( - `printf '%s' ${shellQuote(saved.replace(version, "15.14.1.173"))} > ${descriptor}`, + `printf '%s' ${shellQuote(saved.replace(version, "15.19.0.002"))} > ${descriptor}`, ); expect(yield* target.restoreSnapshot("key")).toBe(false); expect(yield* contents(target.data)).toBe(""); diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index 36909f2216..1e6dcad08a 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -49,12 +49,12 @@ import * as Pooler from "./Pooler.ts"; type TestCreation = RecipeCreation<"rest", Record> & { readonly service: "rest"; - readonly version: "v16.2"; + readonly version: "v16.4"; }; const creation: TestCreation = { service: "rest", - version: "v16.2", + version: "v16.4", config: {}, }; diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index 7a0be70b33..f7bb6d74e7 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -1058,7 +1058,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.writeFileString(path.join(nativeRoot, "data", "PG_VERSION"), "17\n"); yield* fs.writeFileString( path.join(nativeRoot, ".supabase-database-ready.json"), - '{"version":"17.6.1.173","runtime":"native","profile":"supabase"}', + '{"version":"17.11.0.002","runtime":"native","profile":"supabase"}', ); const nativeSnapshots = yield* makeDatabaseSnapshots({ instanceRoot: nativeRoot, From bfa292674c4692c289233e6ec84037b634144b70 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 12:26:12 +0200 Subject: [PATCH 13/45] fix(cli): compare docker.io and slim image tags past the revision suffix dockerImageTag used lastIndexOf(":") directly on the image string, so a slim ref's @sha256: suffix made it return the raw digest hex instead of the tag; a slim ref's own -r revision suffix then also kept a docker.io tag at the bare upstream version from ever comparing equal to it. Both defeated the "tags match but the image family doesn't" staleness check. Strip the digest unconditionally and the -r suffix for a slim ref before comparing. Confirmed against origin/develop with a throwaway worktree: this specific pgdelta.seam.integration.test.ts case passes there, so this branch caused the regression. Root-caused per dockerImageTag's own digest-suffix bug, plus the catalog's real (non-placeholder) postgres pin now differing from the Dockerfile's own pinned version, which the test's fixture catalog mock (mirroring slim-images.unit.test.ts's pattern) now accounts for. --- .../shared/pgdelta.seam.integration.test.ts | 30 +++++++++++++++++++ .../commands/db/shared/pgdelta.seam.layer.ts | 16 ++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts b/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts index 7d2a9ab96e..65d69b1fff 100644 --- a/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts +++ b/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts @@ -33,6 +33,36 @@ import { DeclarativeShadowDbError } from "./pgdelta.errors.ts"; import { declarativeSeamLayer } from "./pgdelta.seam.layer.ts"; import { DeclarativeSeam } from "./pgdelta.seam.service.ts"; +// The real catalog's postgres pin has moved past the Dockerfile's own pinned `pg` tag (the +// "catalog leads the Dockerfile until Dependabot catches up" gap), so `toSlimImage` would find no +// match and fall back to the upstream (non-slim) image regardless of `SUPABASE_USE_SLIM_IMAGES` — +// masking the family-mismatch check below. Pin a fixture catalog entry at the Dockerfile's actual +// `pg` tag (17.6.1.171) instead, the same pattern `slim-images.unit.test.ts` uses. `vi.mock` +// factories are hoisted above every other top-level statement, so the tag is inlined rather than +// referencing `dockerfileServiceImageRaw` here. +vi.mock("@supabase/stack/internal/artifacts", () => { + const digest = "d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c"; + const nativePin = { archive: digest, manifest: digest }; + return { + catalogPins: () => [ + { + service: "database", + sourceService: "postgres", + pin: { + upstreamVersion: "17.6.1.171", + revision: 0, + image: `ghcr.io/supabase/cli/postgres:17.6.1.171-r0@sha256:${digest}`, + natives: { + "darwin-arm64": nativePin, + "linux-amd64": nativePin, + "linux-arm64": nativePin, + }, + }, + }, + ], + }; +}); + /** * Integration coverage for the fully-native `declarativeSeamLayer`: `generate`/`sync`'s own * tests stub `DeclarativeSeam` entirely, so this file is the only place the real local-database diff --git a/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts b/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts index 514fba89f0..60948b08c7 100644 --- a/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts +++ b/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts @@ -281,11 +281,21 @@ type StartLocalDatabaseDeps = ? R : never; +/** Trailing `-r` immutable-revision suffix a slim tag carries; docker.io tags never have one. */ +const REVISION_SUFFIX = /-r(0|[1-9][0-9]*)$/; + +/** + * The tag portion of `image`, ignoring any `@sha256:…` digest suffix and, for a slim ref, its + * `-r` revision suffix — so a slim `-r` tag still compares equal to a docker.io + * container running the same `` version. + */ function dockerImageTag(image: string): string { const trimmed = image.trim(); - const index = trimmed.lastIndexOf(":"); - if (index < 0 || index === trimmed.length - 1) return ""; - return trimmed.slice(index + 1); + const withoutDigest = trimmed.split("@")[0] ?? trimmed; + const index = withoutDigest.lastIndexOf(":"); + if (index < 0 || index === withoutDigest.length - 1) return ""; + const tag = withoutDigest.slice(index + 1); + return isSlimImageRef(trimmed) ? tag.replace(REVISION_SUFFIX, "") : tag; } export function isMissingContainerInspectError(stderr: string): boolean { From beb9300e7a391af81e19a31d48735107ef7a0d4b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 13:14:09 +0200 Subject: [PATCH 14/45] fix(stack): template stack-owned vector config instead of enabling interpolation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --dangerously-allow-env-var-interpolation applies process-wide, covering both --config files vector loads: the recipe's own api config and a caller-suppliable pipeline config (Config.configPath). The prior fix only needed it for the API address, so it was broader than justified. Template the real address into the stack-owned api config directly at write time (now done in `args`, once the launch's real endpoint is known, instead of `prepare`, before it is). The default pipeline config has no `${VAR}` usage, so it needs no rendering. `configPath` is not reachable from user CLI config today (grepped both apps/cli and the stack; `stack-config.ts` never sets it), but the stack's own `tests/whole-stack` fixture does supply one that references `${LOGFLARE_URL}`/`${LOGFLARE_PRIVATE_ACCESS_TOKEN}`, relying on Vector's interpolation. Rather than pass the flag for that case (which would expose the whole process environment, the exact over-broad behavior this fix removes), render only the recipe's own closed set of placeholders into a recipe-owned copy of a caller's pipeline before Vector loads it. The recipe never passes the interpolation flag now, for either config. Supersedes 8014870ac, which framed this as "the address is our own env var, so opting back in is safe" — that justification covered only the api config, not the caller-suppliable pipeline config the same flag also applied to. --- packages/stack/src/services/Vector.ts | 100 +++++++++++++++++++------- 1 file changed, 74 insertions(+), 26 deletions(-) diff --git a/packages/stack/src/services/Vector.ts b/packages/stack/src/services/Vector.ts index 23314bcafb..aeab546a93 100644 --- a/packages/stack/src/services/Vector.ts +++ b/packages/stack/src/services/Vector.ts @@ -20,8 +20,35 @@ export interface Endpoints extends Schema.Schema.Type {} export const Creation = serviceCreation("vector", Config, Endpoints); export interface Creation extends Schema.Schema.Type {} -// Vector has no API flag or env var, so the recipe loads this alongside the pipeline config. -const apiConfig = 'api:\n enabled: true\n address: "${VECTOR_API_ADDRESS}"\n'; +/** + * Vector has no API flag or env var, so the recipe loads this alongside the pipeline config. + * `address` is templated directly into the file content at write time (not through Vector's own + * `${VAR}` config interpolation, which 0.58 disabled by default) — this is the only stack-owned + * config Vector loads that ever varied per launch, so no interpolation flag is needed at all. + */ +const apiConfigFor = (address: string | undefined): string => + address === undefined + ? "api:\n enabled: false\n" + : `api:\n enabled: true\n address: "${address}"\n`; + +/** + * Every `${VAR}` name the recipe's own `env` sets and documents for a pipeline config + * (`Config.analyticsUrl`/`apiKey`, mirrored into `LOGFLARE_URL`/`LOGFLARE_PRIVATE_ACCESS_TOKEN`). + * A caller-supplied pipeline (`Config.configPath`) may reference these the same way the recipe's + * own API config used to. `renderKnownPlaceholders` substitutes only this closed, recipe-owned + * set directly into the file at write time — never a caller's or the process's arbitrary + * environment — so a caller config keeps working on Vector 0.58 without the recipe ever passing + * `--dangerously-allow-env-var-interpolation` (which would expose every env var, not just these). + */ +const renderKnownPlaceholders = ( + content: string, + values: Readonly>, +): string => + Object.entries(values).reduce( + (rendered, [name, value]) => + value === undefined ? rendered : rendered.replaceAll(`\${${name}}`, value), + content, + ); // Vector requires at least one source and sink; the default forwards nothing. const defaultPipelineConfig = [ @@ -73,7 +100,10 @@ const makeSpec = ( const configRoot = path.join(instanceRoot, "runtime", "vector"); const apiConfigPath = path.join(configRoot, "vector-api.yaml"); const defaultPipelinePath = path.join(configRoot, "vector.yaml"); - const pipelinePath = (creation: Creation) => creation.config.configPath ?? defaultPipelinePath; + const renderedPipelinePath = path.join(configRoot, "vector.rendered.yaml"); + /** What Vector actually loads: a caller's file is rendered to a recipe-owned copy first. */ + const resolvedPipelinePath = (creation: Creation) => + creation.config.configPath === undefined ? defaultPipelinePath : renderedPipelinePath; const ownedInstance = (operation: string) => /^[a-zA-Z0-9_-]+$/u.test(instanceId) ? Effect.void @@ -83,33 +113,47 @@ const makeSpec = ( executable: "bin/vector", ports: { http: 9001 }, healthPath: "/health", - env: (creation, endpoints, container) => + env: (creation, _endpoints, _container) => requiredInput("vector", "analyticsUrl", creation.config.analyticsUrl).pipe( - Effect.map((analyticsUrl) => { - const http = endpoints.get("http"); - return { - ...(http === undefined - ? {} - : { VECTOR_API_ADDRESS: `${container ? "0.0.0.0" : "127.0.0.1"}:${http.port}` }), - LOGFLARE_URL: analyticsUrl, - ...(creation.config.apiKey === undefined - ? {} - : { LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey }), - }; - }), + Effect.map((analyticsUrl) => ({ + LOGFLARE_URL: analyticsUrl, + ...(creation.config.apiKey === undefined + ? {} + : { LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey }), + })), ), - args: (creation, _endpoints, context) => - Effect.succeed([ - // Vector 0.58 disabled `${VAR}` config interpolation by default; the API address is our - // own `VECTOR_API_ADDRESS` env var, not user-supplied config, so opting back in is safe. - "--dangerously-allow-env-var-interpolation", - ...(context.container + args: (creation, endpoints, context) => + Effect.gen(function* () { + const http = endpoints.get("http"); + const address = + http === undefined + ? undefined + : `${context.container ? "0.0.0.0" : "127.0.0.1"}:${http.port}`; + yield* writeAtomically(fs, path, apiConfigPath, apiConfigFor(address)); + if (creation.config.configPath !== undefined) { + const source = yield* fs.readFileString(creation.config.configPath).pipe( + Effect.mapError( + (cause) => + new ServiceError({ + operation: "prepare", + message: "Unable to read Vector configPath", + cause, + }), + ), + ); + const rendered = renderKnownPlaceholders(source, { + LOGFLARE_URL: creation.config.analyticsUrl, + LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey, + }); + yield* writeAtomically(fs, path, renderedPipelinePath, rendered); + } + return context.container ? ["--config", containerPipelinePath, "--config", containerApiPath] - : ["--config", pipelinePath(creation), "--config", apiConfigPath]), - ]), + : ["--config", resolvedPipelinePath(creation), "--config", apiConfigPath]; + }), mounts: (creation) => Effect.succeed([ - { source: pipelinePath(creation), target: containerPipelinePath, readOnly: true }, + { source: resolvedPipelinePath(creation), target: containerPipelinePath, readOnly: true }, { source: apiConfigPath, target: containerApiPath, readOnly: true }, ]), startupCommands: [], @@ -139,6 +183,7 @@ const makeSpec = ( const owned = yield* Effect.all([ canonical(apiConfigPath), canonical(defaultPipelinePath), + canonical(renderedPipelinePath), ]).pipe( Effect.mapError( (cause) => @@ -155,7 +200,9 @@ const makeSpec = ( message: "Vector configPath must not point at a stack-owned Vector config file", }); } - yield* writeAtomically(fs, path, apiConfigPath, apiConfig); + // A safe placeholder until `args` writes the real address — the listening port isn't + // known until endpoints are reserved for an actual launch, which happens after `prepare`. + yield* writeAtomically(fs, path, apiConfigPath, apiConfigFor(undefined)); if (callerPath === undefined) yield* writeAtomically(fs, path, defaultPipelinePath, defaultPipelineConfig); }), @@ -165,6 +212,7 @@ const makeSpec = ( yield* ownedInstance("destroy"); yield* fs.remove(apiConfigPath, { force: true }); yield* fs.remove(defaultPipelinePath, { force: true }); + yield* fs.remove(renderedPipelinePath, { force: true }); if (yield* fs.exists(configRoot)) for (const entry of yield* fs.readDirectory(configRoot)) if (entry.startsWith(temporaryPrefix)) From aa77768bed5a0b9e1f15a69ad58a1fc68ba407ab Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 13:14:18 +0200 Subject: [PATCH 15/45] fix(cli): compare slim image revisions when checking the local database image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two issues in ensureLocalPostgresImageCurrent's dockerImageTag: 1. It re-implemented digest stripping (split("@")[0]) and revision stripping (its own REVISION_SUFFIX), one commit after a139af150 unified the digest-aware split into slim-images.ts's imageTag and the revision split into services.shared.ts's upstreamVersionFromTag. Compose those exported helpers instead of a third parser. 2. Revision stripping was unconditional, so a container on postgres:17.11.0.002-r0 compared equal to a catalog pin on -r1 — masking exactly the hotfix-revision drift this staleness check exists to catch. Only strip the revision across families (docker.io vs slim, where a `-r` has no docker.io equivalent); within the same family, compare the digest when both refs carry one, or the full release version otherwise. Added tests for same-family-different-revision (stale, generic remediation), same-family-same-revision (current), alongside the existing docker.io-vs-slim-matching-upstream case (family-mismatch remediation, unchanged). --- .../shared/pgdelta.seam.integration.test.ts | 42 ++++++++++++++++ .../commands/db/shared/pgdelta.seam.layer.ts | 48 +++++++++---------- .../src/shared/services/services.shared.ts | 2 +- apps/cli/src/shared/services/slim-images.ts | 6 +++ 4 files changed, 72 insertions(+), 26 deletions(-) diff --git a/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts b/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts index 65d69b1fff..ba3662a42a 100644 --- a/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts +++ b/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts @@ -237,6 +237,48 @@ describe("declarativeSeamLayer.ensureLocalPostgresImageCurrent", () => { }, ); + it.effect("flags a stale slim container when only its revision has drifted from a hotfix", () => { + vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); + const dir = tmp.current; + // Same upstream version and family as the fixture catalog's pin (17.6.1.171), but at a + // different revision (r1, a different digest) — the hotfix-drift case this model exists + // to catch, and the one a bare upstream-version comparison would mask. + const { layer } = setup(dir, { + dbInspectImage: + "ghcr.io/supabase/cli/postgres:17.6.1.171-r1@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }); + return Effect.gen(function* () { + const seam = yield* DeclarativeSeam; + const exit = yield* seam.ensureLocalPostgresImageCurrent.pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + const error = failError(exit); + expect(error).toBeInstanceOf(DeclarativeShadowDbError); + expect((error as DeclarativeShadowDbError).message).toContain( + "local Postgres container image is stale", + ); + // Same family (slim vs slim): the generic remediation, not the family-mismatch wording. + expect((error as DeclarativeShadowDbError).message).not.toContain( + "same SUPABASE_USE_SLIM_IMAGES setting", + ); + expect((error as DeclarativeShadowDbError).message).toContain("--no-backup"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("passes when a slim container matches the expected image's release and digest", () => { + vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); + const dir = tmp.current; + // The exact image (release version and digest) the fixture catalog pins at r0. + const { layer } = setup(dir, { + dbInspectImage: + "ghcr.io/supabase/cli/postgres:17.6.1.171-r0@sha256:d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c", + }); + return Effect.gen(function* () { + const seam = yield* DeclarativeSeam; + const exit = yield* seam.ensureLocalPostgresImageCurrent.pipe(Effect.exit); + expect(Exit.isSuccess(exit)).toBe(true); + }).pipe(Effect.provide(layer)); + }); + it.effect("bails out when inspect succeeds but the image name is unparseable", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const dir = tmp.current; diff --git a/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts b/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts index 60948b08c7..1223142205 100644 --- a/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts +++ b/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts @@ -9,7 +9,8 @@ import { resolveDbImage } from "../../../command-internal/db-image.ts"; import { readDbToml } from "../../../command-internal/db-config.toml-read.ts"; import { getRegistryImageUrl } from "../../../command-internal/docker-registry.ts"; import { isDockerDaemonUnreachable } from "../../../command-internal/docker-suggest.ts"; -import { isSlimImageRef } from "../../../shared/services/slim-images.ts"; +import { imageDigest, imageTag, isSlimImageRef } from "../../../shared/services/slim-images.ts"; +import { upstreamVersionFromTag } from "../../../shared/services/services.shared.ts"; import { isLocalDbRunning } from "../../../command-internal/db-bootstrap/local-db-running.ts"; import { startLocalDatabase } from "../../../command-internal/db-bootstrap/start-local-database.ts"; import { resolveLocalProjectId, localDbContainerId } from "../../../command-internal/docker-ids.ts"; @@ -250,20 +251,34 @@ export const declarativeSeamLayer = Layer.effect( ), Effect.map((value) => value.trim()), ); - const actualTag = dockerImageTag(actual); - const expectedTag = dockerImageTag(expected); - if (actual.length === 0 || actualTag.length === 0 || expectedTag.length === 0) { + const actualTag = imageTag(actual); + const expectedTag = imageTag(expected); + if (actual.length === 0 || actualTag === undefined || expectedTag === undefined) { return; } // Slim refs never go through a registry mirror, so a family mismatch // (e.g. a docker.io container satisfying a ghcr.io/supabase/cli - // expectation) is stale even when the tags happen to match. + // expectation) is stale even when the upstream versions happen to match. const familyMismatch = isSlimImageRef(expected) !== isSlimImageRef(actual); - if (!familyMismatch && actualTag === expectedTag) { - return; + if (!familyMismatch) { + // Same family: within slim, a `-r` hotfix bump must still be caught, so compare + // the digest when both refs carry one (most precise), or the full tag otherwise — + // never the bare upstream version, which would mask a same-upstream revision drift. + const actualDigest = imageDigest(actual); + const expectedDigest = imageDigest(expected); + const current = + actualDigest !== undefined && expectedDigest !== undefined + ? actualDigest === expectedDigest + : actualTag === expectedTag; + if (current) return; } + // Across families, only the upstream version is comparable (a slim tag's `-r` + // has no docker.io equivalent) — used to pick the remediation wording, not staleness: + // a family mismatch is always stale. + const upstreamMatches = + upstreamVersionFromTag(actualTag) === upstreamVersionFromTag(expectedTag); const remediation = - familyMismatch && actualTag === expectedTag + familyMismatch && upstreamMatches ? "The tags match but the image family does not (slim vs docker.io). Run supabase stop, then supabase start with the same SUPABASE_USE_SLIM_IMAGES setting before syncing declarative schemas." : "Run supabase stop --all --no-backup, then supabase start before syncing declarative schemas."; return yield* new DeclarativeShadowDbError({ @@ -281,23 +296,6 @@ type StartLocalDatabaseDeps = ? R : never; -/** Trailing `-r` immutable-revision suffix a slim tag carries; docker.io tags never have one. */ -const REVISION_SUFFIX = /-r(0|[1-9][0-9]*)$/; - -/** - * The tag portion of `image`, ignoring any `@sha256:…` digest suffix and, for a slim ref, its - * `-r` revision suffix — so a slim `-r` tag still compares equal to a docker.io - * container running the same `` version. - */ -function dockerImageTag(image: string): string { - const trimmed = image.trim(); - const withoutDigest = trimmed.split("@")[0] ?? trimmed; - const index = withoutDigest.lastIndexOf(":"); - if (index < 0 || index === withoutDigest.length - 1) return ""; - const tag = withoutDigest.slice(index + 1); - return isSlimImageRef(trimmed) ? tag.replace(REVISION_SUFFIX, "") : tag; -} - export function isMissingContainerInspectError(stderr: string): boolean { return stderr.toLowerCase().includes("no such container"); } diff --git a/apps/cli/src/shared/services/services.shared.ts b/apps/cli/src/shared/services/services.shared.ts index c0726d778f..5a7b79a98e 100644 --- a/apps/cli/src/shared/services/services.shared.ts +++ b/apps/cli/src/shared/services/services.shared.ts @@ -214,7 +214,7 @@ export interface ServiceVersionRow { const RELEASE_REVISION_SUFFIX = /^(?.+)-r(?:0|[1-9][0-9]*)$/; /** Strips a slim release tag's `-r` suffix, if any, back to its upstream version. */ -function upstreamVersionFromTag(tag: string): string { +export function upstreamVersionFromTag(tag: string): string { return RELEASE_REVISION_SUFFIX.exec(tag)?.groups?.upstream ?? tag; } diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index f0f2d5d75b..a8b54d891e 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -140,6 +140,12 @@ export function imageTag(image: string): string | undefined { return tagSeparator === -1 ? undefined : withoutDigest.slice(tagSeparator + 1); } +/** The `@sha256:…` digest suffix of `image`, if it carries one. */ +export function imageDigest(image: string): string | undefined { + const at = image.indexOf("@"); + return at === -1 ? undefined : image.slice(at + 1); +} + /** Replaces `image`'s tag with `tag`, dropping any `@sha256:…` digest — a new tag invalidates it. */ export function replaceImageTag(image: string, tag: string): string { const withoutDigest = image.split("@")[0] ?? image; From 130d06030dfa8fb54c3c036e6be0c0f45fefe045 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 13:35:27 +0200 Subject: [PATCH 16/45] fix(stack): drop an import left unused by the rebase-3 conflict resolution prepareNativeArtifact's only remaining call site in this test was replaced by the develop flake fix's nativeRestArtifact helper during the rebase 3 conflict resolution, leaving the import unused. --- packages/stack/src/services/ProcessRecipe.integration.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index 1e6dcad08a..6bb6b8014b 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -24,7 +24,7 @@ import { systemError } from "effect/PlatformError"; import * as Net from "node:net"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- the collision fixture owns a local HTTP listener. import * as NodeHttp from "node:http"; -import { prepareNativeArtifact, resolveArtifact, type ServiceKind } from "../Artifacts.ts"; +import { resolveArtifact, type ServiceKind } from "../Artifacts.ts"; import { makeArtifactStore, type ArtifactRequest, From 50d51dabcebfbdd409285ffddacb1a1b51ff5179 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 14:08:29 +0200 Subject: [PATCH 17/45] test(cli): follow the pinned catalog in the slim start e2e MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With SUPABASE_USE_SLIM_IMAGES, the older slim path only uses a slim image when the Dockerfile tag matches a catalog pin; otherwise it keeps the upstream image with the upstream healthcheck. Today that's auth, pgmeta, edge-runtime and pooler — not every service, as the tests assumed. realtime is on Dockerfile v2.135.3 against catalog v2.140.3, so its upstream (Elixir, no wget) container failed the BusyBox wget probe the old test always expected. - The BusyBox wget test now iterates catalogPins() directly (the pinned digest refs) for the services that use the wget healthcheck, independent of whether the Dockerfile currently matches. It no longer depends on the Dockerfile at all. - The start test derives each service's expected image the same way slim mode does (toSlimImage(alias, dockerfileTag) ?? upstream), then asserts the container runs that image and its healthcheck matches that image's family (slim BusyBox wget vs. each service's own upstream argv) — never assuming a fixed family. auth naturally exercises the slim branch today; realtime and storage exercise whichever branch the Dockerfile/catalog alignment puts them in. - Renamed both tests: "latest slim images" no longer describes a path that depends on Dockerfile/catalog alignment. --- .../start/start.slim-images.e2e.test.ts | 130 ++++++++++++++---- 1 file changed, 106 insertions(+), 24 deletions(-) diff --git a/apps/cli/src/commands/start/start.slim-images.e2e.test.ts b/apps/cli/src/commands/start/start.slim-images.e2e.test.ts index 80f9ff9a8a..897e37df08 100644 --- a/apps/cli/src/commands/start/start.slim-images.e2e.test.ts +++ b/apps/cli/src/commands/start/start.slim-images.e2e.test.ts @@ -2,9 +2,10 @@ import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/ import { BunServices } from "@effect/platform-bun"; import { Clock, Data, Effect, FileSystem, Layer, Path, Schema } from "effect"; import { beforeAll, describe, expect, it } from "@effect/vitest"; +import { catalogPins, type ServiceKind } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImageRaw } from "../../shared/services/dockerfile-images.ts"; -import { toSlimImage } from "../../shared/services/slim-images.ts"; +import { isSlimImageRef, toSlimImage } from "../../shared/services/slim-images.ts"; import { buildHealthCmdArg } from "../../command-internal/db-bootstrap/docker-create-args.ts"; import { slimWgetHealthcheck, @@ -63,20 +64,29 @@ const PULL_ALIASES = [ "mailpit", "kong", ] as const; -/** Slim images whose in-container probe is BusyBox wget. */ -const WGET_PROBE_ALIASES = [ - "gotrue", +/** + * Catalog services whose slim image ships BusyBox wget as its in-container probe (not the + * Dockerfile-derived image: whether the Dockerfile's tag currently matches that catalog pin is + * unrelated to whether the pinned slim image itself accepts the BusyBox argv). + */ +const WGET_PROBE_SERVICES: ReadonlyArray = [ + "auth", "realtime", "storage", - "logflare", - "supavisor", + "analytics", "vector", -] as const; + "pooler", +]; + +function wgetProbeCatalogImages(): ReadonlyArray { + return catalogPins() + .filter((entry) => WGET_PROBE_SERVICES.includes(entry.service)) + .map((entry) => entry.pin.image); +} function latestImagesToPull(): ReadonlyArray { - return [...new Set([...PULL_ALIASES, ...WGET_PROBE_ALIASES])].map((alias) => - expectedSlimImage(alias), - ); + const dockerfileImages = PULL_ALIASES.map((alias) => expectedSlimImage(alias)); + return [...new Set([...dockerfileImages, ...wgetProbeCatalogImages()])]; } function readSectionPort(config: string, section: string): number { @@ -97,6 +107,71 @@ function expectedSlimImage(alias: string): string { return toSlimImage(alias, raw) ?? raw; } +/** + * The Dockerfile aliases whose spec builder switches its healthcheck on `usesSlimImageRuntime` + * (`*.service.ts`). Whether each one is actually running its slim image today depends on whether + * the Dockerfile's tag currently matches that service's catalog pin (`expectedSlimImage`) — + * Dependabot moves that independently for every service, so the assertions below always derive + * the expected healthcheck from the image actually resolved, never from an assumption that it's + * one family or the other. + */ +type HealthcheckedAlias = "gotrue" | "storage" | "realtime"; + +/** Mirrors each service's own upstream (non-slim) `healthcheck.test`, `CMD` prefix included. */ +function upstreamHealthcheckTest(alias: HealthcheckedAlias): ReadonlyArray { + switch (alias) { + case "gotrue": + return [ + "CMD", + "wget", + "--no-verbose", + "--tries=1", + "--spider", + "http://127.0.0.1:9999/health", + ]; + case "storage": + return [ + "CMD", + "wget", + "--no-verbose", + "--tries=1", + "--spider", + "http://127.0.0.1:5000/status", + ]; + case "realtime": + return [ + "CMD", + "curl", + "-sSfL", + "--head", + "-o", + "/dev/null", + "-H", + `Host:${REALTIME_TENANT_ID}`, + "http://127.0.0.1:4000/api/ping", + ]; + } +} + +/** Mirrors each service's own slim (BusyBox wget) `healthcheck.test`, `CMD` prefix included. */ +function slimHealthcheckTest(alias: HealthcheckedAlias): ReadonlyArray { + switch (alias) { + case "gotrue": + return slimWgetHealthcheck("http://127.0.0.1:9999/health").test; + case "storage": + return slimWgetHealthcheck("http://127.0.0.1:5000/status").test; + case "realtime": + return slimWgetHealthcheck("http://127.0.0.1:4000/api/ping", { + header: `Host:${REALTIME_TENANT_ID}`, + }).test; + } +} + +/** The `healthcheck.test` a container running `image` must have — matched to its family. */ +function expectedHealthcheckTest(alias: HealthcheckedAlias, image: string): ReadonlyArray { + return isSlimImageRef(image) ? slimHealthcheckTest(alias) : upstreamHealthcheckTest(alias); +} + const containerHealthcheckTest = Effect.fnUntraced(function* (name: string) { const { stdout } = yield* runDockerEffect([ "inspect", @@ -186,19 +261,20 @@ describe("supabase start slim images (e2e)", () => { ); it.live( - "every slim wget image accepts the BusyBox healthcheck argv", + "every pinned slim wget image accepts the BusyBox healthcheck argv", () => Effect.gen(function* () { - for (const alias of WGET_PROBE_ALIASES) { - const image = expectedSlimImage(alias); + for (const entry of catalogPins()) { + if (!WGET_PROBE_SERVICES.includes(entry.service)) continue; + const image = entry.pin.image; const probe = - alias === "realtime" + entry.service === "realtime" ? slimWgetHealthcheck("http://127.0.0.1:9/", { header: `Host:${REALTIME_TENANT_ID}`, }) : slimWgetHealthcheck("http://127.0.0.1:9/"); expectBusyBoxAccepted(yield* runWgetInImage(image, probe.test.slice(2)), image); - if (alias === "vector") { + if (entry.service === "vector") { const waitArgs = slimWgetWaitCommand("http://127.0.0.1:9/").split(" ").slice(1); expectBusyBoxAccepted(yield* runWgetInImage(image, waitArgs), `${image} wait`); } @@ -208,7 +284,7 @@ describe("supabase start slim images (e2e)", () => { ); it.live( - "starts the latest slim images, serves a function without a version pin, and keeps the Dockerfile tag", + "starts each service on its resolved image (slim or upstream), matching healthchecks to family, and serves a function without a version pin", () => Effect.gen(function* () { const projectDir = yield* makeProject("sb-slim-start-e2e-"); @@ -245,25 +321,31 @@ describe("supabase start slim images (e2e)", () => { }); expect(start.exitCode, `stdout:\n${start.stdout}\nstderr:\n${start.stderr}`).toBe(0); + // Each service's expected image follows the Dockerfile: slim when its tag matches the + // catalog pin, upstream otherwise (`expectedSlimImage`). Today only auth, pgmeta, + // edge-runtime and pooler match — Dependabot moves the Dockerfile independently of the + // catalog, so which of these is slim today is not assumed anywhere below. + const authImage = expectedSlimImage("gotrue"); + const realtimeImage = expectedSlimImage("realtime"); + const storageImage = expectedSlimImage("storage"); + expect(yield* containerImage(dbContainer)).toBe(expectedSlimImage("pg")); - expect(yield* containerImage(storageContainer)).toBe(expectedSlimImage("storage")); + expect(yield* containerImage(storageContainer)).toBe(storageImage); expect(yield* containerImage(edgeRuntimeContainer)).toBe(expectedSlimImage("edgeruntime")); + expect(yield* containerImage(authContainer)).toBe(authImage); + expect(yield* containerImage(realtimeContainer)).toBe(realtimeImage); expect(yield* containerHealthcheckTest(authContainer)).toEqual([ "CMD-SHELL", - buildHealthCmdArg(slimWgetHealthcheck("http://127.0.0.1:9999/health").test), + buildHealthCmdArg(expectedHealthcheckTest("gotrue", authImage)), ]); expect(yield* containerHealthcheckTest(realtimeContainer)).toEqual([ "CMD-SHELL", - buildHealthCmdArg( - slimWgetHealthcheck("http://127.0.0.1:4000/api/ping", { - header: `Host:${REALTIME_TENANT_ID}`, - }).test, - ), + buildHealthCmdArg(expectedHealthcheckTest("realtime", realtimeImage)), ]); expect(yield* containerHealthcheckTest(storageContainer)).toEqual([ "CMD-SHELL", - buildHealthCmdArg(slimWgetHealthcheck("http://127.0.0.1:5000/status").test), + buildHealthCmdArg(expectedHealthcheckTest("storage", storageImage)), ]); expect(yield* containerHealthStatus(authContainer)).toBe("healthy"); expect(yield* containerHealthStatus(realtimeContainer)).toBe("healthy"); From e00174332198646f25f0681efaa866dd0c37ee61 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 16:10:56 +0200 Subject: [PATCH 18/45] feat(cli): generate slim-capable Dockerfile lines from the stack catalog Adds `upstreamImage` to each catalog `ArtifactPin` (the exact upstream image slim-services built or mirrored each pin from), backfilled through the sync script's manual mode. A new generator (apps/cli/scripts/render-service-dockerfile.ts) rewrites only the slim-capable `FROM ... AS ` lines of the Dockerfile in place from that field, asserting the repository stays the one the Dockerfile already uses (imgproxy is the one documented exception); everything else (kong, Dependabot-managed job images) passes through untouched. Its drift test runs as part of the required apps/cli unit suite. Adds the `pg15` (generated) and `pg14` (hand-pinned, no slim build) Dockerfile stages as the single version table for legacy Postgres resolution. `postgresImageForDbMajorVersion` now reads them instead of three hardcoded fallback constants; the Go CLI's own `pg14`/`pg15` constants are replaced with the same Dockerfile-derived fields, closing the second version table. One-time regeneration bumps every slim-capable service to its currently pinned upstream version. --- .../scripts/sync-artifacts-catalog.test.ts | 106 ++++++++++ .github/scripts/sync-artifacts-catalog.ts | 179 +++++++++++++++- apps/cli-go/pkg/config/config.go | 6 +- apps/cli-go/pkg/config/constants.go | 7 +- apps/cli-go/pkg/config/templates/Dockerfile | 28 ++- ...nder-service-dockerfile.rules.unit.test.ts | 115 +++++++++++ apps/cli/scripts/render-service-dockerfile.ts | 192 ++++++++++++++++++ .../render-service-dockerfile.unit.test.ts | 59 ++++++ .../command-internal/db-image.unit.test.ts | 51 ++--- apps/cli/src/shared/services/Dockerfile | 28 ++- .../services/dockerfile-go-sync.unit.test.ts | 18 -- .../src/shared/services/services.shared.ts | 18 +- .../services/services.shared.unit.test.ts | 17 +- ...slim-images.catalog-alignment.unit.test.ts | 50 +++++ apps/cli/src/shared/services/slim-images.ts | 6 +- .../shared/services/slim-images.unit.test.ts | 11 +- packages/stack/src/Artifacts.ts | 38 +++- packages/stack/src/Artifacts.unit.test.ts | 11 +- 18 files changed, 844 insertions(+), 96 deletions(-) create mode 100644 apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts create mode 100644 apps/cli/scripts/render-service-dockerfile.ts create mode 100644 apps/cli/scripts/render-service-dockerfile.unit.test.ts delete mode 100644 apps/cli/src/shared/services/dockerfile-go-sync.unit.test.ts create mode 100644 apps/cli/src/shared/services/slim-images.catalog-alignment.unit.test.ts diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index 599c09d529..4a6f3c2c7e 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -583,6 +583,112 @@ describe("refreshCatalogPin", () => { }); }); +const vectorFixture = `const workloadCatalog = { + vector: definition("vector", placeholderPin("vector", "0.53.0"), "bin/vector"), +}; +`; + +/** Manifest fixture for a derived (nix-built) service, agreeing across every native target. */ +function manifestFixture(upstreamImage: string): RevisionIo["fetchManifest"] { + return async () => JSON.stringify({ upstream_image: upstreamImage }); +} + +describe("refreshCatalogPin backfills upstreamImage", () => { + test("resolves a derived service's upstreamImage from its per-target manifests", async () => { + const digests = nativeDigests("y"); + const io: RevisionIo = { + listReleaseTags: async () => ["analytics-v1.50.9-r0", "analytics-v1.50.9-r1"], + fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r1", digests), + imageDigest: async () => digest("z"), + s3Sha256: matchingS3("analytics", "v1.50.9-r1", digests), + fetchManifest: manifestFixture("supabase/logflare:1.50.9"), + }; + + const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); + + expect(result.source).toContain('upstreamImage: "supabase/logflare:1.50.9"'); + }); + + test("falls back to source_image when a target's manifest has no upstream_image (image-derived build)", async () => { + const digests = nativeDigests("aa"); + const io: RevisionIo = { + listReleaseTags: async () => ["analytics-v1.50.9-r0"], + fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r0", digests), + imageDigest: async () => digest("ab"), + s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), + fetchManifest: async () => JSON.stringify({ source_image: "supabase/logflare:1.50.9" }), + }; + + const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); + + expect(result.source).toContain('upstreamImage: "supabase/logflare:1.50.9"'); + }); + + test("strips a docker.io prefix and a digest from the resolved upstreamImage", async () => { + const digests = nativeDigests("ac"); + const io: RevisionIo = { + listReleaseTags: async () => ["analytics-v1.50.9-r0"], + fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r0", digests), + imageDigest: async () => digest("ad"), + s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), + fetchManifest: manifestFixture("docker.io/supabase/logflare:1.50.9@sha256:deadbeef"), + }; + + const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); + + expect(result.source).toContain('upstreamImage: "supabase/logflare:1.50.9"'); + }); + + test("fails loudly when a derived service's manifests disagree across native targets", async () => { + const digests = nativeDigests("ae"); + let call = 0; + const io: RevisionIo = { + listReleaseTags: async () => ["analytics-v1.50.9-r0"], + fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r0", digests), + imageDigest: async () => digest("af"), + s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), + fetchManifest: async () => { + call += 1; + return JSON.stringify({ upstream_image: `supabase/logflare:1.50.${call}` }); + }, + }; + + await expect(refreshCatalogPin({ catalog: fixture, service: "analytics", io })).rejects.toThrow( + /manifests disagree/, + ); + }); + + test("resolves a mirrored service's upstreamImage from its oci-provenance source", async () => { + const digests = nativeDigests("ag"); + const io: RevisionIo = { + listReleaseTags: async () => ["vector-0.53.0-r0"], + fetchChecksums: async () => checksumsFor("vector", "0.53.0-r0", digests), + imageDigest: async () => digest("ah"), + s3Sha256: matchingS3("vector", "0.53.0-r0", digests), + fetchProvenance: async () => + JSON.stringify({ source: "docker.io/timberio/vector:0.53.0-alpine" }), + }; + + const result = await refreshCatalogPin({ catalog: vectorFixture, service: "vector", io }); + + expect(result.source).toContain('upstreamImage: "timberio/vector:0.53.0-alpine"'); + }); + + test("leaves upstreamImage absent when io has no manifest/provenance fetchers", async () => { + const digests = nativeDigests("ai"); + const io: RevisionIo = { + listReleaseTags: async () => ["analytics-v1.50.9-r0"], + fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r0", digests), + imageDigest: async () => digest("aj"), + s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), + }; + + const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); + + expect(result.source).not.toContain("upstreamImage"); + }); +}); + describe("against the real catalog", () => { test("every slim Dockerfile alias is a catalog entry", async () => { const dockerfile = await Bun.file("apps/cli/src/shared/services/Dockerfile").text(); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index 5654547f90..ff783bf2d7 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -111,6 +111,12 @@ interface ResolvedPin { readonly upstreamVersion: string; readonly revision: number; readonly image: string; + /** + * The pin's `ArtifactPin.upstreamImage`. Left undefined on the Dependabot (`planArtifactCatalogUpdate`) + * path, which has no use for it and whose `io` doesn't carry `fetchManifest`/`fetchProvenance`; + * populated by manual mode (`refreshCatalogPin`) via `resolveUpstreamImage` below. + */ + readonly upstreamImage?: string; readonly natives: Readonly< Record >; @@ -133,6 +139,27 @@ export interface RevisionIo { readonly fetchChecksums: (service: string, releaseVersion: string) => Promise; readonly imageDigest: (service: string, releaseVersion: string) => Promise; readonly s3Sha256: (url: string) => Promise; + /** + * Raw contents of a native target's `.manifest.json` release asset, for a derived service's + * `upstream_image` (or `source_image` on an image-derived build, e.g. postgrest's Linux + * targets). Optional: only manual mode's `upstreamImage` backfill (`resolveUpstreamImage`) + * calls this, so a test `io` that doesn't exercise that path can omit it. + */ + readonly fetchManifest?: ( + service: string, + releaseVersion: string, + target: NativeTargetName, + ) => Promise; + /** + * Raw contents of a mirrored service's `-.oci-provenance.json` + * release asset, whose `source` field is the mirrored upstream image. Optional for the same + * reason as `fetchManifest`. + */ + readonly fetchProvenance?: ( + service: string, + releaseVersion: string, + upstreamVersion: string, + ) => Promise; } function desiredImage(service: string, releaseVersion: string, digest: string): string { @@ -227,13 +254,109 @@ export async function resolveRevisionPin( }; } +/** + * Services slim-services mirrors from an unmodified upstream image rather than building from + * source: their `upstreamImage` comes from the release's `oci-provenance.json` `source` field, + * not a native target's manifest. Keep this in sync with `SlimServicesSource`'s mirror-mode + * services. + */ +const MIRROR_MODE_SOURCE_SERVICES: ReadonlySet = new Set(["vector", "mailpit", "imgproxy"]); + +function parseJsonRecord(raw: string): Record | undefined { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return undefined; + } + return typeof parsed === "object" && parsed !== null + ? (parsed as Record) + : undefined; +} + +function stringField(record: Record | undefined, key: string): string | undefined { + const value = record?.[key]; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +/** Strips a `docker.io/` prefix and any `@sha256:…` digest, to match the Dockerfile's `FROM` form. */ +function normalizeUpstreamImage(image: string): string { + const withoutDigest = image.split("@")[0] ?? image; + return withoutDigest.startsWith("docker.io/") + ? withoutDigest.slice("docker.io/".length) + : withoutDigest; +} + +/** + * Resolves `service`'s `upstreamImage` for the release `releaseVersion` (`-r`): + * a mirrored service's `oci-provenance.json` `source`, or a derived service's per-target + * manifest `upstream_image` (falling back to `source_image` for an image-derived build, e.g. + * postgrest's Linux targets) — cross-checked across every native target, so a derived service + * whose manifests disagree fails instead of silently picking one. Only manual mode + * (`refreshCatalogPin`) calls this; `io` must carry `fetchManifest`/`fetchProvenance`. + */ +async function resolveUpstreamImage( + service: string, + releaseVersion: string, + upstreamVersion: string, + io: RevisionIo, +): Promise { + if (MIRROR_MODE_SOURCE_SERVICES.has(service)) { + if (io.fetchProvenance === undefined) { + throw new InvalidPayloadError( + `${service} needs an io.fetchProvenance to resolve upstreamImage.`, + ); + } + const provenance = await io.fetchProvenance(service, releaseVersion, upstreamVersion); + if (provenance === undefined) { + throw new InvalidPayloadError(`${service}-${releaseVersion} has no oci-provenance asset.`); + } + const source = stringField(parseJsonRecord(provenance), "source"); + if (source === undefined) { + throw new InvalidPayloadError( + `${service}-${releaseVersion} oci-provenance has no 'source' field.`, + ); + } + return normalizeUpstreamImage(source); + } + + if (io.fetchManifest === undefined) { + throw new InvalidPayloadError(`${service} needs an io.fetchManifest to resolve upstreamImage.`); + } + const values = new Set(); + for (const target of NATIVE_TARGETS) { + const manifest = await io.fetchManifest(service, releaseVersion, target); + if (manifest === undefined) { + throw new InvalidPayloadError( + `${service}-${releaseVersion}-${target} has no manifest asset.`, + ); + } + const record = parseJsonRecord(manifest); + const value = stringField(record, "upstream_image") ?? stringField(record, "source_image"); + if (value === undefined) { + throw new InvalidPayloadError( + `${service}-${releaseVersion}-${target} manifest has neither 'upstream_image' nor 'source_image'.`, + ); + } + values.add(normalizeUpstreamImage(value)); + } + if (values.size !== 1) { + throw new InvalidPayloadError( + `${service}-${releaseVersion} manifests disagree on the upstream image: ${[...values].sort().join(", ")}.`, + ); + } + return [...values][0] as string; +} + /** Serializes a resolved pin into the object literal `Artifacts.ts` embeds, in catalog order. */ function serializePin(pin: ResolvedPin): string { const natives = NATIVE_TARGETS.map((target) => { const native = pin.natives[target]; return `"${target}": { archive: "${native.archive}", manifest: "${native.manifest}" }`; }).join(", "); - return `{ upstreamVersion: "${pin.upstreamVersion}", revision: ${pin.revision}, image: "${pin.image}", natives: { ${natives} } }`; + const upstreamImage = + pin.upstreamImage === undefined ? "" : ` upstreamImage: "${pin.upstreamImage}",`; + return `{ upstreamVersion: "${pin.upstreamVersion}", revision: ${pin.revision}, image: "${pin.image}",${upstreamImage} natives: { ${natives} } }`; } interface PinSpan { @@ -633,7 +756,22 @@ export async function refreshCatalogPin(input: { if (resolution.status !== "resolved") { throw new InvalidPayloadError(resolution.message); } - const written = writePin(input.catalog, entry, resolution.pin); + // Manual mode also backfills `upstreamImage`, so a plain `io` (most tests, and the + // Dependabot-driven `planArtifactCatalogUpdate` path) can still exercise revision resolution + // without stubbing the extra fetchers. + const pin = + input.io.fetchManifest === undefined && input.io.fetchProvenance === undefined + ? resolution.pin + : { + ...resolution.pin, + upstreamImage: await resolveUpstreamImage( + input.service, + `${resolution.pin.upstreamVersion}-r${resolution.pin.revision}`, + upstream, + input.io, + ), + }; + const written = writePin(input.catalog, entry, pin); if (!written.changed) return { source: input.catalog }; return { source: written.source, @@ -718,8 +856,43 @@ async function s3Sha256(url: string): Promise { return hasher.digest("hex"); } +async function fetchManifest( + service: string, + releaseVersion: string, + target: NativeTargetName, +): Promise { + const files = nativeFileNames(service, releaseVersion, target); + const response = await fetch( + `${RELEASE_DOWNLOAD_BASE}/${service}-${releaseVersion}/${files.manifest}`, + ); + if (response.status === 404) return undefined; + if (!response.ok) { + throw new InvalidPayloadError( + `manifest download failed for ${service}-${releaseVersion} ${target} (HTTP ${response.status}).`, + ); + } + return response.text(); +} + +async function fetchProvenance( + service: string, + releaseVersion: string, + upstreamVersion: string, +): Promise { + const response = await fetch( + `${RELEASE_DOWNLOAD_BASE}/${service}-${releaseVersion}/${service}-${upstreamVersion}.oci-provenance.json`, + ); + if (response.status === 404) return undefined; + if (!response.ok) { + throw new InvalidPayloadError( + `oci-provenance download failed for ${service}-${releaseVersion} (HTTP ${response.status}).`, + ); + } + return response.text(); +} + function defaultRevisionIo(): RevisionIo { - return { listReleaseTags, fetchChecksums, imageDigest, s3Sha256 }; + return { listReleaseTags, fetchChecksums, imageDigest, s3Sha256, fetchManifest, fetchProvenance }; } function parseFlags(argv: ReadonlyArray): ReadonlyMap { diff --git a/apps/cli-go/pkg/config/config.go b/apps/cli-go/pkg/config/config.go index 01c426edd8..d6603e8710 100644 --- a/apps/cli-go/pkg/config/config.go +++ b/apps/cli-go/pkg/config/config.go @@ -824,11 +824,11 @@ func (c *config) Load(path string, fsys fs.FS, overrides ...ConfigEditor) error // Update image versions switch c.Db.MajorVersion { case 13: - c.Db.Image = pg15 + c.Db.Image = Images.Pg15 case 14: - c.Db.Image = pg14 + c.Db.Image = Images.Pg14 case 15: - c.Db.Image = pg15 + c.Db.Image = Images.Pg15 } if c.Db.MajorVersion > 14 { if version, err := fs.ReadFile(fsys, builder.PostgresVersionPath); err == nil { diff --git a/apps/cli-go/pkg/config/constants.go b/apps/cli-go/pkg/config/constants.go index 08d572d2da..93bb482f69 100644 --- a/apps/cli-go/pkg/config/constants.go +++ b/apps/cli-go/pkg/config/constants.go @@ -9,14 +9,15 @@ import ( ) const ( - pg13 = "supabase/postgres:13.3.0" - pg14 = "supabase/postgres:14.1.0.89" - pg15 = "supabase/postgres:15.8.1.085" deno1 = "supabase/edge-runtime:v1.68.4" ) type images struct { Pg string `mapstructure:"pg"` + // PG13/15 and PG14 come from the Dockerfile's generated `pg15`/hand-pinned `pg14` stages + // (the single version table), not hardcoded constants. + Pg15 string `mapstructure:"pg15"` + Pg14 string `mapstructure:"pg14"` // Append to Services when adding new dependencies below Kong string `mapstructure:"kong"` Inbucket string `mapstructure:"mailpit"` diff --git a/apps/cli-go/pkg/config/templates/Dockerfile b/apps/cli-go/pkg/config/templates/Dockerfile index a9c85ee0ca..7b404856de 100644 --- a/apps/cli-go/pkg/config/templates/Dockerfile +++ b/apps/cli-go/pkg/config/templates/Dockerfile @@ -1,21 +1,29 @@ -# Exposed for updates by .github/dependabot.yml -FROM supabase/postgres:17.6.1.171 AS pg +# The tag pinned below is generated from packages/stack/src/Artifacts.ts by +# apps/cli/scripts/render-service-dockerfile.ts, for these aliases only: pg, pg15, mailpit, +# postgrest, pgmeta, studio, imgproxy, edgeruntime, vector, supavisor, gotrue, realtime, storage, +# logflare. Do not hand-edit those tags; a CI drift check enforces this. Run the generator after +# a catalog update. Everything else in this file (this comment, kong, pg14, the one-shot job +# images) is hand- or Dependabot-managed. +FROM supabase/postgres:17.11.0.002 AS pg +FROM supabase/postgres:15.19.0.002 AS pg15 +# Postgres 14 has no slim build; bumped by hand (Dependabot ignores supabase/postgres). +FROM supabase/postgres:14.1.0.89 AS pg14 # New always-on service alias: extend SERVICE_IMAGE_ALIASES in # shared/services/services.shared.ts and DOCKERFILE_ALIAS_BY_SERVICE in # commands/start/start.gates.ts. FROM library/kong:2.8.1 AS kong -FROM axllent/mailpit:v1.30.2 AS mailpit -FROM postgrest/postgrest:v16.3 AS postgrest +FROM axllent/mailpit:v1.31.3 AS mailpit +FROM postgrest/postgrest:v16.4 AS postgrest FROM supabase/postgres-meta:v0.99.0 AS pgmeta -FROM supabase/studio:2026.09.14-sha-4dd8a95 AS studio -FROM darthsim/imgproxy:v3.8.0 AS imgproxy +FROM supabase/studio:2026.09.28-sha-5e59b60 AS studio +FROM darthsim/imgproxy:v3.26.0 AS imgproxy FROM supabase/edge-runtime:v1.77.1 AS edgeruntime -FROM timberio/vector:0.53.0-alpine AS vector +FROM timberio/vector:0.58.0-alpine AS vector FROM supabase/supavisor:2.9.13 AS supavisor FROM supabase/gotrue:v2.197.0 AS gotrue -FROM supabase/realtime:v2.135.3 AS realtime -FROM supabase/storage-api:v1.77.0 AS storage -FROM supabase/logflare:1.50.12 AS logflare +FROM supabase/realtime:v2.140.3 AS realtime +FROM supabase/storage-api:v1.79.26 AS storage +FROM supabase/logflare:1.50.15 AS logflare # One-shot job images (not started as long-running containers); differ's alias # is resolved via dockerfileServiceImage("differ") in commands/db/diff/pgadmin-diff.ts. FROM supabase/pgadmin-schema-diff:cli-0.0.5 AS differ diff --git a/apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts b/apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts new file mode 100644 index 0000000000..73080b1628 --- /dev/null +++ b/apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, test, vi } from "vitest"; + +import { renderDockerfile } from "./render-service-dockerfile.ts"; + +// A minimal fixture catalog covering every slim-capable alias, so `renderDockerfile`'s +// "every alias needs exactly one line" check is satisfied by default; each test only mutates +// what it's exercising. `vi.mock` factories are hoisted above every other top-level statement, +// so the fixture pins are built inline here rather than imported from an outer module. +vi.mock("@supabase/stack/internal/artifacts", () => { + const nativePin = { archive: "a".repeat(64), manifest: "b".repeat(64) }; + const natives = { "darwin-arm64": nativePin, "linux-amd64": nativePin, "linux-arm64": nativePin }; + const pin = (sourceService: string, upstreamImage: string, isDefault: boolean) => ({ + service: sourceService, + sourceService, + isDefault, + pin: { + upstreamVersion: upstreamImage.split(":").at(-1), + revision: 0, + image: `ghcr.io/supabase/cli/${sourceService}:fixture-r0@sha256:${"c".repeat(64)}`, + upstreamImage, + natives, + }, + }); + return { + catalogPins: () => [ + pin("postgres", "supabase/postgres:17.0.0", true), + pin("postgres", "supabase/postgres:15.0.0", false), + pin("mailpit", "axllent/mailpit:v1.0.0", true), + pin("postgrest", "postgrest/postgrest:v1.0.0", true), + pin("pgmeta", "supabase/postgres-meta:v1.0.0", true), + pin("studio", "supabase/studio:1.0.0", true), + pin("imgproxy", "ghcr.io/imgproxy/imgproxy:v1.0.0", true), + pin("edge-runtime", "supabase/edge-runtime:v1.0.0", true), + pin("vector", "timberio/vector:1.0.0-alpine", true), + pin("pooler", "supabase/supavisor:1.0.0", true), + pin("auth", "supabase/gotrue:v1.0.0", true), + pin("realtime", "supabase/realtime:v1.0.0", true), + pin("storage", "supabase/storage-api:v1.0.0", true), + pin("analytics", "supabase/logflare:1.0.0", true), + ], + }; +}); + +/** A minimal Dockerfile with every slim-capable alias, plus kong and pg14 (hand-managed). */ +function baseDockerfile(overrides: Readonly> = {}): string { + const lines: Readonly> = { + pg: "FROM supabase/postgres:16.9.9 AS pg", + pg15: "FROM supabase/postgres:14.9.9 AS pg15", + mailpit: "FROM axllent/mailpit:v0.9.9 AS mailpit", + postgrest: "FROM postgrest/postgrest:v0.9.9 AS postgrest", + pgmeta: "FROM supabase/postgres-meta:v0.9.9 AS pgmeta", + studio: "FROM supabase/studio:0.9.9 AS studio", + imgproxy: "FROM darthsim/imgproxy:v0.9.9 AS imgproxy", + edgeruntime: "FROM supabase/edge-runtime:v0.9.9 AS edgeruntime", + vector: "FROM timberio/vector:0.9.9-alpine AS vector", + supavisor: "FROM supabase/supavisor:0.9.9 AS supavisor", + gotrue: "FROM supabase/gotrue:v0.9.9 AS gotrue", + realtime: "FROM supabase/realtime:v0.9.9 AS realtime", + storage: "FROM supabase/storage-api:v0.9.9 AS storage", + logflare: "FROM supabase/logflare:0.9.9 AS logflare", + kong: "FROM library/kong:2.8.1 AS kong", + }; + const merged = { ...lines, ...overrides }; + return `# hand-authored header\n${Object.values(merged).join("\n")}\n# hand-authored footer\n`; +} + +describe("renderDockerfile: pin selection", () => { + test("pg takes the default pin and pg15 takes the additional one, by isDefault, not a version-string check", () => { + const rendered = renderDockerfile(baseDockerfile()); + expect(rendered).toContain("FROM supabase/postgres:17.0.0 AS pg\n"); + expect(rendered).toContain("FROM supabase/postgres:15.0.0 AS pg15\n"); + }); +}); + +describe("renderDockerfile: repository rule", () => { + test("keeps the existing repository and only replaces the tag", () => { + const rendered = renderDockerfile(baseDockerfile()); + expect(rendered).toContain("FROM supabase/gotrue:v1.0.0 AS gotrue\n"); + }); + + test("fails loudly when the Dockerfile's repository no longer matches the catalog's, for a non-overridden alias", () => { + expect(() => + renderDockerfile(baseDockerfile({ gotrue: "FROM some-other-org/gotrue:v0.9.9 AS gotrue" })), + ).toThrow(/gotrue.*repository/i); + }); + + test("tolerates imgproxy's documented repository mismatch (the allowlisted override)", () => { + const rendered = renderDockerfile(baseDockerfile()); + // Catalog upstreamImage repo is ghcr.io/imgproxy/imgproxy; the Dockerfile keeps darthsim. + expect(rendered).toContain("FROM darthsim/imgproxy:v1.0.0 AS imgproxy\n"); + }); + + test("still fails loudly when imgproxy's Dockerfile repository drifts from its override entry", () => { + expect(() => + renderDockerfile( + baseDockerfile({ imgproxy: "FROM some-other-org/imgproxy:v0.9.9 AS imgproxy" }), + ), + ).toThrow(/imgproxy.*REPOSITORY_OVERRIDES/i); + }); +}); + +describe("renderDockerfile: line preservation", () => { + test("never inserts a line for a missing slim-capable alias", () => { + const withoutPg15 = baseDockerfile().replace(/FROM supabase\/postgres:14\.9\.9 AS pg15\n/, ""); + expect(() => renderDockerfile(withoutPg15)).toThrow(/pg15/); + }); + + test("leaves non-slim-capable lines (kong) and comments byte-for-byte untouched", () => { + const current = baseDockerfile(); + const rendered = renderDockerfile(current); + expect(rendered).toContain("FROM library/kong:2.8.1 AS kong"); + expect(rendered).toContain("# hand-authored header"); + expect(rendered).toContain("# hand-authored footer"); + }); +}); diff --git a/apps/cli/scripts/render-service-dockerfile.ts b/apps/cli/scripts/render-service-dockerfile.ts new file mode 100644 index 0000000000..da5d146ffc --- /dev/null +++ b/apps/cli/scripts/render-service-dockerfile.ts @@ -0,0 +1,192 @@ +// Rewrites the slim-capable `FROM ... AS ` lines of +// apps/cli/src/shared/services/Dockerfile (and its byte-identical Go copy, +// apps/cli-go/pkg/config/templates/Dockerfile) in place, from the stack catalog +// (packages/stack/src/Artifacts.ts): each such line's tag comes from that service's catalog pin +// `upstreamImage`. Every other line — comments, kong, the Postgres 14 pin (no slim build), and +// the one-shot job images — is hand- or Dependabot-managed and left byte-for-byte untouched. +// +// bun apps/cli/scripts/render-service-dockerfile.ts # writes both files +// bun apps/cli/scripts/render-service-dockerfile.ts --check # fails on drift, writes nothing +// +// The `--check` mode is what CI runs (as a `render-service-dockerfile.unit.test.ts` assertion) to +// catch a hand-edited generated line, or a catalog change that hasn't been regenerated yet. +import { fileURLToPath } from "node:url"; +import { catalogPins, type ArtifactPin } from "@supabase/stack/internal/artifacts"; + +// Resolved from this module's own URL, so both the CLI invocation (cwd = repo root) and the +// vitest unit test (cwd = apps/cli) read the same files. +export const TS_DOCKERFILE_PATH = fileURLToPath( + new URL("../src/shared/services/Dockerfile", import.meta.url), +); +export const GO_DOCKERFILE_PATH = fileURLToPath( + new URL("../../cli-go/pkg/config/templates/Dockerfile", import.meta.url), +); + +type CatalogEntry = ReturnType[number]; + +/** + * Dockerfile repository overrides, for the one documented exception to the repository rule + * below. Keep this list to that exception; add a reason alongside any addition. + */ +const REPOSITORY_OVERRIDES: Readonly> = { + // slim-services mirrors ghcr.io/imgproxy/imgproxy (its `upstreamImage`), but the Dockerfile — + // and `mirror-template-images.yml` and the docker.io registry rewrite — keep darthsim/imgproxy, + // which carries the same tags on Docker Hub. See the design doc's "Repository rule". + imgproxy: "darthsim/imgproxy", +}; + +interface SlimAliasSpec { + readonly sourceService: string; + /** Selects the catalog's default pin (every alias but `pg15`) or its lone additional pin. */ + readonly wantDefault: boolean; +} + +/** Every slim-capable Dockerfile alias, mapped to the catalog pin it tracks. */ +const SLIM_ALIAS_SPECS: ReadonlyMap = new Map([ + ["pg", { sourceService: "postgres", wantDefault: true }], + ["pg15", { sourceService: "postgres", wantDefault: false }], + ["mailpit", { sourceService: "mailpit", wantDefault: true }], + ["postgrest", { sourceService: "postgrest", wantDefault: true }], + ["pgmeta", { sourceService: "pgmeta", wantDefault: true }], + ["studio", { sourceService: "studio", wantDefault: true }], + ["imgproxy", { sourceService: "imgproxy", wantDefault: true }], + ["edgeruntime", { sourceService: "edge-runtime", wantDefault: true }], + ["vector", { sourceService: "vector", wantDefault: true }], + ["supavisor", { sourceService: "pooler", wantDefault: true }], + ["gotrue", { sourceService: "auth", wantDefault: true }], + ["realtime", { sourceService: "realtime", wantDefault: true }], + ["storage", { sourceService: "storage", wantDefault: true }], + ["logflare", { sourceService: "analytics", wantDefault: true }], +]); + +function selectPin( + alias: string, + spec: SlimAliasSpec, + pins: ReadonlyArray, +): ArtifactPin { + const candidates = pins.filter( + (entry) => entry.sourceService === spec.sourceService && entry.isDefault === spec.wantDefault, + ); + if (candidates.length !== 1) { + throw new Error( + `expected exactly one ${spec.wantDefault ? "default" : "additional"} catalog pin for ` + + `'${spec.sourceService}' (alias '${alias}'), found ${candidates.length}`, + ); + } + return candidates[0]!.pin; +} + +function tagOf(upstreamImage: string): string { + const sep = upstreamImage.lastIndexOf(":"); + if (sep === -1) throw new Error(`upstreamImage has no tag: ${upstreamImage}`); + return upstreamImage.slice(sep + 1); +} + +function repositoryOf(upstreamImage: string): string { + const sep = upstreamImage.lastIndexOf(":"); + if (sep === -1) throw new Error(`upstreamImage has no tag: ${upstreamImage}`); + return upstreamImage.slice(0, sep); +} + +/** Matches a `FROM : AS ` line, keeping every other byte for reuse. */ +const FROM_LINE = /^(FROM\s+)(.+):([^:\s]+)(\s+AS\s+)([^\s#]+)(.*)$/i; + +/** + * Rewrites only the slim-capable `FROM ... AS ` lines of `currentDockerfile`, in place: + * the repository is read from the existing line (and asserted to still match the catalog pin's + * `upstreamImage` repository, except for `REPOSITORY_OVERRIDES`); only the tag is replaced, from + * that pin's `upstreamImage`. Every other line — comments, kong, `pg14`, the job images — passes + * through untouched. A slim-capable alias missing from the file is an error; the generator never + * inserts a line. + */ +export function renderDockerfile(currentDockerfile: string): string { + const pins = catalogPins(); + const seen = new Set(); + + const lines = currentDockerfile.split("\n").map((line) => { + const match = FROM_LINE.exec(line); + if (match === null) return line; + const [, fromKeyword, repository, , asKeyword, alias, rest] = match as unknown as [ + string, + string, + string, + string, + string, + string, + string, + ]; + const spec = SLIM_ALIAS_SPECS.get(alias); + if (spec === undefined) return line; + seen.add(alias); + + const pin = selectPin(alias, spec, pins); + const pinRepository = repositoryOf(pin.upstreamImage); + const override = REPOSITORY_OVERRIDES[alias]; + if (override === undefined) { + if (repository !== pinRepository) { + throw new Error( + `${alias}'s Dockerfile repository '${repository}' no longer matches the catalog's ` + + `'${pinRepository}'. If this is intentional, add '${alias}' to REPOSITORY_OVERRIDES with a reason.`, + ); + } + } else if (repository !== override) { + throw new Error( + `${alias}'s Dockerfile repository '${repository}' no longer matches its ` + + `REPOSITORY_OVERRIDES entry '${override}'. Update the override if this is intentional.`, + ); + } + + return `${fromKeyword}${repository}:${tagOf(pin.upstreamImage)}${asKeyword}${alias}${rest}`; + }); + + for (const alias of SLIM_ALIAS_SPECS.keys()) { + if (!seen.has(alias)) { + throw new Error( + `no Dockerfile line for slim-capable alias '${alias}'; the generator never inserts one`, + ); + } + } + + return lines.join("\n"); +} + +async function main(argv: ReadonlyArray): Promise { + const check = argv.includes("--check"); + const current = await Bun.file(TS_DOCKERFILE_PATH).text(); + const rendered = renderDockerfile(current); + + if (check) { + const go = await Bun.file(GO_DOCKERFILE_PATH).text(); + let failed = false; + if (current !== rendered) { + console.log( + `::error ::${TS_DOCKERFILE_PATH} has drifted from packages/stack/src/Artifacts.ts. Run ` + + "`bun apps/cli/scripts/render-service-dockerfile.ts` and commit the result.", + ); + failed = true; + } + if (go !== rendered) { + console.log( + `::error ::${GO_DOCKERFILE_PATH} is not a byte copy of ${TS_DOCKERFILE_PATH}. Run ` + + "`bun apps/cli/scripts/render-service-dockerfile.ts` and commit the result.", + ); + failed = true; + } + if (failed) { + process.exit(1); + } + console.log("Both Dockerfiles match the catalog."); + return; + } + + await Bun.write(TS_DOCKERFILE_PATH, rendered); + await Bun.write(GO_DOCKERFILE_PATH, rendered); + console.log(`Regenerated ${TS_DOCKERFILE_PATH} and ${GO_DOCKERFILE_PATH} from the catalog.`); +} + +if (import.meta.main) { + main(process.argv.slice(2)).catch((error: unknown) => { + console.log(`::error ::${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + }); +} diff --git a/apps/cli/scripts/render-service-dockerfile.unit.test.ts b/apps/cli/scripts/render-service-dockerfile.unit.test.ts new file mode 100644 index 0000000000..55862853a2 --- /dev/null +++ b/apps/cli/scripts/render-service-dockerfile.unit.test.ts @@ -0,0 +1,59 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, test } from "vitest"; + +import { + GO_DOCKERFILE_PATH, + TS_DOCKERFILE_PATH, + renderDockerfile, +} from "./render-service-dockerfile.ts"; + +// Both against the real, checked-in Dockerfile and the real catalog — no hardcoded version +// literal, so a catalog bump never makes this drift-detection test itself go stale. +const currentTsDockerfile = readFileSync(TS_DOCKERFILE_PATH, "utf8"); + +describe("renderDockerfile against the real catalog and Dockerfile", () => { + test("round-trips: rewriting the checked-in Dockerfile reproduces it byte for byte (no drift)", () => { + expect(renderDockerfile(currentTsDockerfile)).toBe(currentTsDockerfile); + }); + + test("rendering twice is stable", () => { + expect(renderDockerfile(currentTsDockerfile)).toBe(renderDockerfile(currentTsDockerfile)); + }); + + // The Go tree still `go:embed`s its own copy for a dependency that hasn't been removed yet; + // this is the single place that keeps the two copies in sync (folded from the former + // dockerfile-go-sync.unit.test.ts), until apps/cli-go is deleted. + test("the Go tree's embedded Dockerfile is a byte copy of the TS-owned one", () => { + const goDockerfile = readFileSync(GO_DOCKERFILE_PATH, "utf8"); + expect(goDockerfile).toBe(currentTsDockerfile); + }); + + test("detects drift when a generated line is hand-edited", () => { + // A hand-edit to a generated tag (bumping gotrue without touching the catalog) must not + // round-trip back to itself — this is the drift check's whole point. + const tampered = currentTsDockerfile.replace( + /FROM supabase\/gotrue:v[^\s]+ AS gotrue/, + "FROM supabase/gotrue:v9.9.9 AS gotrue", + ); + expect(tampered).not.toBe(currentTsDockerfile); + expect(renderDockerfile(tampered)).not.toBe(tampered); + // It heals back to the catalog-pinned tag, not just "detects a difference". + expect(renderDockerfile(tampered)).toBe(currentTsDockerfile); + }); + + test("leaves every hand-/Dependabot-managed line untouched, including comments and pg14", () => { + const rendered = renderDockerfile(currentTsDockerfile); + for (const line of [ + "FROM library/kong:2.8.1 AS kong", + "FROM supabase/pgadmin-schema-diff:cli-0.0.5 AS differ", + "FROM supabase/migra:3.0.1663481299 AS migra", + "FROM supabase/pg_prove:3.36 AS pgprove", + ]) { + expect(rendered).toContain(line); + } + // pg14 has no slim build: its tag is whatever the checked-in file already pins, verbatim. + const pg14Line = currentTsDockerfile.split("\n").find((line) => line.endsWith("AS pg14")); + expect(pg14Line).toBeDefined(); + expect(rendered).toContain(pg14Line); + }); +}); diff --git a/apps/cli/src/command-internal/db-image.unit.test.ts b/apps/cli/src/command-internal/db-image.unit.test.ts index abbf38809d..22fd3a96d8 100644 --- a/apps/cli/src/command-internal/db-image.unit.test.ts +++ b/apps/cli/src/command-internal/db-image.unit.test.ts @@ -10,17 +10,16 @@ import { dockerfileServiceImage, dockerfileServiceImageRaw, } from "../shared/services/dockerfile-images.ts"; -import { - POSTGRES_FALLBACK_IMAGE_PG14, - POSTGRES_FALLBACK_IMAGE_PG15, - POSTGRES_FALLBACK_IMAGE_PG15_SLIM, -} from "../shared/services/services.shared.ts"; import { imageTag, toSlimImage } from "../shared/services/slim-images.ts"; import { resolveDbImage } from "./db-image.ts"; const currentPostgres = dockerfileServiceImageRaw("pg"); const currentPostgresTag = imageTag(currentPostgres) ?? ""; -const pg15SlimTag = imageTag(POSTGRES_FALLBACK_IMAGE_PG15_SLIM) ?? ""; +// PG13/15 and PG14 now come from the Dockerfile's generated `pg15`/hand-pinned `pg14` stages +// (the single version table), not hardcoded fallback constants. +const pg15Image = dockerfileServiceImageRaw("pg15"); +const pg15Tag = imageTag(pg15Image) ?? ""; +const pg14Image = dockerfileServiceImageRaw("pg14"); const withTemp = () => mkdtempSync(join(tmpdir(), "db-image-")); @@ -49,16 +48,16 @@ describe("resolveDbImage", () => { const dir = withTemp(); return Effect.gen(function* () { expect(yield* resolve(dir, 13)).toEqual({ - image: POSTGRES_FALLBACK_IMAGE_PG15, - configImage: POSTGRES_FALLBACK_IMAGE_PG15, + image: pg15Image, + configImage: pg15Image, }); expect(yield* resolve(dir, 14)).toEqual({ - image: POSTGRES_FALLBACK_IMAGE_PG14, - configImage: POSTGRES_FALLBACK_IMAGE_PG14, + image: pg14Image, + configImage: pg14Image, }); expect(yield* resolve(dir, 15)).toEqual({ - image: POSTGRES_FALLBACK_IMAGE_PG15, - configImage: POSTGRES_FALLBACK_IMAGE_PG15, + image: pg15Image, + configImage: pg15Image, }); expect(yield* resolve(dir, 17)).toEqual({ image: dockerfileServiceImage("pg"), @@ -93,8 +92,8 @@ describe("resolveDbImage", () => { const dir = withTemp(); return Effect.gen(function* () { expect(yield* resolve(dir, 14, "16.0.0.1")).toEqual({ - image: POSTGRES_FALLBACK_IMAGE_PG14, - configImage: POSTGRES_FALLBACK_IMAGE_PG14, + image: pg14Image, + configImage: pg14Image, }); rmSync(dir, { recursive: true, force: true }); }); @@ -106,8 +105,8 @@ describe("resolveDbImage", () => { const dir = withTemp(); return Effect.gen(function* () { expect(yield* resolve(dir, 14)).toEqual({ - image: POSTGRES_FALLBACK_IMAGE_PG14, - configImage: POSTGRES_FALLBACK_IMAGE_PG14, + image: pg14Image, + configImage: pg14Image, }); rmSync(dir, { recursive: true, force: true }); }); @@ -118,16 +117,12 @@ describe("resolveDbImage", () => { const dir = withTemp(); return Effect.gen(function* () { expect(yield* resolve(dir, 15)).toEqual({ - image: - toSlimImage("pg", POSTGRES_FALLBACK_IMAGE_PG15_SLIM) ?? - POSTGRES_FALLBACK_IMAGE_PG15_SLIM, - configImage: POSTGRES_FALLBACK_IMAGE_PG15_SLIM, + image: toSlimImage("pg", pg15Image) ?? pg15Image, + configImage: pg15Image, }); expect(yield* resolve(dir, 13)).toEqual({ - image: - toSlimImage("pg", POSTGRES_FALLBACK_IMAGE_PG15_SLIM) ?? - POSTGRES_FALLBACK_IMAGE_PG15_SLIM, - configImage: POSTGRES_FALLBACK_IMAGE_PG15_SLIM, + image: toSlimImage("pg", pg15Image) ?? pg15Image, + configImage: pg15Image, }); rmSync(dir, { recursive: true, force: true }); }); @@ -149,13 +144,11 @@ describe("resolveDbImage", () => { it.effect("rewrites a current PG15 pin to the slim registry", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const dir = withTemp(); - writePin(dir, pg15SlimTag); + writePin(dir, pg15Tag); return Effect.gen(function* () { expect(yield* resolve(dir, 15)).toEqual({ - image: - toSlimImage("pg", POSTGRES_FALLBACK_IMAGE_PG15_SLIM) ?? - POSTGRES_FALLBACK_IMAGE_PG15_SLIM, - configImage: POSTGRES_FALLBACK_IMAGE_PG15_SLIM, + image: toSlimImage("pg", pg15Image) ?? pg15Image, + configImage: pg15Image, }); rmSync(dir, { recursive: true, force: true }); }); diff --git a/apps/cli/src/shared/services/Dockerfile b/apps/cli/src/shared/services/Dockerfile index a9c85ee0ca..7b404856de 100644 --- a/apps/cli/src/shared/services/Dockerfile +++ b/apps/cli/src/shared/services/Dockerfile @@ -1,21 +1,29 @@ -# Exposed for updates by .github/dependabot.yml -FROM supabase/postgres:17.6.1.171 AS pg +# The tag pinned below is generated from packages/stack/src/Artifacts.ts by +# apps/cli/scripts/render-service-dockerfile.ts, for these aliases only: pg, pg15, mailpit, +# postgrest, pgmeta, studio, imgproxy, edgeruntime, vector, supavisor, gotrue, realtime, storage, +# logflare. Do not hand-edit those tags; a CI drift check enforces this. Run the generator after +# a catalog update. Everything else in this file (this comment, kong, pg14, the one-shot job +# images) is hand- or Dependabot-managed. +FROM supabase/postgres:17.11.0.002 AS pg +FROM supabase/postgres:15.19.0.002 AS pg15 +# Postgres 14 has no slim build; bumped by hand (Dependabot ignores supabase/postgres). +FROM supabase/postgres:14.1.0.89 AS pg14 # New always-on service alias: extend SERVICE_IMAGE_ALIASES in # shared/services/services.shared.ts and DOCKERFILE_ALIAS_BY_SERVICE in # commands/start/start.gates.ts. FROM library/kong:2.8.1 AS kong -FROM axllent/mailpit:v1.30.2 AS mailpit -FROM postgrest/postgrest:v16.3 AS postgrest +FROM axllent/mailpit:v1.31.3 AS mailpit +FROM postgrest/postgrest:v16.4 AS postgrest FROM supabase/postgres-meta:v0.99.0 AS pgmeta -FROM supabase/studio:2026.09.14-sha-4dd8a95 AS studio -FROM darthsim/imgproxy:v3.8.0 AS imgproxy +FROM supabase/studio:2026.09.28-sha-5e59b60 AS studio +FROM darthsim/imgproxy:v3.26.0 AS imgproxy FROM supabase/edge-runtime:v1.77.1 AS edgeruntime -FROM timberio/vector:0.53.0-alpine AS vector +FROM timberio/vector:0.58.0-alpine AS vector FROM supabase/supavisor:2.9.13 AS supavisor FROM supabase/gotrue:v2.197.0 AS gotrue -FROM supabase/realtime:v2.135.3 AS realtime -FROM supabase/storage-api:v1.77.0 AS storage -FROM supabase/logflare:1.50.12 AS logflare +FROM supabase/realtime:v2.140.3 AS realtime +FROM supabase/storage-api:v1.79.26 AS storage +FROM supabase/logflare:1.50.15 AS logflare # One-shot job images (not started as long-running containers); differ's alias # is resolved via dockerfileServiceImage("differ") in commands/db/diff/pgadmin-diff.ts. FROM supabase/pgadmin-schema-diff:cli-0.0.5 AS differ diff --git a/apps/cli/src/shared/services/dockerfile-go-sync.unit.test.ts b/apps/cli/src/shared/services/dockerfile-go-sync.unit.test.ts deleted file mode 100644 index d8d058c146..0000000000 --- a/apps/cli/src/shared/services/dockerfile-go-sync.unit.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { readFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; -import { describe, expect, test } from "vitest"; -import serviceImagesDockerfile from "./Dockerfile" with { type: "text" }; - -// The Go tree still `go:embed`s its own copy for a dependency that hasn't been removed -// yet; this keeps the two copies in sync until apps/cli-go is deleted, at which point -// this test should be deleted alongside it. -const GO_DOCKERFILE_PATH = fileURLToPath( - new URL("../../../../cli-go/pkg/config/templates/Dockerfile", import.meta.url), -); - -describe("Go Dockerfile sync guard", () => { - test("keeps the Go tree's embedded Dockerfile byte-identical to the TS-owned copy", () => { - const goDockerfile = readFileSync(GO_DOCKERFILE_PATH, "utf8"); - expect(goDockerfile).toBe(serviceImagesDockerfile); - }); -}); diff --git a/apps/cli/src/shared/services/services.shared.ts b/apps/cli/src/shared/services/services.shared.ts index 5a7b79a98e..20680bdc43 100644 --- a/apps/cli/src/shared/services/services.shared.ts +++ b/apps/cli/src/shared/services/services.shared.ts @@ -10,6 +10,7 @@ import { ErrorActionabilityId, } from "../telemetry/error-actionability.ts"; import { + dockerfileServiceImageRaw, dockerfileServiceImages, parseDockerfileServiceImages, type DockerfileImageSpec, @@ -119,19 +120,20 @@ export function localServiceImagesFromDockerfile( const LOCAL_SERVICE_IMAGES = localServiceImagesFromSpecs(dockerfileServiceImages); -export const POSTGRES_FALLBACK_IMAGE_PG14 = "supabase/postgres:14.1.0.89"; -/** Flag-off PG13/15 docker.io pin. */ -export const POSTGRES_FALLBACK_IMAGE_PG15 = "supabase/postgres:15.8.1.085"; -/** Published slim PG15 pin; flag-on majors 13/15 slim-translate this, not 15.8. */ -export const POSTGRES_FALLBACK_IMAGE_PG15_SLIM = "supabase/postgres:15.14.1.167"; - +/** + * Resolves PG13/14/15 against the Dockerfile's `pg15`/`pg14` stages — the single version table, + * generated (`pg15`) or hand-pinned (`pg14`, no slim build) from the stack catalog. Always the + * raw docker.io reference; slim translation happens downstream via the same `toSlimImage("pg", + * …)` path every other slim-capable service uses, since a slim-capable service's Dockerfile tag + * always matches a catalog upstream version by construction. + */ export function postgresImageForDbMajorVersion(majorVersion: number): string | undefined { switch (majorVersion) { case 13: case 15: - return slimImagesEnabled() ? POSTGRES_FALLBACK_IMAGE_PG15_SLIM : POSTGRES_FALLBACK_IMAGE_PG15; + return dockerfileServiceImageRaw("pg15"); case 14: - return POSTGRES_FALLBACK_IMAGE_PG14; + return dockerfileServiceImageRaw("pg14"); default: return undefined; } diff --git a/apps/cli/src/shared/services/services.shared.unit.test.ts b/apps/cli/src/shared/services/services.shared.unit.test.ts index d41982a1a3..ef4bdc2be5 100644 --- a/apps/cli/src/shared/services/services.shared.unit.test.ts +++ b/apps/cli/src/shared/services/services.shared.unit.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { Effect, Redacted } from "effect"; import { FetchHttpClient } from "effect/unstable/http"; import serviceImagesDockerfile from "./Dockerfile" with { type: "text" }; +import { dockerfileServiceImageRaw } from "./dockerfile-images.ts"; import { fetchLinkedServiceVersions, listLocalServiceVersions, @@ -108,12 +109,18 @@ describe("services shared", () => { ]); }); - test("keeps the established PG13/15 fallback unless the slim flag is on", () => { - expect(postgresImageForDbMajorVersion(13)).toBe("supabase/postgres:15.8.1.085"); - expect(postgresImageForDbMajorVersion(15)).toBe("supabase/postgres:15.8.1.085"); + test("resolves PG13/14/15 from the Dockerfile's pg15/pg14 stages, regardless of the slim flag", () => { + const pg15 = dockerfileServiceImageRaw("pg15"); + const pg14 = dockerfileServiceImageRaw("pg14"); + expect(postgresImageForDbMajorVersion(13)).toBe(pg15); + expect(postgresImageForDbMajorVersion(15)).toBe(pg15); + expect(postgresImageForDbMajorVersion(14)).toBe(pg14); + // Always the raw docker.io reference; slim translation is a separate, downstream concern + // (`toSlimImage`/`slimImageForCurrentPin`), so this function itself doesn't read the flag. vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); - expect(postgresImageForDbMajorVersion(13)).toBe("supabase/postgres:15.14.1.167"); - expect(postgresImageForDbMajorVersion(15)).toBe("supabase/postgres:15.14.1.167"); + expect(postgresImageForDbMajorVersion(13)).toBe(pg15); + expect(postgresImageForDbMajorVersion(15)).toBe(pg15); + expect(postgresImageForDbMajorVersion(14)).toBe(pg14); }); test("slim-translates a version override that matches a catalog pin", () => { diff --git a/apps/cli/src/shared/services/slim-images.catalog-alignment.unit.test.ts b/apps/cli/src/shared/services/slim-images.catalog-alignment.unit.test.ts new file mode 100644 index 0000000000..12baff5a9b --- /dev/null +++ b/apps/cli/src/shared/services/slim-images.catalog-alignment.unit.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { dockerfileServiceImageRaw } from "./dockerfile-images.ts"; +import { slimCatalogPin, toSlimImage } from "./slim-images.ts"; + +/** + * Every slim-capable Dockerfile alias, against the real (unmocked) catalog. The Dockerfile is + * generated from that same catalog (`render-service-dockerfile.ts`), so each alias's raw tag is + * always one of the catalog's own upstream versions by construction — no fixture, no + * `vi.mock`, exercising the true "one-time alignment" invariant end to end. + */ +const SLIM_CAPABLE_ALIASES = [ + "pg", + "pg15", + "gotrue", + "postgrest", + "realtime", + "storage", + "edgeruntime", + "studio", + "pgmeta", + "logflare", + "supavisor", + "vector", + "imgproxy", + "mailpit", +] as const; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("slim mode against the real catalog", () => { + it.each(SLIM_CAPABLE_ALIASES)("resolves the %s alias to its pinned catalog image", (alias) => { + vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); + const raw = dockerfileServiceImageRaw(alias); + const pin = slimCatalogPin(alias, raw); + expect(pin).toBeDefined(); + const resolved = toSlimImage(alias, raw); + expect(resolved).toBeDefined(); + expect(resolved).toMatch(/^ghcr\.io\/supabase\/cli\//); + }); + + it("has no slim entry for aliases with no slim build", () => { + for (const alias of ["kong", "pg14", "differ", "migra", "pgprove"]) { + const raw = dockerfileServiceImageRaw(alias); + expect(toSlimImage(alias, raw)).toBeUndefined(); + } + }); +}); diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index a8b54d891e..4e9dabbb92 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -5,11 +5,13 @@ const SLIM_IMAGE_PREFIX = "ghcr.io/supabase/cli/"; /** * Maps embedded-Dockerfile aliases onto the slim service catalog. Aliases with - * no slim build (kong, the `differ`/`migra`/`pgprove` job images) are absent and - * keep their docker.io reference. OrioleDB tags are excluded in `slimCatalogPin`. + * no slim build (kong, `pg14`, the `differ`/`migra`/`pgprove` job images) are + * absent and keep their docker.io reference. OrioleDB tags are excluded in + * `slimCatalogPin`. */ const SLIM_SERVICE_BY_ALIAS = { pg: "postgres", + pg15: "postgres", gotrue: "auth", postgrest: "postgrest", realtime: "realtime", diff --git a/apps/cli/src/shared/services/slim-images.unit.test.ts b/apps/cli/src/shared/services/slim-images.unit.test.ts index da6303d63f..9122588ec3 100644 --- a/apps/cli/src/shared/services/slim-images.unit.test.ts +++ b/apps/cli/src/shared/services/slim-images.unit.test.ts @@ -51,6 +51,7 @@ afterEach(() => { describe("slimCatalogPin", () => { it.each([ ["pg", "postgres"], + ["pg15", "postgres"], ["gotrue", "auth"], ["postgrest", "postgrest"], ["realtime", "realtime"], @@ -129,7 +130,7 @@ describe("slimCatalogPin", () => { }); it("is absent for aliases with no slim build", () => { - for (const alias of ["kong", "differ", "migra", "pgprove"]) { + for (const alias of ["kong", "pg14", "differ", "migra", "pgprove"]) { expect(slimCatalogPin(alias, dockerfileServiceImageRaw(alias))).toBeUndefined(); } }); @@ -251,4 +252,12 @@ describe("slimImageForCurrentPin", () => { const current = "supabase/gotrue:v2.197.0"; expect(slimImageForCurrentPin("gotrue", current, "v1.67.0")).toBe("supabase/gotrue:v1.67.0"); }); + + it("falls back to the upstream image on the linked-pin fallback path (a hosted version that doesn't match the pin)", () => { + vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); + const current = "supabase/gotrue:v2.197.0"; + // The linked project's hosted version (v1.67.0) differs from the catalog's pinned + // upstream version (v2.197.0): stay on the upstream (non-slim) image instead of guessing. + expect(slimImageForCurrentPin("gotrue", current, "v1.67.0")).toBe("supabase/gotrue:v1.67.0"); + }); }); diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 0bd6497587..f188b5b343 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -45,6 +45,14 @@ export interface ArtifactPin { readonly revision: number; /** `ghcr.io/supabase/cli/:@sha256:`. */ readonly image: string; + /** + * The exact upstream image this release was built or mirrored from, as slim-services recorded + * it: the release manifest's `upstream_image` for a derived service, or the release's + * `oci-provenance.json` `source` for a mirrored one (vector, mailpit, imgproxy). Normalized to + * the Dockerfile's `FROM` form — no leading `docker.io/`, no digest — so legacy non-slim mode + * can use it as the upstream tag directly. + */ + readonly upstreamImage: string; readonly natives: Readonly>; } @@ -102,6 +110,7 @@ const definitions: Readonly> = { revision: 0, image: "ghcr.io/supabase/cli/postgres:17.11.0.002-r0@sha256:5a551a204a41267c4f78a2e5b34657f5c6fdd39eddccd50262466481b95739cc", + upstreamImage: "supabase/postgres:17.11.0.002", natives: { "darwin-arm64": { archive: "10410e402c77210a0543539d9bafcabd0c04923d8e796928cbd2a053cf7b4f4f", @@ -125,6 +134,7 @@ const definitions: Readonly> = { revision: 0, image: "ghcr.io/supabase/cli/postgres:15.19.0.002-r0@sha256:a3f343f19323497a5766fa4e86a51dba495a8eca354cc128d879b4632c6bd017", + upstreamImage: "supabase/postgres:15.19.0.002", natives: { "darwin-arm64": { archive: "59b05ca76db9d807803840264d885a5ff435e552f294d7f22ca831fac1ffb4a3", @@ -149,6 +159,7 @@ const definitions: Readonly> = { revision: 0, image: "ghcr.io/supabase/cli/postgrest:v16.4-r0@sha256:63a8d4acfdeb107b6568f4582759c78072100ef07951a7fbe58c9a51241138a7", + upstreamImage: "postgrest/postgrest:v16.4", natives: { "darwin-arm64": { archive: "a1f5449d739404cbd042ec9dbabadea6dcf810db636e44c7176b859d5e78ab07", @@ -173,6 +184,7 @@ const definitions: Readonly> = { revision: 0, image: "ghcr.io/supabase/cli/auth:v2.197.0-r0@sha256:7eb303831d170865840dbb3f9018b48561c988704ad1d9a2bafabb4e93e92da5", + upstreamImage: "supabase/gotrue:v2.197.0", natives: { "darwin-arm64": { archive: "eb365c4aea1e1cd04998ed16cd49b1b90e6a097ce633e5ee55f8bbbebdd349c7", @@ -197,6 +209,7 @@ const definitions: Readonly> = { revision: 0, image: "ghcr.io/supabase/cli/realtime:v2.140.3-r0@sha256:af7b883647770351eef150cc16e1f12d7fe4b4920324e2e40487961ad3b22789", + upstreamImage: "supabase/realtime:v2.140.3", natives: { "darwin-arm64": { archive: "e015ced15ed2110c2a0123b6e75edb2aeb12ac6bdbacffdfb296262a4215f97e", @@ -222,6 +235,7 @@ const definitions: Readonly> = { revision: 0, image: "ghcr.io/supabase/cli/storage:v1.79.26-r0@sha256:277bac6513c4b6d5e45dc80a5a98021a407d83852393fd368585e51e0d11e41b", + upstreamImage: "supabase/storage-api:v1.79.26", natives: { "darwin-arm64": { archive: "5039b176ed7fce641c9bb94a6cfb232450775680c7bb268f50910b52b4bd5047", @@ -247,6 +261,7 @@ const definitions: Readonly> = { revision: 0, image: "ghcr.io/supabase/cli/imgproxy:v3.26.0-r0@sha256:5871582cf6c5140d3b50b21e0d1aa236fdf46ee6133e3bfec1baf63545e01cf0", + upstreamImage: "ghcr.io/imgproxy/imgproxy:v3.26.0", natives: { "darwin-arm64": { archive: "1f82084b056bf4806bf492bca0b6f963068d07a6776a2a122b61da940732683b", @@ -271,6 +286,7 @@ const definitions: Readonly> = { revision: 0, image: "ghcr.io/supabase/cli/edge-runtime:v1.77.1-r0@sha256:98582ce39914bba6ac856a5bf6ed3c1584ce4616687ca359c8acd5f59dd9ff59", + upstreamImage: "supabase/edge-runtime:v1.77.1", natives: { "darwin-arm64": { archive: "e52fdbedccf258fd9a427b19bd34569ca210c1ca257a85fc23f66b4e18c1d6a1", @@ -295,6 +311,7 @@ const definitions: Readonly> = { revision: 0, image: "ghcr.io/supabase/cli/studio:2026.09.28-sha-5e59b60-r0@sha256:4cf4f70978bb0866d1644b43cb0d23acc4b4ef7a898592043ad0d13cad8059be", + upstreamImage: "supabase/studio:2026.09.28-sha-5e59b60", natives: { "darwin-arm64": { archive: "37f420f6af3d5ee7dab884e8c39fe2c3bcddcee0d90e53d1e075a05ae3cb4b73", @@ -319,6 +336,7 @@ const definitions: Readonly> = { revision: 0, image: "ghcr.io/supabase/cli/pgmeta:v0.99.0-r0@sha256:c19f6bba3ab66fcf30c8737d2361ec9f8e12a4fa8a071481f53366dc13e83340", + upstreamImage: "supabase/postgres-meta:v0.99.0", natives: { "darwin-arm64": { archive: "337f8cc6a23d93f3f9cfeed12de2a86d686ce6f4346ff9b334b5dfdcba7e890f", @@ -343,6 +361,7 @@ const definitions: Readonly> = { revision: 0, image: "ghcr.io/supabase/cli/mailpit:v1.31.3-r0@sha256:ed9b00c609e77e99c79b93f1178255ebc271868920f2c69a8d166bd5634ed10d", + upstreamImage: "axllent/mailpit:v1.31.3", natives: { "darwin-arm64": { archive: "d460a6a55693a2a321ad83ebb83417b8078324751a744f982bb3512da0632712", @@ -367,6 +386,7 @@ const definitions: Readonly> = { revision: 0, image: "ghcr.io/supabase/cli/analytics:v1.50.15-r0@sha256:cea1595bafa6ab32df4854d407261ddef6e35394e300ae29770ba48c49c8dc7e", + upstreamImage: "supabase/logflare:1.50.15", natives: { "darwin-arm64": { archive: "336c78c501aff270b0fcd2a42229b76d9f2feafa646f49a59c3de29358e3de17", @@ -392,6 +412,7 @@ const definitions: Readonly> = { revision: 0, image: "ghcr.io/supabase/cli/vector:0.58.0-r0@sha256:5dcf67db0ee378caa87f3395cb9484ebe3e97bb0334d119f2ac33116e00c5773", + upstreamImage: "timberio/vector:0.58.0-alpine", natives: { "darwin-arm64": { archive: "567245cf9a7d54eee45ecf74e1c9a61ca6d02cdca103edc7b32b005e54f4e632", @@ -417,6 +438,7 @@ const definitions: Readonly> = { revision: 0, image: "ghcr.io/supabase/cli/pooler:v2.9.13-r0@sha256:3a32b56d03675ed24e84408afcbb12fa52b7ec6e7741f913770fea5b66c2ddd3", + upstreamImage: "supabase/supavisor:2.9.13", natives: { "darwin-arm64": { archive: "c05285be2a945a29d5be491661926f8c88d976540d49ddbcab10aa7276b29934", @@ -548,15 +570,25 @@ export const postgresVersion = (version: string): string => /** Service kinds in artifact catalog order. */ export const artifactServiceKinds = (): ReadonlyArray => Record.keys(definitions); -/** Every catalog pin in catalog order, including additional upstream lines. */ +/** + * Every catalog pin in catalog order, including additional upstream lines. `isDefault` marks the + * pin `resolveArtifact` picks when no version is requested (postgres's 17.x line today); every + * other pin (postgres's 15.x additional line) carries `isDefault: false`. + */ export const catalogPins = (): ReadonlyArray<{ readonly service: ServiceKind; readonly sourceService: string; readonly pin: ArtifactPin; + readonly isDefault: boolean; }> => artifactServiceKinds().flatMap((service) => { - const { sourceService, pins } = definitions[service]; - return Object.values(pins).map((pin) => ({ service, sourceService, pin })); + const { sourceService, defaultVersion, pins } = definitions[service]; + return Object.values(pins).map((pin) => ({ + service, + sourceService, + pin, + isDefault: pin.upstreamVersion === defaultVersion, + })); }); const artifactKey = (artifact: SlimServicesArtifact): string => diff --git a/packages/stack/src/Artifacts.unit.test.ts b/packages/stack/src/Artifacts.unit.test.ts index 0e0ecf510d..50c55fd78c 100644 --- a/packages/stack/src/Artifacts.unit.test.ts +++ b/packages/stack/src/Artifacts.unit.test.ts @@ -1,5 +1,14 @@ import { expect, it } from "@effect/vitest"; -import { slimImageMirrors } from "./Artifacts.ts"; +import { catalogPins, slimImageMirrors } from "./Artifacts.ts"; + +it("carries a normalized upstreamImage for every catalog pin", () => { + for (const { pin } of catalogPins()) { + expect(pin.upstreamImage).not.toBe(""); + expect(pin.upstreamImage).not.toContain("docker.io/"); + expect(pin.upstreamImage).not.toContain("@sha256:"); + expect(pin.upstreamImage.split(":").at(-1)).not.toBe(""); + } +}); it("rewrites a GHCR catalog image onto ECR Public and keeps the tag and digest", () => { expect( From 82b0e850e9af76fce6fc318d8846cab5b985df97 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 16:11:12 +0200 Subject: [PATCH 19/45] test: derive pinned versions from the catalog Replaces literals that happened to equal the current catalog/Dockerfile pin with values derived from the catalog (`catalogPins()`/`postgresVersion`) or the Dockerfile (`dockerfileServiceImageRaw`), so a routine version bump no longer turns these integration tests red. Mocked, self-contained fixture catalogs (services.shared/slim-images unit tests) are unaffected: they only ever compare against their own literals. --- .../shared/pgdelta.seam.integration.test.ts | 42 ++++++++------ .../services/services.integration.test.ts | 57 ++++++++++++++++--- ...dRunner.initialization.integration.test.ts | 16 ++++-- .../DatabaseSnapshot.integration.test.ts | 9 ++- .../ProcessRecipe.integration.test.ts | 15 ++++- .../DockerDatabaseStorage.integration.test.ts | 4 +- 6 files changed, 108 insertions(+), 35 deletions(-) diff --git a/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts b/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts index ba3662a42a..3cdd90f8a7 100644 --- a/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts +++ b/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts @@ -33,13 +33,25 @@ import { DeclarativeShadowDbError } from "./pgdelta.errors.ts"; import { declarativeSeamLayer } from "./pgdelta.seam.layer.ts"; import { DeclarativeSeam } from "./pgdelta.seam.service.ts"; -// The real catalog's postgres pin has moved past the Dockerfile's own pinned `pg` tag (the -// "catalog leads the Dockerfile until Dependabot catches up" gap), so `toSlimImage` would find no -// match and fall back to the upstream (non-slim) image regardless of `SUPABASE_USE_SLIM_IMAGES` — -// masking the family-mismatch check below. Pin a fixture catalog entry at the Dockerfile's actual -// `pg` tag (17.6.1.171) instead, the same pattern `slim-images.unit.test.ts` uses. `vi.mock` -// factories are hoisted above every other top-level statement, so the tag is inlined rather than -// referencing `dockerfileServiceImageRaw` here. +// This fixture catalog's pin must be keyed to the Dockerfile's own `pg` tag, or `toSlimImage` +// would find no match and fall back to the upstream (non-slim) image regardless of +// `SUPABASE_USE_SLIM_IMAGES` — masking the family-mismatch check below. `vi.hoisted` runs before +// every top-level `import` (including this file's own), so `dockerfileServiceImageRaw` isn't +// bound yet when this runs; `require` (CJS, unaffected by that ESM hoisting order) reads the +// Dockerfile directly instead, keeping this correct across every future Dockerfile bump. +const pgTag = vi.hoisted(() => { + const fs: typeof import("node:fs") = require("node:fs"); + const url: typeof import("node:url") = require("node:url"); + const dockerfilePath = url.fileURLToPath( + new URL("../../../shared/services/Dockerfile", import.meta.url), + ); + const match = /^FROM\s+supabase\/postgres:(\S+)\s+AS\s+pg$/m.exec( + fs.readFileSync(dockerfilePath, "utf8"), + ); + if (match?.[1] === undefined) throw new Error("pg tag not found in the Dockerfile"); + return match[1]; +}); + vi.mock("@supabase/stack/internal/artifacts", () => { const digest = "d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c"; const nativePin = { archive: digest, manifest: digest }; @@ -49,9 +61,9 @@ vi.mock("@supabase/stack/internal/artifacts", () => { service: "database", sourceService: "postgres", pin: { - upstreamVersion: "17.6.1.171", + upstreamVersion: pgTag, revision: 0, - image: `ghcr.io/supabase/cli/postgres:17.6.1.171-r0@sha256:${digest}`, + image: `ghcr.io/supabase/cli/postgres:${pgTag}-r0@sha256:${digest}`, natives: { "darwin-arm64": nativePin, "linux-amd64": nativePin, @@ -240,12 +252,11 @@ describe("declarativeSeamLayer.ensureLocalPostgresImageCurrent", () => { it.effect("flags a stale slim container when only its revision has drifted from a hotfix", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const dir = tmp.current; - // Same upstream version and family as the fixture catalog's pin (17.6.1.171), but at a - // different revision (r1, a different digest) — the hotfix-drift case this model exists - // to catch, and the one a bare upstream-version comparison would mask. + // Same upstream version and family as the fixture catalog's pin, but at a different + // revision (r1, a different digest) — the hotfix-drift case this model exists to catch, + // and the one a bare upstream-version comparison would mask. const { layer } = setup(dir, { - dbInspectImage: - "ghcr.io/supabase/cli/postgres:17.6.1.171-r1@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + dbInspectImage: `ghcr.io/supabase/cli/postgres:${pgTag}-r1@sha256:${"a".repeat(64)}`, }); return Effect.gen(function* () { const seam = yield* DeclarativeSeam; @@ -269,8 +280,7 @@ describe("declarativeSeamLayer.ensureLocalPostgresImageCurrent", () => { const dir = tmp.current; // The exact image (release version and digest) the fixture catalog pins at r0. const { layer } = setup(dir, { - dbInspectImage: - "ghcr.io/supabase/cli/postgres:17.6.1.171-r0@sha256:d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c", + dbInspectImage: `ghcr.io/supabase/cli/postgres:${pgTag}-r0@sha256:d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c`, }); return Effect.gen(function* () { const seam = yield* DeclarativeSeam; diff --git a/apps/cli/src/commands/services/services.integration.test.ts b/apps/cli/src/commands/services/services.integration.test.ts index 26720f44d4..b6f6d3b3cf 100644 --- a/apps/cli/src/commands/services/services.integration.test.ts +++ b/apps/cli/src/commands/services/services.integration.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "@effect/vitest"; +import { catalogPins } from "@supabase/stack/internal/artifacts"; import { BunServices } from "@effect/platform-bun"; import { CliOutput, Command } from "effect/unstable/cli"; import { @@ -42,6 +43,25 @@ import { services } from "./services.handler.ts"; const LOCAL_POSTGRES_VERSION = dockerfileServiceImageRaw("pg").split(":")[1] ?? ""; +/** + * The real stack catalog's pinned upstream version for `sourceService` — the default pin, or + * (for postgres) the additional 15.x line when `additional` is set. Derived rather than + * hardcoded, so a catalog bump never makes these "stack backend" assertions go stale. + */ +function catalogUpstreamVersion( + sourceService: string, + options: { readonly additional?: boolean } = {}, +): string { + const wantDefault = !(options.additional ?? false); + const entry = catalogPins().find( + (candidate) => candidate.sourceService === sourceService && candidate.isDefault === wantDefault, + ); + if (entry === undefined) { + throw new Error(`No catalog pin for '${sourceService}' (default=${wantDefault}).`); + } + return entry.pin.upstreamVersion; +} + /** Shape of one row in the `--output json` services array. */ const ServiceRows = Schema.Array( Schema.Struct({ @@ -333,13 +353,22 @@ describe("services", () => { expect(out.stderrText).toBe(""); expect(rows).toHaveLength(13); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "15.19.0.002" }), + expect.objectContaining({ + name: "ghcr.io/supabase/cli/postgres", + local: catalogUpstreamVersion("postgres", { additional: true }), + }), ); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/mailpit", local: "v1.31.3" }), + expect.objectContaining({ + name: "ghcr.io/supabase/cli/mailpit", + local: catalogUpstreamVersion("mailpit"), + }), ); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/vector", local: "0.58.0" }), + expect.objectContaining({ + name: "ghcr.io/supabase/cli/vector", + local: catalogUpstreamVersion("vector"), + }), ); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); @@ -359,7 +388,10 @@ describe("services", () => { const rows = yield* decodeServiceRows(out.stdoutText); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "15.19.0.002" }), + expect.objectContaining({ + name: "ghcr.io/supabase/cli/postgres", + local: catalogUpstreamVersion("postgres", { additional: true }), + }), ); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); @@ -377,10 +409,16 @@ describe("services", () => { const rows = yield* decodeServiceRows(out.stdoutText); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "17.11.0.002" }), + expect.objectContaining({ + name: "ghcr.io/supabase/cli/postgres", + local: catalogUpstreamVersion("postgres"), + }), ); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/auth", local: "v2.197.0" }), + expect.objectContaining({ + name: "ghcr.io/supabase/cli/auth", + local: catalogUpstreamVersion("auth"), + }), ); expect(out.stderrText).toContain("unsupported PostgreSQL major version: 16"); expect(out.stderrText).toContain("using default stack catalog versions"); @@ -396,7 +434,10 @@ describe("services", () => { const rows = yield* decodeServiceRows(out.stdoutText); expect(rows).toContainEqual( - expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", local: "17.11.0.002" }), + expect.objectContaining({ + name: "ghcr.io/supabase/cli/postgres", + local: catalogUpstreamVersion("postgres"), + }), ); expect(out.stderrText).toMatch(/^failed to read config:/); expect(out.stderrText).toContain("using default stack catalog versions"); @@ -580,7 +621,7 @@ major_version = 15 expect(stackRows).toContainEqual( expect.objectContaining({ name: "ghcr.io/supabase/cli/postgres", - local: "17.11.0.002", + local: catalogUpstreamVersion("postgres"), remote: "17.6.1.200", }), ); diff --git a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts index bd97dffa11..269d9307c2 100644 --- a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts +++ b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts @@ -2,7 +2,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Context, Deferred, Effect, Fiber, FileSystem, Layer, Path, Ref } from "effect"; import { TestClock } from "effect/testing"; -import { resolveArtifact } from "../Artifacts.ts"; +import { catalogPins, resolveArtifact } from "../Artifacts.ts"; import { initialization } from "../Commands.ts"; import { makeArtifactStore, @@ -22,6 +22,14 @@ const target = ? "linux-arm64" : undefined; +// The real catalog's default auth pin, not hardcoded — `resolveArtifact({ service: "auth", +// version })` below would otherwise fail once a catalog bump moves past a literal. +const authVersion = catalogPins().find((entry) => entry.sourceService === "auth" && entry.isDefault) + ?.pin.upstreamVersion; +if (authVersion === undefined) { + throw new Error("no default auth catalog pin found"); +} + const prepareAuthArtifact = Effect.fn(function* (cacheRoot: string, script: string) { if (target === undefined) return yield* Effect.fail("Unsupported test platform"); const fs = yield* FileSystem.FileSystem; @@ -96,7 +104,7 @@ it.live.skipIf(target === undefined || process.platform === "win32")( const error = yield* runner .run({ command: initialization.auth({ - version: "v2.197.0", + version: authVersion, databaseUrl: "postgresql://invalid:invalid@127.0.0.1:1/postgres", }), credentials, @@ -139,7 +147,7 @@ it.live.skipIf(target === undefined || process.platform === "win32")( const command = yield* runner .run({ command: initialization.auth({ - version: "v2.197.0", + version: authVersion, databaseUrl: "postgresql://invalid:invalid@127.0.0.1:1/postgres", }), credentials, @@ -197,7 +205,7 @@ it.effect.skipIf(target === undefined || process.platform === "win32")( const command = yield* runner .run({ command: initialization.auth({ - version: "v2.197.0", + version: authVersion, databaseUrl: "postgresql://invalid:invalid@127.0.0.1:1/postgres", }), credentials, diff --git a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts index 4c36d7e624..a89e190335 100644 --- a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts +++ b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts @@ -2,7 +2,7 @@ import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Crypto, Data, Effect, Exit, FileSystem, Path, Ref, Schema, Scope, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { resolveArtifact } from "../Artifacts.ts"; +import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; import { makeContainerRuntime } from "../runtime/Container.ts"; import { makeDockerDatabaseStorage } from "../storage/DockerDatabaseStorage.ts"; import { makeDockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; @@ -10,7 +10,8 @@ import { shellQuote } from "../storage/DockerSnapshotBackend.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { makeDatabaseSnapshots, type SnapshotScope } from "./DatabaseSnapshot.ts"; -const version = "17.11.0.002"; +// Derived from the real catalog (not hardcoded), so a Postgres pin bump never makes this go stale. +const version = postgresVersion("17"); class ShellError extends Data.TaggedError("ShellError")<{ readonly message: string }> {} @@ -510,8 +511,10 @@ for (const { name, make } of engines) const saved = yield* shell(`${tool}cat ${descriptor}`); const target = yield* engine.instance("target"); + // Any version different from `version` proves the mismatch check; this one is a + // synthetic sentinel, not a real catalog pin, so it stays correct across catalog bumps. yield* shell( - `printf '%s' ${shellQuote(saved.replace(version, "15.19.0.002"))} > ${descriptor}`, + `printf '%s' ${shellQuote(saved.replace(version, "0.0.0-version-mismatch"))} > ${descriptor}`, ); expect(yield* target.restoreSnapshot("key")).toBe(false); expect(yield* contents(target.data)).toBe(""); diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index 6bb6b8014b..9f14cc70ce 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -24,7 +24,7 @@ import { systemError } from "effect/PlatformError"; import * as Net from "node:net"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- the collision fixture owns a local HTTP listener. import * as NodeHttp from "node:http"; -import { resolveArtifact, type ServiceKind } from "../Artifacts.ts"; +import { catalogPins, resolveArtifact, type ServiceKind } from "../Artifacts.ts"; import { makeArtifactStore, type ArtifactRequest, @@ -49,12 +49,21 @@ import * as Pooler from "./Pooler.ts"; type TestCreation = RecipeCreation<"rest", Record> & { readonly service: "rest"; - readonly version: "v16.4"; + readonly version: string; }; +// The real catalog's default postgrest pin, not hardcoded — `makeProcessRecipe` resolves this +// through the real catalog, which would otherwise fail once a bump moves past a literal. +const postgrestVersion = catalogPins().find( + (entry) => entry.sourceService === "postgrest" && entry.isDefault, +)?.pin.upstreamVersion; +if (postgrestVersion === undefined) { + throw new Error("no default postgrest catalog pin found"); +} + const creation: TestCreation = { service: "rest", - version: "v16.4", + version: postgrestVersion, config: {}, }; diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index f7bb6d74e7..14021aa972 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -1058,7 +1058,9 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.writeFileString(path.join(nativeRoot, "data", "PG_VERSION"), "17\n"); yield* fs.writeFileString( path.join(nativeRoot, ".supabase-database-ready.json"), - '{"version":"17.11.0.002","runtime":"native","profile":"supabase"}', + // `makeDatabaseSnapshots` below resolves `version: "17"` through the real catalog + // (`postgresVersion`), so the ready marker must carry that same resolved version. + `{"version":"${postgresVersion("17")}","runtime":"native","profile":"supabase"}`, ); const nativeSnapshots = yield* makeDatabaseSnapshots({ instanceRoot: nativeRoot, From 61772bd8c163a831c401fb9e04f6e27dae553410 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 16:47:38 +0200 Subject: [PATCH 20/45] fix(ci): validate and escape recorded upstream images sync-artifacts-catalog.ts writes upstreamImage straight into Artifacts.ts as a TypeScript string literal. Validate the resolved value against an anchored image-reference pattern before returning it from resolveUpstreamImage, and serialize every field in serializePin with JSON.stringify instead of manual "${...}" interpolation, so a release manifest or provenance file carrying a quote or expression can't inject code into the generated catalog. --- .../scripts/sync-artifacts-catalog.test.ts | 15 ++++++++ .github/scripts/sync-artifacts-catalog.ts | 34 +++++++++++++++---- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index 4a6f3c2c7e..8d378247af 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -687,6 +687,21 @@ describe("refreshCatalogPin backfills upstreamImage", () => { expect(result.source).not.toContain("upstreamImage"); }); + + test("rejects a manifest upstream_image carrying a quote or template expression, writing nothing", async () => { + const digests = nativeDigests("ak"); + const io: RevisionIo = { + listReleaseTags: async () => ["analytics-v1.50.9-r0"], + fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r0", digests), + imageDigest: async () => digest("al"), + s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), + fetchManifest: manifestFixture('supabase/logflare:1.50.9"] }; import("evil"); //'), + }; + + await expect(refreshCatalogPin({ catalog: fixture, service: "analytics", io })).rejects.toThrow( + InvalidPayloadError, + ); + }); }); describe("against the real catalog", () => { diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index ff783bf2d7..c2a96497ea 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -287,6 +287,23 @@ function normalizeUpstreamImage(image: string): string { : withoutDigest; } +/** + * A normalized `upstreamImage`: one or more lowercase `registry`/`repository` path segments + * (each `[a-z0-9]`, optionally separated internally by `.`/`_`/`-`), a `:`, and a tag matching + * Docker's own tag grammar. Anchored, so no whitespace, quote, or template/expression syntax can + * slip through — this value gets embedded as a TypeScript string literal in `Artifacts.ts`. + */ +const IMAGE_REFERENCE_PATTERN = + /^[a-z0-9]+(?:[._-][a-z0-9]+)*(?:\/[a-z0-9]+(?:[._-][a-z0-9]+)*)*:[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$/; + +/** Validates a normalized `upstreamImage` before it's ever written to `Artifacts.ts`. */ +function validateUpstreamImage(image: string, context: string): string { + if (!IMAGE_REFERENCE_PATTERN.test(image)) { + throw new InvalidPayloadError(`${context} has an invalid upstreamImage '${image}'.`); + } + return image; +} + /** * Resolves `service`'s `upstreamImage` for the release `releaseVersion` (`-r`): * a mirrored service's `oci-provenance.json` `source`, or a derived service's per-target @@ -317,7 +334,7 @@ async function resolveUpstreamImage( `${service}-${releaseVersion} oci-provenance has no 'source' field.`, ); } - return normalizeUpstreamImage(source); + return validateUpstreamImage(normalizeUpstreamImage(source), `${service}-${releaseVersion}`); } if (io.fetchManifest === undefined) { @@ -345,18 +362,23 @@ async function resolveUpstreamImage( `${service}-${releaseVersion} manifests disagree on the upstream image: ${[...values].sort().join(", ")}.`, ); } - return [...values][0] as string; + return validateUpstreamImage([...values][0] as string, `${service}-${releaseVersion}`); } -/** Serializes a resolved pin into the object literal `Artifacts.ts` embeds, in catalog order. */ +/** + * Serializes a resolved pin into the object literal `Artifacts.ts` embeds, in catalog order. + * Every string field goes through `JSON.stringify`, not manual `"${…}"` interpolation — this + * text is written straight into TypeScript source that later gets imported, so an unescaped + * quote or template expression in any field (release metadata included) would inject code. + */ function serializePin(pin: ResolvedPin): string { const natives = NATIVE_TARGETS.map((target) => { const native = pin.natives[target]; - return `"${target}": { archive: "${native.archive}", manifest: "${native.manifest}" }`; + return `${JSON.stringify(target)}: { archive: ${JSON.stringify(native.archive)}, manifest: ${JSON.stringify(native.manifest)} }`; }).join(", "); const upstreamImage = - pin.upstreamImage === undefined ? "" : ` upstreamImage: "${pin.upstreamImage}",`; - return `{ upstreamVersion: "${pin.upstreamVersion}", revision: ${pin.revision}, image: "${pin.image}",${upstreamImage} natives: { ${natives} } }`; + pin.upstreamImage === undefined ? "" : ` upstreamImage: ${JSON.stringify(pin.upstreamImage)},`; + return `{ upstreamVersion: ${JSON.stringify(pin.upstreamVersion)}, revision: ${pin.revision}, image: ${JSON.stringify(pin.image)},${upstreamImage} natives: { ${natives} } }`; } interface PinSpan { From ce3fbd895a2a08754419affe35a1db7c791a8cb8 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 16:47:46 +0200 Subject: [PATCH 21/45] test: decouple remaining fixtures from pinned versions start.integration.test.ts's stack-backend database fixture derives its version from the real catalog (postgresVersion("17")) instead of a hardcoded pin, so a routine Postgres bump can't desync it from the version the code under test resolves. render-service-dockerfile's hand-managed-line preservation test no longer freezes kong/differ/ migra/pgprove's literal tags; it now tampers them with a synthetic marker and asserts the generator leaves them untouched, so a Dependabot bump of any of those can't fail this required check. --- .../render-service-dockerfile.unit.test.ts | 33 +++++++++++-------- .../db/start/start.integration.test.ts | 5 ++- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/apps/cli/scripts/render-service-dockerfile.unit.test.ts b/apps/cli/scripts/render-service-dockerfile.unit.test.ts index 55862853a2..0c4f9036d9 100644 --- a/apps/cli/scripts/render-service-dockerfile.unit.test.ts +++ b/apps/cli/scripts/render-service-dockerfile.unit.test.ts @@ -41,19 +41,26 @@ describe("renderDockerfile against the real catalog and Dockerfile", () => { expect(renderDockerfile(tampered)).toBe(currentTsDockerfile); }); - test("leaves every hand-/Dependabot-managed line untouched, including comments and pg14", () => { - const rendered = renderDockerfile(currentTsDockerfile); - for (const line of [ - "FROM library/kong:2.8.1 AS kong", - "FROM supabase/pgadmin-schema-diff:cli-0.0.5 AS differ", - "FROM supabase/migra:3.0.1663481299 AS migra", - "FROM supabase/pg_prove:3.36 AS pgprove", - ]) { - expect(rendered).toContain(line); + test("preserves hand-/Dependabot-managed lines verbatim, even when their own tags change", () => { + // Simulates a Dependabot bump of kong/differ/migra/pgprove: give each a synthetic marker + // tag (never a real pin) and confirm the generator leaves those exact lines alone rather + // than reverting or otherwise touching them. + const marker = "99.99.99-fixture-marker"; + let tampered = currentTsDockerfile; + for (const alias of ["kong", "differ", "migra", "pgprove"]) { + const pattern = new RegExp(`^FROM (\\S+):(\\S+) AS ${alias}$`, "m"); + const match = pattern.exec(tampered); + expect(match, `no line for hand-managed alias '${alias}'`).not.toBeNull(); + tampered = tampered.replace(pattern, `FROM $1:${marker} AS ${alias}`); + } + expect(tampered).not.toBe(currentTsDockerfile); + + const rendered = renderDockerfile(tampered); + for (const alias of ["kong", "differ", "migra", "pgprove"]) { + const tamperedLine = tampered.split("\n").find((line) => line.endsWith(`AS ${alias}`)); + expect(tamperedLine).toBeDefined(); + expect(tamperedLine).toContain(marker); + expect(rendered).toContain(tamperedLine); } - // pg14 has no slim build: its tag is whatever the checked-in file already pins, verbatim. - const pg14Line = currentTsDockerfile.split("\n").find((line) => line.endsWith("AS pg14")); - expect(pg14Line).toBeDefined(); - expect(rendered).toContain(pg14Line); }); }); diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index cf59bbd77c..debb4eae72 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -1577,7 +1577,10 @@ describe("db start stack backend", () => { const databaseCreation: Extract = { service: "database", config: { - version: "17.11.0.002", + // Derived from the real catalog's major-17 pin, not hardcoded: `dbStart`'s stack-backend + // path resolves the desired version through the same catalog, so a bump would otherwise + // make the "resumes the existing database" test below see a spurious version mismatch. + version: postgresVersion("17"), databasePassword: Redacted.make("secret"), jwtSecret: Redacted.make("secret"), jwtExpiry: 3600, From 4d1e98b013b4c551a5e1713b31d727846790e02e Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 17:13:09 +0200 Subject: [PATCH 22/45] feat(ci): open catalog bump and hotfix PRs from slim releases slim-release-published now reconciles a service against every committed slim-services release (planSlimUpdates) and opens or updates one PR per hotfix and/or upgrade it finds, instead of only matching the release named in the dispatch. Plan and apply run from the same checkout in a single job so a re-run always recomputes from develop. Manual mode gains --release to pin an exact committed revision. --- .github/scripts/slim-mirror-payload.ts | 2 +- .../scripts/sync-artifacts-catalog.test.ts | 573 +++++++++--------- .github/scripts/sync-artifacts-catalog.ts | 555 +++++++++-------- .github/workflows/slim-release-published.yml | 205 ++++--- .github/workflows/sync-artifacts-catalog.yml | 98 --- 5 files changed, 724 insertions(+), 709 deletions(-) delete mode 100644 .github/workflows/sync-artifacts-catalog.yml diff --git a/.github/scripts/slim-mirror-payload.ts b/.github/scripts/slim-mirror-payload.ts index 74191dfebf..5b61fff58e 100644 --- a/.github/scripts/slim-mirror-payload.ts +++ b/.github/scripts/slim-mirror-payload.ts @@ -4,7 +4,7 @@ */ const SERVICE_PATTERN = /^[a-z][a-z0-9-]*$/; -export const VERSION_PATTERN = /^[A-Za-z0-9._-]+$/; +const VERSION_PATTERN = /^[A-Za-z0-9._-]+$/; export const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; export const SOURCE_REGISTRY = "ghcr.io/supabase/cli"; diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index 8d378247af..dd732a71ca 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -3,13 +3,10 @@ import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { parseDockerfileServiceImages } from "../../apps/cli/src/shared/services/parse-dockerfile-service-images.ts"; -import { isOrioleImage, slimCatalogPin } from "../../apps/cli/src/shared/services/slim-images.ts"; import { InvalidPayloadError, nativeFileNames, nativeObjectUrl } from "./slim-mirror-payload.ts"; import { CATALOG_PATH, - findHotfixMatches, - planArtifactCatalogUpdate, + planSlimUpdates, refreshCatalogPin, validateSlimReleasePublishedPayload, type RevisionIo, @@ -52,7 +49,7 @@ function matchingS3( } /** - * Runs the repo's pinned `oxfmt` binary over `source`, the way `sync-artifacts-catalog.yml` + * Runs the repo's pinned `oxfmt` binary over `source`, the way `slim-release-published.yml` * formats the catalog after every write, so parsing tests exercise real formatter output * (line-wrapping, trailing commas) instead of a hand-written single-line literal. */ @@ -91,85 +88,64 @@ const fixture = `const workloadCatalog = { }; `; -/** A catalog whose Postgres 15 additional pin is already resolved (not a `placeholderPin`), at `r3`. */ -const resolvedFixture = fixture.replace( - '{ "15.14.1.168": placeholderPin("postgres", "15.14.1.168") },', - '{ "15.14.1.168": { upstreamVersion: "15.14.1.168", revision: 3, image: "ghcr.io/supabase/cli/postgres:15.14.1.168-r3@sha256:0000000000000000000000000000000000000000000000000000000000000", natives: {} } },', -); - -describe("findHotfixMatches", () => { - test("selects the Postgres 15 entry for a higher revision of the same upstream", () => { - const matches = findHotfixMatches(resolvedFixture, { - service: "postgres", - upstream_version: "15.14.1.168", - revision: 4, - release_version: "15.14.1.168-r4", - }); - - expect(matches).toEqual([{ target: "additional", currentRevision: 3 }]); - }); - - test("ignores an equal or lower revision of the same upstream", () => { - for (const revision of [3, 2]) { - const matches = findHotfixMatches(resolvedFixture, { - service: "postgres", - upstream_version: "15.14.1.168", - revision, - release_version: `15.14.1.168-r${revision}`, - }); - expect(matches).toEqual([]); - } - }); - - test("ignores a higher revision of a different upstream version", () => { - const matches = findHotfixMatches(resolvedFixture, { - service: "postgres", - upstream_version: "15.14.1.999", - revision: 4, - release_version: "15.14.1.999-r4", - }); - - expect(matches).toEqual([]); - }); - - test("matches the default pin too, once it is resolved", () => { - const withResolvedDefault = resolvedFixture.replace( - 'placeholderPin("postgres", "17.6.1.168")', - '{ upstreamVersion: "17.6.1.168", revision: 1, image: "ghcr.io/supabase/cli/postgres:17.6.1.168-r1@sha256:1111111111111111111111111111111111111111111111111111111111111", natives: {} }', - ); - const matches = findHotfixMatches(withResolvedDefault, { - service: "postgres", - upstream_version: "17.6.1.168", - revision: 2, - release_version: "17.6.1.168-r2", - }); - - expect(matches).toEqual([{ target: "default", currentRevision: 1 }]); - }); +const vectorFixture = `const workloadCatalog = { + vector: definition("vector", placeholderPin("vector", "0.53.0"), "bin/vector"), +}; +`; - test("treats an unresolved placeholderPin as always eligible for a hotfix", () => { - const matches = findHotfixMatches(fixture, { - service: "postgrest", - upstream_version: "v16.2", +/** + * Fully resolved (not `placeholderPin`) fixtures for `planSlimUpdates`: a committed catalog + * always carries resolved pins (see the design doc's one-time alignment), so these give every + * pin a real revision instead of the `-1` an unresolved placeholder would carry. + */ +const plannerFixture = `const workloadCatalog = { + rest: definition( + "postgrest", + { + upstreamVersion: "v16.2", revision: 0, - release_version: "v16.2-r0", - }); - - expect(matches).toEqual([{ target: "default", currentRevision: -1 }]); - }); - - test("survives real oxfmt formatting", async () => { - const formatted = await formatWithOxfmt(resolvedFixture); - const matches = findHotfixMatches(formatted, { - service: "postgres", - upstream_version: "15.14.1.168", - revision: 4, - release_version: "15.14.1.168-r4", - }); + image: "ghcr.io/supabase/cli/postgrest:v16.2-r0@sha256:1111111111111111111111111111111111111111111111111111111111111", + natives: {}, + }, + "bin/postgrest", + ), + storage: definition( + "storage", + { + upstreamVersion: "v1.73.0", + revision: 0, + image: "ghcr.io/supabase/cli/storage:v1.73.0-r0@sha256:2222222222222222222222222222222222222222222222222222222222222", + natives: {}, + }, + "bin/storage", + ["bin/storage"], + ), +}; +`; - expect(matches).toEqual([{ target: "additional", currentRevision: 3 }]); - }); -}); +/** Postgres carries two lines (17 default, 15 additional), both resolved. */ +const postgresPlannerFixture = `const workloadCatalog = { + database: definition( + "postgres", + { + upstreamVersion: "17.6.1.168", + revision: 1, + image: "ghcr.io/supabase/cli/postgres:17.6.1.168-r1@sha256:3333333333333333333333333333333333333333333333333333333333333", + natives: {}, + }, + "bin/supabase-postgres-start", + ["bin/supabase-postgres-start"], + { + "15.14.1.168": { + upstreamVersion: "15.14.1.168", + revision: 3, + image: "ghcr.io/supabase/cli/postgres:15.14.1.168-r3@sha256:4444444444444444444444444444444444444444444444444444444444444", + natives: {}, + }, + }, + ), +}; +`; describe("validateSlimReleasePublishedPayload", () => { test("rejects a newline injected into upstream_version", () => { @@ -227,183 +203,212 @@ describe("validateSlimReleasePublishedPayload", () => { }); }); -describe("planArtifactCatalogUpdate", () => { - test("pins to the highest committed revision, writing every native target's digests", async () => { - const digests = nativeDigests("c"); - const io: RevisionIo = { - listReleaseTags: async () => [ - "postgrest-v16.3-r0", - "postgrest-v16.3-r1", - "postgrest-v16.3-r2", - "unrelated-tag", - ], - fetchChecksums: async (service, releaseVersion) => - service === "postgrest" && releaseVersion === "v16.3-r2" - ? checksumsFor("postgrest", "v16.3-r2", digests) - : undefined, - imageDigest: async () => digest("d"), - s3Sha256: matchingS3("postgrest", "v16.3-r2", digests), - }; - - const plan = await planArtifactCatalogUpdate({ - baseDockerfile: "FROM postgrest/postgrest:v16.2 AS postgrest\n", - dockerfile: "FROM postgrest/postgrest:v16.3 AS postgrest\n", - catalog: fixture, - io, - }); +describe("planSlimUpdates", () => { + test("hotfix only: a higher committed revision of the pinned upstream", () => { + const updates = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.2-r1"]); - expect(plan.skipped).toEqual([]); - expect(plan.updates).toEqual([ + expect(updates).toEqual([ { - service: "postgrest", - version: "v16.3", - revision: 2, - previousVersion: "v16.2", - target: "default", + kind: "hotfix", + line: undefined, + branch: "slim-hotfix/postgrest", + title: "chore(stack): pin postgrest v16.2-r1", + fromRelease: "v16.2-r0", + toUpstream: "v16.2", + toRelease: "v16.2-r1", }, ]); - expect(plan.source).toContain('upstreamVersion: "v16.3"'); - expect(plan.source).toContain("revision: 2"); - expect(plan.source).toContain( - `image: "ghcr.io/supabase/cli/postgrest:v16.3-r2@${digest("d")}"`, - ); - for (const target of NATIVE_TARGETS) { - expect(plan.source).toContain(`"${target}": { archive: "${digests[target].archive}"`); - expect(plan.source).toContain(`manifest: "${digests[target].manifest}" }`); - } - // The other postgres line is untouched. - expect(plan.source).toContain('placeholderPin("postgres", "15.14.1.168")'); }); - test("a legacy release with no -rN tag is a blocking missing pin", async () => { - const io: RevisionIo = { - listReleaseTags: async () => ["postgrest-v16.3"], - fetchChecksums: async () => undefined, - imageDigest: async () => undefined, - s3Sha256: async () => undefined, - }; + test("upgrade only: a newer committed upstream on the same line", () => { + const updates = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.4-r0"]); - const plan = await planArtifactCatalogUpdate({ - baseDockerfile: "FROM postgrest/postgrest:v16.2 AS postgrest\n", - dockerfile: "FROM postgrest/postgrest:v16.3 AS postgrest\n", - catalog: fixture, - io, - }); + expect(updates).toEqual([ + { + kind: "upgrade", + line: undefined, + branch: "slim-bump/postgrest", + title: "chore(stack): bump postgrest to v16.4-r0", + fromRelease: "v16.2-r0", + toUpstream: "v16.4", + toRelease: "v16.4-r0", + }, + ]); + }); - expect(plan.updates).toEqual([]); - expect(plan.source).toBe(fixture); - expect(plan.skipped).toEqual([ + test("both a hotfix and an upgrade can be planned in the same run", () => { + const updates = planSlimUpdates(plannerFixture, "postgrest", [ + "postgrest-v16.2-r1", + "postgrest-v16.4-r0", + ]); + + expect(updates).toEqual([ + { + kind: "hotfix", + line: undefined, + branch: "slim-hotfix/postgrest", + title: "chore(stack): pin postgrest v16.2-r1", + fromRelease: "v16.2-r0", + toUpstream: "v16.2", + toRelease: "v16.2-r1", + }, { - alias: "postgrest", - reason: "postgrest:v16.3 has no published slim-services revision.", - blocking: true, + kind: "upgrade", + line: undefined, + branch: "slim-bump/postgrest", + title: "chore(stack): bump postgrest to v16.4-r0", + fromRelease: "v16.2-r0", + toUpstream: "v16.4", + toRelease: "v16.4-r0", }, ]); }); - test("a longer upstream version's release tag does not satisfy a prefix collision", async () => { - const io: RevisionIo = { - // A decoy tag for a different (longer) upstream version must not count as v1.79.2's. - listReleaseTags: async () => ["storage-v1.79.23-r0"], - fetchChecksums: async () => undefined, - imageDigest: async () => undefined, - s3Sha256: async () => undefined, - }; + test("an older committed upstream is a backlog republish: it plans nothing", () => { + const updates = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.1-r0"]); - const plan = await planArtifactCatalogUpdate({ - baseDockerfile: "FROM supabase/storage-api:v1.73.0 AS storage\n", - dockerfile: "FROM supabase/storage-api:v1.79.2 AS storage\n", - catalog: fixture, - io, - }); + expect(updates).toEqual([]); + }); - expect(plan.updates).toEqual([]); - expect(plan.skipped).toEqual([ + test("postgres: upgrade on the 17 line, hotfix on the 15 line, a major-18 release ignored", () => { + const updates = planSlimUpdates(postgresPlannerFixture, "postgres", [ + "postgres-17.11.0.002-r0", + "postgres-15.14.1.168-r4", + "postgres-18.0.0.001-r0", + ]); + + expect(updates).toEqual([ + { + kind: "upgrade", + line: "17", + branch: "slim-bump/postgres-17", + title: "chore(stack): bump postgres to 17.11.0.002-r0", + fromRelease: "17.6.1.168-r1", + toUpstream: "17.11.0.002", + toRelease: "17.11.0.002-r0", + }, { - alias: "storage", - reason: "storage:v1.79.2 has no published slim-services revision.", - blocking: true, + kind: "hotfix", + line: "15", + branch: "slim-hotfix/postgres-15", + title: "chore(stack): pin postgres 15.14.1.168-r4", + fromRelease: "15.14.1.168-r3", + toUpstream: "15.14.1.168", + toRelease: "15.14.1.168-r4", }, ]); }); - test("a stale S3 copy is a blocking error", async () => { - const digests = nativeDigests("e"); - const io: RevisionIo = { - listReleaseTags: async () => ["postgrest-v16.3-r0"], - fetchChecksums: async () => checksumsFor("postgrest", "v16.3-r0", digests), - imageDigest: async () => digest("f"), - s3Sha256: async (url) => { - const files = nativeFileNames("postgrest", "v16.3-r0", "darwin-arm64"); - if (url === nativeObjectUrl("postgrest", "v16.3-r0", files.archive)) return hex("stale"); - return matchingS3("postgrest", "v16.3-r0", digests)(url); - }, - }; + test("Studio: a newer date upgrades; the same date with a different sha does not", () => { + const studioFixture = `const workloadCatalog = { + studio: definition( + "studio", + { + upstreamVersion: "2026.09.14-sha-aaaaaaa", + revision: 0, + image: "ghcr.io/supabase/cli/studio:2026.09.14-sha-aaaaaaa-r0@sha256:1111111111111111111111111111111111111111111111111111111111111", + natives: {}, + }, + "bin/studio", + ), +}; +`; - const plan = await planArtifactCatalogUpdate({ - baseDockerfile: "FROM postgrest/postgrest:v16.2 AS postgrest\n", - dockerfile: "FROM postgrest/postgrest:v16.3 AS postgrest\n", - catalog: fixture, - io, - }); + const sameDate = planSlimUpdates(studioFixture, "studio", ["studio-2026.09.14-sha-bbbbbbb-r0"]); + expect(sameDate).toEqual([]); - expect(plan.updates).toEqual([]); - expect(plan.source).toBe(fixture); - expect(plan.skipped).toEqual([ + const newerDate = planSlimUpdates(studioFixture, "studio", [ + "studio-2026.09.28-sha-ccccccc-r0", + ]); + expect(newerDate).toEqual([ { - alias: "postgrest", - reason: "S3 copy of v16.3-r0 darwin-arm64 is stale; run the slim-services mirror backfill", - blocking: true, + kind: "upgrade", + line: undefined, + branch: "slim-bump/studio", + title: "chore(stack): bump studio to 2026.09.28-sha-ccccccc-r0", + fromRelease: "2026.09.14-sha-aaaaaaa-r0", + toUpstream: "2026.09.28-sha-ccccccc", + toRelease: "2026.09.28-sha-ccccccc-r0", }, ]); }); - test("a Dependabot bump older than the catalog pin is a non-blocking, unchanged skip", async () => { - const ahead = fixture.replace( - 'placeholderPin("postgrest", "v16.2")', - 'placeholderPin("postgrest", "v16.5")', - ); - const io: RevisionIo = { - listReleaseTags: async () => { - throw new Error("should not be reached for an older bump"); + test("a non-comparable version (OrioleDB-style suffix) is ignored", () => { + const updates = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.2-orioledb-r0"]); + + expect(updates).toEqual([]); + }); + + test("a legacy tag with no -rN is ignored", () => { + const updates = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.4"]); + + expect(updates).toEqual([]); + }); + + test("postgrest tags never match postgres, even as a prefix", () => { + const updates = planSlimUpdates(postgresPlannerFixture, "postgres", [ + "postgrest-v16.4-r0", + "postgres-17.11.0.002-r0", + ]); + + expect(updates).toEqual([ + { + kind: "upgrade", + line: "17", + branch: "slim-bump/postgres-17", + title: "chore(stack): bump postgres to 17.11.0.002-r0", + fromRelease: "17.6.1.168-r1", + toUpstream: "17.11.0.002", + toRelease: "17.11.0.002-r0", }, - fetchChecksums: async () => undefined, - imageDigest: async () => undefined, - s3Sha256: async () => undefined, - }; + ]); + }); - const plan = await planArtifactCatalogUpdate({ - baseDockerfile: "FROM postgrest/postgrest:v16.2 AS postgrest\n", - dockerfile: "FROM postgrest/postgrest:v16.3 AS postgrest\n", - catalog: ahead, - io, - }); + test("branch and title carry no - suffix for a service with a single line", () => { + const updates = planSlimUpdates(plannerFixture, "storage", ["storage-v1.74.0-r0"]); - expect(plan.updates).toEqual([]); - expect(plan.source).toBe(ahead); - expect(plan.skipped).toEqual([ + expect(updates).toEqual([ { - alias: "postgrest", - reason: "postgrest v16.3 is older than the catalog pin v16.5.", - blocking: false, + kind: "upgrade", + line: undefined, + branch: "slim-bump/storage", + title: "chore(stack): bump storage to v1.74.0-r0", + fromRelease: "v1.73.0-r0", + toUpstream: "v1.74.0", + toRelease: "v1.74.0-r0", }, ]); - expect(plan.skipped.some((skip) => skip.blocking)).toBe(false); }); - test("refuses a tag that would escape the catalog string", async () => { - await expect( - planArtifactCatalogUpdate({ - dockerfile: 'FROM supabase/gotrue:v1" AS gotrue\n', - catalog: fixture, - io: { - listReleaseTags: async () => [], - fetchChecksums: async () => undefined, - imageDigest: async () => undefined, - s3Sha256: async () => undefined, - }, - }), - ).rejects.toThrow(InvalidPayloadError); + test("rejects an emitted value that fails the anchored patterns, even from a trusted-looking catalog", () => { + // The catalog's own pinned upstream carries a space here — never written by `refreshCatalogPin` + // (which validates every field it resolves), but this simulates a corrupted catalog, or a + // release tag whose upstream portion isn't newline-only-unsafe (`.` already excludes + // newlines) yet still fails `UPSTREAM_VERSION_PATTERN`. Ruling 6 requires every value the + // planner emits to be checked, regardless of source. + const adversarial = `const workloadCatalog = { + rest: definition( + "postgrest", + { + upstreamVersion: "v16.2 injected", + revision: 0, + image: "ghcr.io/supabase/cli/postgrest:v16.2-r0@sha256:2222222222222222222222222222222222222222222222222222222222222", + natives: {}, + }, + "bin/postgrest", + ), +}; +`; + + expect(() => + planSlimUpdates(adversarial, "postgrest", ["postgrest-v16.2 injected-r1"]), + ).toThrow(InvalidPayloadError); + }); + + test("plans nothing for a service the catalog does not model", () => { + const updates = planSlimUpdates(plannerFixture, "no-such-service", ["no-such-service-v1.0-r0"]); + + expect(updates).toEqual([]); }); }); @@ -583,15 +588,64 @@ describe("refreshCatalogPin", () => { }); }); -const vectorFixture = `const workloadCatalog = { - vector: definition("vector", placeholderPin("vector", "0.53.0"), "bin/vector"), -}; -`; +describe("refreshCatalogPin --release", () => { + test("pins exactly the requested committed release, not the highest one", async () => { + const digests = nativeDigests("release-0"); + const io: RevisionIo = { + listReleaseTags: async () => ["postgrest-v16.2-r0", "postgrest-v16.2-r1"], + fetchChecksums: async () => checksumsFor("postgrest", "v16.2-r0", digests), + imageDigest: async () => digest("release-digest-0"), + s3Sha256: matchingS3("postgrest", "v16.2-r0", digests), + }; -/** Manifest fixture for a derived (nix-built) service, agreeing across every native target. */ -function manifestFixture(upstreamImage: string): RevisionIo["fetchManifest"] { - return async () => JSON.stringify({ upstream_image: upstreamImage }); -} + const result = await refreshCatalogPin({ + catalog: fixture, + service: "postgrest", + release: "v16.2-r0", + io, + }); + + expect(result.update).toEqual({ + service: "postgrest", + version: "v16.2", + revision: 0, + previousVersion: "v16.2", + target: "default", + }); + }); + + test("fails when the requested release is not committed", async () => { + const io: RevisionIo = { + listReleaseTags: async () => ["postgrest-v16.2-r0"], + fetchChecksums: async () => undefined, + imageDigest: async () => undefined, + s3Sha256: async () => undefined, + }; + + await expect( + refreshCatalogPin({ catalog: fixture, service: "postgrest", release: "v16.2-r1", io }), + ).rejects.toThrow(InvalidPayloadError); + }); + + test("rejects --upstream and --release given together", async () => { + const io: RevisionIo = { + listReleaseTags: async () => [], + fetchChecksums: async () => undefined, + imageDigest: async () => undefined, + s3Sha256: async () => undefined, + }; + + await expect( + refreshCatalogPin({ + catalog: fixture, + service: "postgrest", + upstream: "v16.2", + release: "v16.2-r0", + io, + }), + ).rejects.toThrow(InvalidPayloadError); + }); +}); describe("refreshCatalogPin backfills upstreamImage", () => { test("resolves a derived service's upstreamImage from its per-target manifests", async () => { @@ -601,7 +655,7 @@ describe("refreshCatalogPin backfills upstreamImage", () => { fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r1", digests), imageDigest: async () => digest("z"), s3Sha256: matchingS3("analytics", "v1.50.9-r1", digests), - fetchManifest: manifestFixture("supabase/logflare:1.50.9"), + fetchManifest: async () => JSON.stringify({ upstream_image: "supabase/logflare:1.50.9" }), }; const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); @@ -631,7 +685,8 @@ describe("refreshCatalogPin backfills upstreamImage", () => { fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r0", digests), imageDigest: async () => digest("ad"), s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), - fetchManifest: manifestFixture("docker.io/supabase/logflare:1.50.9@sha256:deadbeef"), + fetchManifest: async () => + JSON.stringify({ upstream_image: "docker.io/supabase/logflare:1.50.9@sha256:deadbeef" }), }; const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); @@ -695,7 +750,8 @@ describe("refreshCatalogPin backfills upstreamImage", () => { fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r0", digests), imageDigest: async () => digest("al"), s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), - fetchManifest: manifestFixture('supabase/logflare:1.50.9"] }; import("evil"); //'), + fetchManifest: async () => + JSON.stringify({ upstream_image: 'supabase/logflare:1.50.9"] }; import("evil"); //' }), }; await expect(refreshCatalogPin({ catalog: fixture, service: "analytics", io })).rejects.toThrow( @@ -705,55 +761,6 @@ describe("refreshCatalogPin backfills upstreamImage", () => { }); describe("against the real catalog", () => { - test("every slim Dockerfile alias is a catalog entry", async () => { - const dockerfile = await Bun.file("apps/cli/src/shared/services/Dockerfile").text(); - const catalog = await Bun.file(CATALOG_PATH).text(); - - const pins = parseDockerfileServiceImages(dockerfile) - .filter((from) => !isOrioleImage(from.image)) - .map((from) => slimCatalogPin(from.alias, from.image)) - .filter((pin): pin is NonNullable => pin !== undefined); - - const resolved = pins.map((pin) => { - const releaseVersion = `${pin.version}-r0`; - const digests = nativeDigests(pin.service); - return { - service: pin.service, - releaseVersion, - checksums: checksumsFor(pin.service, releaseVersion, digests), - digests, - }; - }); - - const io: RevisionIo = { - listReleaseTags: async () => - resolved.map((entry) => `${entry.service}-${entry.releaseVersion}`), - fetchChecksums: async (service, releaseVersion) => - resolved.find( - (entry) => entry.service === service && entry.releaseVersion === releaseVersion, - )?.checksums, - imageDigest: async () => digest("real"), - s3Sha256: async (url) => { - for (const entry of resolved) { - for (const target of NATIVE_TARGETS) { - const files = nativeFileNames(entry.service, entry.releaseVersion, target); - if (url === nativeObjectUrl(entry.service, entry.releaseVersion, files.archive)) { - return entry.digests[target].archive; - } - if (url === nativeObjectUrl(entry.service, entry.releaseVersion, files.manifest)) { - return entry.digests[target].manifest; - } - } - } - return undefined; - }, - }; - - const plan = await planArtifactCatalogUpdate({ dockerfile, catalog, io }); - - expect(plan.skipped.filter((skip) => skip.blocking)).toEqual([]); - }); - test("a real catalog entry survives real formatting and can be refreshed again", async () => { const catalog = await Bun.file(CATALOG_PATH).text(); const pinnedVersion = /definition\(\s*"auth",\s*\{\s*upstreamVersion:\s*"([^"]+)"/.exec( diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index c2a96497ea..d0a94b0237 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -3,24 +3,25 @@ * revisions (`-r`). Every entry is pinned by content: the GHCR * image digest, plus an archive and manifest sha256 per native target. * - * Dependabot mode (used by CI): rewrites every catalog entry whose slim - * service moved between `dockerfile` and `base-dockerfile`, to the highest - * committed revision of its new upstream version. + * The catalog is the single version table (supabase/cli#6883): the Dockerfile's + * slim-capable lines are a generated view of it + * (`apps/cli/scripts/render-service-dockerfile.ts`), never the other way + * around. Updates land through two modes: * - * bun .github/scripts/sync-artifacts-catalog.ts [base-dockerfile] + * Manual mode: refreshes one catalog entry, either to a specific committed + * release (`--release`) or to the highest committed revision of a given + * upstream version, or of its currently pinned upstream version when neither + * is given. `--upstream` and `--release` are mutually exclusive. * - * Manual mode: refreshes one catalog entry to the highest committed revision - * of a given upstream version (or of its currently pinned upstream version, - * when `--upstream` is omitted). + * bun .github/scripts/sync-artifacts-catalog.ts --service [--upstream | --release -r] * - * bun .github/scripts/sync-artifacts-catalog.ts --service [--upstream ] + * Plan-updates mode (used by the `slim-release-published` dispatch workflow): + * lists a service's committed slim-services releases and computes, per + * release line, the hotfix and/or upgrade a workflow should apply. Pure + * planning: `planSlimUpdates` takes the release tag list as an argument and + * makes no network calls. * - * Hotfix-matches mode (used by the `slim-release-published` dispatch - * workflow): prints, as JSON, which of a service's catalog pins a hotfix - * revision should refresh. - * - * bun .github/scripts/sync-artifacts-catalog.ts hotfix-matches --service \ - * --upstream --revision + * bun .github/scripts/sync-artifacts-catalog.ts plan-updates --service [--format lines] * * Validate-payload mode (used by the same workflow, before anything else): * checks an untrusted `slim-release-published` dispatch payload against @@ -31,13 +32,10 @@ * --upstream --revision --release */ -import { parseDockerfileServiceImages } from "../../apps/cli/src/shared/services/parse-dockerfile-service-images.ts"; -import { isOrioleImage, slimCatalogPin } from "../../apps/cli/src/shared/services/slim-images.ts"; import { DIGEST_PATTERN, InvalidPayloadError, SOURCE_REGISTRY, - VERSION_PATTERN, checksumFor, escapeRegExp, nativeFileNames, @@ -45,7 +43,6 @@ import { } from "./slim-mirror-payload.ts"; export const CATALOG_PATH = "packages/stack/src/Artifacts.ts"; -const DOCKERFILE_PATH = "apps/cli/src/shared/services/Dockerfile"; const SLIM_IMAGE_PREFIX = `${SOURCE_REGISTRY}/`; @@ -64,25 +61,44 @@ function releaseLine(version: string): string { } /** - * True when every numeric prefix of `next` is older than `current`. Equal prefixes are not - * older. Compares upstream versions only; revisions never enter this comparison. + * Matches a `--r` release tag. With `upstream` given, matches only that + * exact upstream (what `resolveRevisionPin` needs); with it omitted, captures any upstream as + * group 1 and the revision as group 2 (what the planner needs to enumerate every committed + * revision of every upstream a service carries). A tag with no `-r` suffix — legacy — never + * matches either shape. */ -function isOlderRelease(next: string, current: string): boolean { - const nextParts = next.replace(/^[vV]/, "").split("."); - const currentParts = current.replace(/^[vV]/, "").split("."); - const count = Math.max(nextParts.length, currentParts.length); - for (let index = 0; index < count; index++) { - const nextValue = leadingInteger(nextParts[index] ?? "0"); - const currentValue = leadingInteger(currentParts[index] ?? "0"); - if (nextValue === undefined || currentValue === undefined) return false; - if (nextValue !== currentValue) return nextValue < currentValue; - } - return false; +function releaseTagPattern(service: string, upstream?: string): RegExp { + const upstreamPart = upstream === undefined ? "(.+)" : escapeRegExp(upstream); + return new RegExp(`^${escapeRegExp(service)}-${upstreamPart}-r(0|[1-9][0-9]*)$`); } -function leadingInteger(part: string): number | undefined { - const match = /^(\d+)/.exec(part); - return match === null ? undefined : Number(match[1]); +/** + * Strips a leading `v`/`V` and one trailing `-sha-`, then parses the remainder as dot- + * separated integers. Returns undefined when the remainder isn't `^\d+(\.\d+)*$` — a version that + * isn't comparable this way, such as an OrioleDB-style suffix. + */ +function comparableVersion(version: string): ReadonlyArray | undefined { + const stripped = version.replace(/^[vV]/, "").replace(/-sha-[0-9a-f]+$/i, ""); + if (!/^\d+(\.\d+)*$/.test(stripped)) return undefined; + return stripped.split(".").map(Number); +} + +/** + * Numeric ordering of two upstream versions: `-1` when `a` is older, `0` when neither is newer + * (including two versions that only differ in a stripped `-sha-` suffix, e.g. two Studio + * builds on the same date), `1` when `a` is newer. Undefined when either isn't comparable. + */ +function compareVersions(a: string, b: string): number | undefined { + const av = comparableVersion(a); + const bv = comparableVersion(b); + if (av === undefined || bv === undefined) return undefined; + const length = Math.max(av.length, bv.length); + for (let index = 0; index < length; index++) { + const left = av[index] ?? 0; + const right = bv[index] ?? 0; + if (left !== right) return left < right ? -1 : 1; + } + return 0; } export interface CatalogPinUpdate { @@ -93,28 +109,14 @@ export interface CatalogPinUpdate { readonly target: "default" | "additional"; } -export interface SkippedCatalogPin { - readonly alias: string; - readonly reason: string; - /** OrioleDB and an older Dependabot bump are the non-blocking skips. */ - readonly blocking: boolean; -} - -export interface CatalogPlan { - readonly source: string; - readonly updates: ReadonlyArray; - readonly skipped: ReadonlyArray; -} - /** Content pin for one resolved `-r` revision, ready to serialize into the catalog. */ interface ResolvedPin { readonly upstreamVersion: string; readonly revision: number; readonly image: string; /** - * The pin's `ArtifactPin.upstreamImage`. Left undefined on the Dependabot (`planArtifactCatalogUpdate`) - * path, which has no use for it and whose `io` doesn't carry `fetchManifest`/`fetchProvenance`; - * populated by manual mode (`refreshCatalogPin`) via `resolveUpstreamImage` below. + * The pin's `ArtifactPin.upstreamImage`. Populated by manual mode (`refreshCatalogPin`) via + * `resolveUpstreamImage` below, when `io` carries `fetchManifest`/`fetchProvenance`. */ readonly upstreamImage?: string; readonly natives: Readonly< @@ -135,6 +137,7 @@ export type RevisionResolution = * directly, the same way the previous `publication` callback was stubbed. */ export interface RevisionIo { + /** Tags of every published, non-draft release — a draft is not yet a committed revision. */ readonly listReleaseTags: () => Promise>; readonly fetchChecksums: (service: string, releaseVersion: string) => Promise; readonly imageDigest: (service: string, releaseVersion: string) => Promise; @@ -170,33 +173,51 @@ function desiredImage(service: string, releaseVersion: string, digest: string): } /** - * Resolves `service`'s highest committed `-r` revision, pinned by content: the - * GHCR manifest digest, and every native target's archive and manifest sha256, cross-checked - * against the S3 mirror copy. See module docs for the five-step protocol. + * Resolves `service`'s committed `-r` revision, pinned by content: the GHCR + * manifest digest, and every native target's archive and manifest sha256, cross-checked against + * the S3 mirror copy. See module docs for the five-step protocol. + * + * With `requiredRevision` given, that exact revision must already be committed — this is what + * pins exactly a planned release (`--release`), never "highest at apply time". Without it, the + * highest committed revision of `upstream` is used (`--upstream`, and the hotfix/upgrade default). */ export async function resolveRevisionPin( service: string, upstream: string, io: RevisionIo, + requiredRevision?: number, ): Promise { - const tagPattern = new RegExp( - `^${escapeRegExp(service)}-${escapeRegExp(upstream)}-r(0|[1-9][0-9]*)$`, - ); + const tagPattern = releaseTagPattern(service, upstream); + const seenRevisions = new Set(); let highest: number | undefined; for (const tag of await io.listReleaseTags()) { const match = tagPattern.exec(tag); if (match === null) continue; const revision = Number(match[1]); + seenRevisions.add(revision); if (highest === undefined || revision > highest) highest = revision; } - if (highest === undefined) { - return { - status: "missing", - message: `${service}:${upstream} has no published slim-services revision.`, - }; + + let target: number; + if (requiredRevision !== undefined) { + if (!seenRevisions.has(requiredRevision)) { + return { + status: "missing", + message: `${service}:${upstream}-r${requiredRevision} is not a committed slim-services release.`, + }; + } + target = requiredRevision; + } else { + if (highest === undefined) { + return { + status: "missing", + message: `${service}:${upstream} has no published slim-services revision.`, + }; + } + target = highest; } - const releaseVersion = `${upstream}-r${highest}`; + const releaseVersion = `${upstream}-r${target}`; const checksums = await io.fetchChecksums(service, releaseVersion); if (checksums === undefined) { return { @@ -247,7 +268,7 @@ export async function resolveRevisionPin( status: "resolved", pin: { upstreamVersion: upstream, - revision: highest, + revision: target, image: desiredImage(service, releaseVersion, digest), natives: natives as Record, }, @@ -459,7 +480,7 @@ interface ServicePins { /** * Finds every pin expression `service` carries in `source`: the `definition("", , * ...)` default pin, plus any additional (release-line-keyed) pins in its trailing object - * argument. Both `selectEntry` and `findHotfixMatches` build on this single traversal. + * argument. Both `selectEntry` and `planSlimUpdates` build on this single traversal. */ function collectServicePins(source: string, service: string): ServicePins | undefined { const prefix = new RegExp(`definition\\(\\s*"${escapeRegExp(service)}"\\s*,\\s*`); @@ -538,49 +559,6 @@ export interface SlimReleasePublishedPayload { readonly release_version: string; } -export interface HotfixMatch { - /** Which of `service`'s catalog pins (default, or one of the additional release lines) matched. */ - readonly target: "default" | "additional"; - /** The revision currently pinned; `-1` when the entry is still an unresolved `placeholderPin`. */ - readonly currentRevision: number; -} - -/** - * Every pin `payload.service` carries in `catalog` (default or additional) whose - * `upstreamVersion` equals `payload.upstream_version` and whose currently pinned revision is - * lower than `payload.revision` — i.e. the entries a `slim-release-published` hotfix dispatch - * should refresh. Pure: callers own running the actual refresh and PR flow. - */ -export function findHotfixMatches( - catalog: string, - payload: SlimReleasePublishedPayload, -): ReadonlyArray { - const pins = collectServicePins(catalog, payload.service); - if (pins === undefined) return []; - - const candidates: Array<{ target: "default" | "additional"; span: PinSpan }> = [ - { target: "default", span: pins.defaultPin }, - ...pins.additional.map((span) => ({ target: "additional" as const, span })), - ]; - - // A service's default pin and every additional pin live on distinct release lines - // (`selectEntry`'s `releaseLine` split), so at most one candidate's `upstreamVersion` can equal - // `payload.upstream_version`: this can return more than one entry only if the catalog itself - // duplicates an upstream version across pins, which `selectEntry` would already treat as - // ambiguous. - const matches: HotfixMatch[] = []; - for (const candidate of candidates) { - if (candidate.span.version !== payload.upstream_version) continue; - const text = catalog.slice(candidate.span.start, candidate.span.end); - const revisionMatch = PIN_REVISION.exec(text); - const currentRevision = revisionMatch === null ? -1 : Number(revisionMatch[1]); - if (currentRevision < payload.revision) { - matches.push({ target: candidate.target, currentRevision }); - } - } - return matches; -} - /** Every upstream tag format the catalog carries, including Studio's `2026.09.04-sha-5a67366`. */ const UPSTREAM_VERSION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; const SERVICE_NAME_PATTERN = /^[a-z0-9-]+$/; @@ -626,25 +604,6 @@ export function validateSlimReleasePublishedPayload(input: { }; } -function skipReason(alias: string, version: string, entry: SelectedEntry): string | undefined { - if (entry.kind === "unmodelled-service") { - return `${CATALOG_PATH} has no slim entry for ${alias}.`; - } - if (entry.kind === "unmodelled-release-line") { - return `${alias} ${version} is not on a release line ${CATALOG_PATH} carries (${entry.known.join(", ")}).`; - } - return undefined; -} - -function slimVersions(dockerfile: string): ReadonlyMap { - const versions = new Map(); - for (const from of parseDockerfileServiceImages(dockerfile)) { - const pin = slimCatalogPin(from.alias, from.image); - if (pin !== undefined) versions.set(from.alias, pin.version); - } - return versions; -} - const normalizeText = (text: string): string => text.replace(/\s+/g, " ").trim(); /** @@ -675,76 +634,168 @@ function writePin( } /** - * Rewrites `catalog` from Dockerfile tags that differ from `baseDockerfile`, pinning each to its - * upstream's highest committed slim-services revision. With no base, every slim tag is in scope. - * A pin that cannot be applied is reported in `skipped`. OrioleDB and a Dependabot bump older - * than the current catalog pin are the non-blocking skips; the catalog is allowed to lead the - * Dockerfile until Dependabot catches up. + * One hotfix or upgrade a `slim-release-published` run should apply, on one of `service`'s + * release lines. `line` is the leading numeric component (`releaseLine`), present only when the + * service carries more than one line (only postgres does today) — it's already folded into + * `branch`, so a caller never needs to consume it separately. */ -export async function planArtifactCatalogUpdate(input: { - readonly dockerfile: string; - readonly baseDockerfile?: string; - readonly catalog: string; - readonly io: RevisionIo; -}): Promise { - let source = input.catalog; - const updates: CatalogPinUpdate[] = []; - const skipped: SkippedCatalogPin[] = []; - const baseVersions = - input.baseDockerfile === undefined ? undefined : slimVersions(input.baseDockerfile); - const changed = (alias: string, version: string | undefined): boolean => - baseVersions === undefined || baseVersions.get(alias) !== version; - - for (const from of parseDockerfileServiceImages(input.dockerfile)) { - if (isOrioleImage(from.image)) { - if (!changed(from.alias, undefined)) continue; - skipped.push({ - alias: from.alias, - reason: `${from.alias} ${from.image} has no slim image.`, - blocking: false, - }); +export interface SlimUpdate { + readonly kind: "hotfix" | "upgrade"; + readonly line?: string; + readonly branch: string; + readonly title: string; + /** The release version pinned before this update, e.g. `v2.195.0-r0`. */ + readonly fromRelease: string; + readonly toUpstream: string; + /** The release version this update pins, e.g. `v2.195.0-r1`. */ + readonly toRelease: string; +} + +/** + * Builds one `SlimUpdate`, validating every value it emits — `service`, `toUpstream` and + * `toRelease` — against the same anchored patterns `validateSlimReleasePublishedPayload` uses, + * before any of them reaches a branch name or PR title. Release tag names (the ultimate source of + * `toUpstream`) come from an external API, so this check applies even to values derived from the + * catalog itself (`fromRelease`'s upstream), not only to values freshly parsed from a tag. + */ +function buildUpdate( + kind: "hotfix" | "upgrade", + service: string, + line: string, + hasLines: boolean, + pinned: { readonly upstream: string; readonly revision: number }, + toUpstream: string, + toRevision: number, +): SlimUpdate { + if (!SERVICE_NAME_PATTERN.test(service)) { + throw new InvalidPayloadError(`invalid service: '${service}'`); + } + if (!UPSTREAM_VERSION_PATTERN.test(toUpstream)) { + throw new InvalidPayloadError(`invalid upstream version: '${toUpstream}'`); + } + const toRelease = `${toUpstream}-r${toRevision}`; + if (!RELEASE_VERSION_PATTERN.test(toRelease)) { + throw new InvalidPayloadError(`invalid release version: '${toRelease}'`); + } + const fromRelease = + pinned.revision < 0 ? pinned.upstream : `${pinned.upstream}-r${pinned.revision}`; + const suffix = hasLines ? `-${line}` : ""; + const branch = + kind === "hotfix" ? `slim-hotfix/${service}${suffix}` : `slim-bump/${service}${suffix}`; + const title = + kind === "hotfix" + ? `chore(stack): pin ${service} ${toRelease}` + : `chore(stack): bump ${service} to ${toRelease}`; + + return { + kind, + line: hasLines ? line : undefined, + branch, + title, + fromRelease, + toUpstream, + toRelease, + }; +} + +/** + * The hotfix and/or upgrade a `slim-release-published` run should apply for `service`, computed + * against `catalog`'s current pins and `releaseTags` (every committed — published, non-draft — + * slim-services release tag; the caller filters drafts before calling this). Pure: no network, no + * file I/O. + * + * Per release line (ruling: the default pin's line, plus one per additional pin — only postgres + * has more than one): a **hotfix** fires when the pinned upstream has a committed revision higher + * than the pinned one; an **upgrade** fires when the newest committed upstream on the line is + * newer than the pinned one (ties, e.g. two Studio builds dated the same day, are not newer). + * Both can fire in the same run. A release tag whose upstream sits on no line the catalog carries + * for `service`, or whose version isn't comparable, is logged and ignored — it never causes a + * plan-updates run to fail. + */ +export function planSlimUpdates( + catalog: string, + service: string, + releaseTags: ReadonlyArray, +): ReadonlyArray { + const pins = collectServicePins(catalog, service); + if (pins === undefined) { + console.log(`::warning ::${CATALOG_PATH} has no slim entry for ${service}; nothing to plan.`); + return []; + } + + const allPins = [pins.defaultPin, ...pins.additional]; + const hasLines = pins.additional.length > 0; + + const pinnedByLine = new Map(); + for (const span of allPins) { + const line = releaseLine(span.version); + const text = catalog.slice(span.start, span.end); + const revisionMatch = PIN_REVISION.exec(text); + const revision = revisionMatch === null ? -1 : Number(revisionMatch[1]); + pinnedByLine.set(line, { upstream: span.version, revision }); + } + + const tagPattern = releaseTagPattern(service); + const candidatesByLine = new Map>(); + for (const tag of releaseTags) { + const match = tagPattern.exec(tag); + if (match === null) continue; // not this service, or a legacy tag with no `-rN`: ignored. + const upstream = match[1] as string; + const revision = Number(match[2]); + const line = releaseLine(upstream); + if (!pinnedByLine.has(line)) { + console.log( + `::warning ::${service} ${upstream} is not on a release line ${CATALOG_PATH} carries for it; ignoring ${tag}.`, + ); continue; } - const pin = slimCatalogPin(from.alias, from.image); - if (pin === undefined || !changed(from.alias, pin.version)) continue; - if (!VERSION_PATTERN.test(pin.version)) { - throw new InvalidPayloadError(`invalid version for ${from.alias}: '${pin.version}'`); - } + const list = candidatesByLine.get(line) ?? []; + list.push({ upstream, revision }); + candidatesByLine.set(line, list); + } - const entry = selectEntry(source, pin.service, pin.version); - const reason = skipReason(from.alias, pin.version, entry); - if (reason !== undefined) { - skipped.push({ alias: from.alias, reason, blocking: true }); - continue; + const updates: SlimUpdate[] = []; + for (const [line, pinned] of pinnedByLine) { + const candidates = candidatesByLine.get(line) ?? []; + + const sameUpstreamRevisions = candidates + .filter((candidate) => candidate.upstream === pinned.upstream) + .map((candidate) => candidate.revision); + if (sameUpstreamRevisions.length > 0) { + const highest = Math.max(...sameUpstreamRevisions); + if (highest > pinned.revision) { + updates.push( + buildUpdate("hotfix", service, line, hasLines, pinned, pinned.upstream, highest), + ); + } } - if (entry.kind !== "default" && entry.kind !== "additional") continue; - if (isOlderRelease(pin.version, entry.version)) { - skipped.push({ - alias: from.alias, - reason: `${pin.service} ${pin.version} is older than the catalog pin ${entry.version}.`, - blocking: false, - }); - continue; + + const highestRevisionByUpstream = new Map(); + for (const candidate of candidates) { + const current = highestRevisionByUpstream.get(candidate.upstream); + if (current === undefined || candidate.revision > current) { + highestRevisionByUpstream.set(candidate.upstream, candidate.revision); + } } - const resolution = await resolveRevisionPin(pin.service, pin.version, input.io); - if (resolution.status !== "resolved") { - skipped.push({ alias: from.alias, reason: resolution.message, blocking: true }); - continue; + let bestUpstream: string | undefined; + for (const upstream of highestRevisionByUpstream.keys()) { + const comparedToPinned = compareVersions(upstream, pinned.upstream); + if (comparedToPinned === undefined) { + console.log(`::warning ::${service} ${upstream} is not a comparable version; ignoring.`); + continue; + } + if (bestUpstream === undefined || (compareVersions(upstream, bestUpstream) ?? 0) > 0) { + bestUpstream = upstream; + } + } + if (bestUpstream !== undefined && compareVersions(bestUpstream, pinned.upstream) === 1) { + const revision = highestRevisionByUpstream.get(bestUpstream) as number; + updates.push(buildUpdate("upgrade", service, line, hasLines, pinned, bestUpstream, revision)); } - const written = writePin(source, entry, resolution.pin); - if (!written.changed) continue; - source = written.source; - updates.push({ - service: pin.service, - version: pin.version, - revision: resolution.pin.revision, - previousVersion: entry.version, - target: entry.kind, - }); } - return { source, updates, skipped }; + return updates; } export interface CatalogRefreshResult { @@ -753,34 +804,55 @@ export interface CatalogRefreshResult { } /** - * Refreshes one catalog entry to the highest committed revision of `upstream` (or, when omitted, - * of its currently pinned upstream version). Pure aside from `io`: callers own reading and - * writing `Artifacts.ts`. This is what the manual CLI mode calls, and what a later hotfix pickup - * workflow can call directly with `{service, upstream_version}` from its dispatch payload. + * Refreshes one catalog entry: to exactly the committed release named by `release` (`-r`, + * required to already be committed — never "highest at apply time"), or to the highest committed + * revision of `upstream` (or, when both are omitted, of the entry's currently pinned upstream + * version). `upstream` and `release` are mutually exclusive. Pure aside from `io`: callers own + * reading and writing `Artifacts.ts`. */ export async function refreshCatalogPin(input: { readonly catalog: string; readonly service: string; readonly upstream?: string; + readonly release?: string; readonly io: RevisionIo; }): Promise { - const entry = selectEntry(input.catalog, input.service, input.upstream); + if (input.upstream !== undefined && input.release !== undefined) { + throw new InvalidPayloadError("--upstream and --release are mutually exclusive."); + } + + let upstream = input.upstream; + let requiredRevision: number | undefined; + if (input.release !== undefined) { + if (!RELEASE_VERSION_PATTERN.test(input.release)) { + throw new InvalidPayloadError(`invalid --release '${input.release}'`); + } + const match = /^(.+)-r(0|[1-9][0-9]*)$/.exec(input.release) as RegExpExecArray; + upstream = match[1]; + requiredRevision = Number(match[2]); + } + + const entry = selectEntry(input.catalog, input.service, upstream); if (entry.kind === "unmodelled-service") { throw new InvalidPayloadError(`${CATALOG_PATH} has no slim entry for ${input.service}.`); } if (entry.kind === "unmodelled-release-line") { throw new InvalidPayloadError( - `${input.service} ${input.upstream ?? ""} is not on a release line ${CATALOG_PATH} carries (${entry.known.join(", ")}).`, + `${input.service} ${upstream ?? ""} is not on a release line ${CATALOG_PATH} carries (${entry.known.join(", ")}).`, ); } - const upstream = input.upstream ?? entry.version; - const resolution = await resolveRevisionPin(input.service, upstream, input.io); + const resolvedUpstream = upstream ?? entry.version; + const resolution = await resolveRevisionPin( + input.service, + resolvedUpstream, + input.io, + requiredRevision, + ); if (resolution.status !== "resolved") { throw new InvalidPayloadError(resolution.message); } - // Manual mode also backfills `upstreamImage`, so a plain `io` (most tests, and the - // Dependabot-driven `planArtifactCatalogUpdate` path) can still exercise revision resolution - // without stubbing the extra fetchers. + // Manual mode also backfills `upstreamImage`, so a plain `io` (most tests) can still exercise + // revision resolution without stubbing the extra fetchers. const pin = input.io.fetchManifest === undefined && input.io.fetchProvenance === undefined ? resolution.pin @@ -789,7 +861,7 @@ export async function refreshCatalogPin(input: { upstreamImage: await resolveUpstreamImage( input.service, `${resolution.pin.upstreamVersion}-r${resolution.pin.revision}`, - upstream, + resolvedUpstream, input.io, ), }; @@ -799,7 +871,7 @@ export async function refreshCatalogPin(input: { source: written.source, update: { service: input.service, - version: upstream, + version: resolvedUpstream, revision: resolution.pin.revision, previousVersion: entry.version, target: entry.kind, @@ -817,6 +889,7 @@ function githubHeaders(): Record { return headers; } +/** Tags of every published, non-draft `supabase/slim-services` release. A draft is never a committed revision. */ async function listReleaseTags(): Promise> { const tags: string[] = []; for (let page = 1; ; page++) { @@ -833,7 +906,9 @@ async function listReleaseTags(): Promise> { throw new InvalidPayloadError("Malformed slim-services releases response."); } for (const item of batch) { - const tagName = (item as { tag_name?: unknown } | null)?.tag_name; + const record = item as { tag_name?: unknown; draft?: unknown } | null; + if (record?.draft === true) continue; + const tagName = record?.tag_name; if (typeof tagName === "string") tags.push(tagName); } if (batch.length < 100) break; @@ -940,7 +1015,7 @@ async function runManual(argv: ReadonlyArray): Promise { const service = flags.get("service"); if (service === undefined) { throw new InvalidPayloadError( - "Usage: sync-artifacts-catalog.ts --service [--upstream ]", + "Usage: sync-artifacts-catalog.ts --service [--upstream | --release -r]", ); } const catalogPath = CATALOG_PATH; @@ -949,6 +1024,7 @@ async function runManual(argv: ReadonlyArray): Promise { catalog, service, upstream: flags.get("upstream"), + release: flags.get("release"), io: defaultRevisionIo(), }); if (result.update === undefined) { @@ -961,28 +1037,39 @@ async function runManual(argv: ReadonlyArray): Promise { ); } -async function runHotfixMatches(argv: ReadonlyArray): Promise { +/** + * Line-oriented encoding of one `SlimUpdate`, for a bash loop: `kind`, `branch`, `title` and + * `release` (the loop's four required fields, driving the checkout/pin/commit/PR steps) plus + * `from` (the release replaced, for the PR body's "from -> to"). Every field is guaranteed + * non-empty by construction. `\x1f` (unit separator) is the delimiter, not a tab: a title can + * carry ordinary whitespace, and `IFS=$'\t' read` would collapse it. + */ +function updateLine(update: SlimUpdate): string { + return [update.kind, update.branch, update.title, update.toRelease, update.fromRelease].join( + "\x1f", + ); +} + +async function runPlanUpdates(argv: ReadonlyArray): Promise { const flags = parseFlags(argv); const service = flags.get("service"); - const upstream = flags.get("upstream"); - const revisionFlag = flags.get("revision"); - if (service === undefined || upstream === undefined || revisionFlag === undefined) { + if (service === undefined) { throw new InvalidPayloadError( - "Usage: sync-artifacts-catalog.ts hotfix-matches --service --upstream --revision ", + "Usage: sync-artifacts-catalog.ts plan-updates --service [--format lines]", ); } - if (!/^(0|[1-9][0-9]*)$/.test(revisionFlag)) { - throw new InvalidPayloadError(`invalid --revision '${revisionFlag}'`); + const format = flags.get("format") ?? "json"; + if (format !== "json" && format !== "lines") { + throw new InvalidPayloadError(`invalid --format '${format}' (expected 'json' or 'lines')`); } - const revision = Number(revisionFlag); const catalog = await Bun.file(CATALOG_PATH).text(); - const matches = findHotfixMatches(catalog, { - service, - upstream_version: upstream, - revision, - release_version: `${upstream}-r${revision}`, - }); - console.log(JSON.stringify(matches)); + const releaseTags = await listReleaseTags(); + const updates = planSlimUpdates(catalog, service, releaseTags); + if (format === "lines") { + for (const update of updates) console.log(updateLine(update)); + return; + } + console.log(JSON.stringify(updates)); } async function runValidatePayload(argv: ReadonlyArray): Promise { @@ -1018,8 +1105,8 @@ async function main(argv: ReadonlyArray): Promise { await runValidatePayload(argv.slice(1)); return; } - if (argv[0] === "hotfix-matches") { - await runHotfixMatches(argv.slice(1)); + if (argv[0] === "plan-updates") { + await runPlanUpdates(argv.slice(1)); return; } if (argv[0]?.startsWith("--") === true) { @@ -1027,34 +1114,10 @@ async function main(argv: ReadonlyArray): Promise { return; } - const [dockerfilePath = DOCKERFILE_PATH, catalogPath = CATALOG_PATH, baseDockerfilePath] = argv; - const dockerfile = await Bun.file(dockerfilePath).text(); - const catalog = await Bun.file(catalogPath).text(); - const baseDockerfile = - baseDockerfilePath === undefined ? undefined : await Bun.file(baseDockerfilePath).text(); - const plan = await planArtifactCatalogUpdate({ - dockerfile, - baseDockerfile, - catalog, - io: defaultRevisionIo(), - }); - for (const skip of plan.skipped) { - console.log(`::warning ::Left ${skip.alias} unchanged: ${skip.reason}`); - } - if (plan.skipped.some((skip) => skip.blocking)) { - console.log("::error ::Refusing to commit a partial catalog update."); - process.exit(1); - } - await Bun.write(catalogPath, plan.source); - if (plan.updates.length === 0) { - console.log("Workload catalog already matches the Dockerfile."); - return; - } - for (const update of plan.updates) { - console.log( - `Pinned ${update.service} ${update.target} ${update.previousVersion} -> ${update.version} r${update.revision}.`, - ); - } + throw new InvalidPayloadError( + "Usage: sync-artifacts-catalog.ts --service [--upstream | --release -r], " + + "or validate-payload / plan-updates --service ", + ); } if (import.meta.main) { diff --git a/.github/workflows/slim-release-published.yml b/.github/workflows/slim-release-published.yml index 25f2a3582c..ed004a0474 100644 --- a/.github/workflows/slim-release-published.yml +++ b/.github/workflows/slim-release-published.yml @@ -1,15 +1,23 @@ name: Slim Release Published # supabase/slim-services sends this dispatch after it mints an immutable release -# `--r`. When that revision is a hotfix of an upstream -# version the workload catalog (packages/stack/src/Artifacts.ts) already pins, this refreshes the -# pin and opens a pull request. +# `--r`. The stack catalog +# (packages/stack/src/Artifacts.ts) is the single version table for the service; this workflow +# treats the dispatch as a trigger and reconciles the catalog against every committed +# (published, non-draft) `-...-r` release, opening or updating one pull request per +# hotfix and/or upgrade it finds (`.github/scripts/sync-artifacts-catalog.ts`'s `planSlimUpdates`). +# +# Plan and apply run from the same checkout of the default branch, in a single job: a re-run +# always recomputes from the latest develop, so a stale plan can never be applied, and a +# superseded PR's branch is rewritten in place (force-pushed) instead of racing a fresh one. A +# backlog republish of an older upstream version naturally plans nothing. # # The payload arrives with whatever authority holds the dispatch token, so it is treated as -# untrusted: fields are pattern- and shape-checked before use, and never interpolated directly -# into a `run:` script (only passed through `env:`). The matching logic itself lives in -# `.github/scripts/sync-artifacts-catalog.ts`'s `findHotfixMatches`, which is unit-tested; this -# workflow only wires it to a PR. +# untrusted: fields are pattern- and shape-checked before use +# (`validateSlimReleasePublishedPayload`), and never interpolated directly into a `run:` script +# (only passed through `env:`). Release tag names come from an external API too, so every value +# `planSlimUpdates` emits is re-validated against the same anchored patterns before it can reach a +# branch name or PR title. on: repository_dispatch: @@ -20,24 +28,39 @@ permissions: contents: read concurrency: - group: slim-release-published-${{ github.event.client_payload.service }}-${{ github.event.client_payload.upstream_version }} + group: slim-release-published-${{ github.event.client_payload.service }} cancel-in-progress: false jobs: pickup: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 permissions: contents: read steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Checkout the default branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ github.event.repository.default_branch }} persist-credentials: false - - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + # Installs the locked workspace dependencies the scripts need: `Artifacts.ts` imports + # `effect`, and `render-service-dockerfile.ts` imports `@supabase/stack/internal/artifacts`. + # Replaces the bare mise step the pre-single-table version of this workflow used. + - name: Setup + uses: ./.github/actions/setup with: - bun-version-file: .bun-version + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + + - name: Record the base commit + id: base + run: echo "sha=$(git rev-parse HEAD)" >>"$GITHUB_OUTPUT" + # Delegates to `validateSlimReleasePublishedPayload` (sync-artifacts-catalog.ts) so every + # field — including `upstream_version` and `release_version`, not just `service` and + # `revision` — is checked against an anchored charset before it is ever written to + # `$GITHUB_OUTPUT`, a branch name, or a PR title. A value that fails validation makes the + # script exit non-zero without printing anything. - name: Validate payload id: validate env: @@ -45,11 +68,6 @@ jobs: UPSTREAM_VERSION: ${{ github.event.client_payload.upstream_version }} REVISION: ${{ github.event.client_payload.revision }} RELEASE_VERSION: ${{ github.event.client_payload.release_version }} - # Delegates to `validateSlimReleasePublishedPayload` (sync-artifacts-catalog.ts) so every - # field — including `upstream_version` and `release_version`, not just `service` and - # `revision` — is checked against an anchored charset before it is ever written to - # `$GITHUB_OUTPUT`, a branch name, or a PR title. A value that fails validation makes the - # script exit non-zero without printing anything. run: | set -euo pipefail if ! output="$(bun .github/scripts/sync-artifacts-catalog.ts validate-payload \ @@ -59,34 +77,31 @@ jobs: fi echo "$output" >>"$GITHUB_OUTPUT" - - name: Find hotfix matches - id: matches + # Reconciles the service against every committed slim-services release (`planSlimUpdates`), + # not just the release that triggered this dispatch — this run is idempotent and safe to + # receive out of order. Written to a file, not a $GITHUB_OUTPUT multi-line value: each line + # is field-separated with `\x1f` (see `updateLine`), which a later step reads directly. + - name: Plan updates + id: plan env: SERVICE: ${{ steps.validate.outputs.service }} - UPSTREAM_VERSION: ${{ steps.validate.outputs.upstream_version }} - REVISION: ${{ steps.validate.outputs.revision }} + GITHUB_TOKEN: ${{ github.token }} run: | set -euo pipefail - result="$(bun .github/scripts/sync-artifacts-catalog.ts hotfix-matches \ - --service "$SERVICE" --upstream "$UPSTREAM_VERSION" --revision "$REVISION")" - count="$(bun -e 'console.log(JSON.parse(await Bun.stdin.text()).length)' <<<"$result")" + bun .github/scripts/sync-artifacts-catalog.ts plan-updates --service "$SERVICE" --format lines \ + > "${RUNNER_TEMP}/slim-updates.tsv" + count="$(wc -l < "${RUNNER_TEMP}/slim-updates.tsv" | tr -d ' ')" echo "count=${count}" >>"$GITHUB_OUTPUT" - - name: No matching catalog pin - if: steps.matches.outputs.count == '0' + - name: Nothing to do + if: steps.plan.outputs.count == '0' env: SERVICE: ${{ steps.validate.outputs.service }} RELEASE_VERSION: ${{ steps.validate.outputs.release_version }} - run: echo "No catalog pin for ${SERVICE} needs ${RELEASE_VERSION}; nothing to do." - - - name: Install toolchains - if: steps.matches.outputs.count != '0' - uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0 - with: - version: 2026.9.0 + run: echo "No catalog updates for ${SERVICE} from ${RELEASE_VERSION}; nothing to do." - name: Install regctl - if: steps.matches.outputs.count != '0' + if: steps.plan.outputs.count != '0' run: | set -euo pipefail install -d "${RUNNER_TEMP}/regctl-bin" @@ -98,7 +113,7 @@ jobs: "${RUNNER_TEMP}/regctl-bin/regctl" version - name: Generate token - if: steps.matches.outputs.count != '0' + if: steps.plan.outputs.count != '0' id: app-token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: @@ -107,58 +122,86 @@ jobs: permission-contents: write permission-pull-requests: write - - name: Pin the catalog to the hotfix revision - if: steps.matches.outputs.count != '0' + # Kept as its own step, before the loop, so the loop body below can be exercised against a + # local bare remote in a dry run (point `origin` there instead of running this step). + - name: Point origin at the authenticated remote + if: steps.plan.outputs.count != '0' env: - SERVICE: ${{ steps.validate.outputs.service }} - UPSTREAM_VERSION: ${{ steps.validate.outputs.upstream_version }} - run: bun .github/scripts/sync-artifacts-catalog.ts --service "$SERVICE" --upstream "$UPSTREAM_VERSION" - - - name: Format catalog - if: steps.matches.outputs.count != '0' - run: bun x oxfmt@0.65.0 --config .oxfmtrc.json packages/stack/src/Artifacts.ts + APP_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git remote set-url origin "https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - - name: Open pull request - if: steps.matches.outputs.count != '0' + # One branch per planned item, built fresh from the base commit recorded above (the default + # branch's head at the start of this run) — never from wherever a previous loop iteration + # left HEAD. Pins exactly the planned release (`--release`, never "highest at apply time"), + # so plan and pin agree by construction: a revision minted a second later arrives with its + # own dispatch, not by racing this one. + - name: Apply planned updates + if: steps.plan.outputs.count != '0' env: - APP_TOKEN: ${{ steps.app-token.outputs.token }} GH_TOKEN: ${{ steps.app-token.outputs.token }} SERVICE: ${{ steps.validate.outputs.service }} - UPSTREAM_VERSION: ${{ steps.validate.outputs.upstream_version }} - REVISION: ${{ steps.validate.outputs.revision }} RELEASE_VERSION: ${{ steps.validate.outputs.release_version }} + BASE_SHA: ${{ steps.base.outputs.sha }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | set -euo pipefail - if git diff --quiet -- packages/stack/src/Artifacts.ts; then - echo "Workload catalog already pins ${SERVICE} ${RELEASE_VERSION}." - exit 0 - fi - - manual_command="bun .github/scripts/sync-artifacts-catalog.ts --service ${SERVICE} --upstream ${UPSTREAM_VERSION}" - branch="slim-hotfix/${SERVICE}-${RELEASE_VERSION}" - - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git checkout -b "$branch" - git add -- packages/stack/src/Artifacts.ts - git commit -m "chore(stack): pin ${SERVICE} ${RELEASE_VERSION}" - git remote set-url origin "https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - - if ! git push origin "HEAD:refs/heads/${branch}"; then - echo "::error ::Failed to push ${branch}. Run manually: ${manual_command}, then open a pull request by hand." - exit 1 - fi - - body="$(printf 'supabase/slim-services published a hotfix revision `%s` for `%s`. This pins the workload catalog to it.\n\nhttps://github.com/supabase/slim-services/releases/tag/%s-%s\n' \ - "$RELEASE_VERSION" "$SERVICE" "$SERVICE" "$RELEASE_VERSION")" - - if ! gh pr create \ - --title "chore(stack): pin ${SERVICE} ${RELEASE_VERSION}" \ - --body "$body" \ - --assignee jgoux \ - --head "$branch" \ - --base "$DEFAULT_BRANCH"; then - echo "::error ::Pushed ${branch} but failed to open the pull request. Run manually: ${manual_command}, then open a pull request from ${branch} against ${DEFAULT_BRANCH} by hand." - exit 1 - fi + manual_prefix="bun .github/scripts/sync-artifacts-catalog.ts --service ${SERVICE} --release" + + while IFS=$'\x1f' read -r kind branch title release from; do + if [ -z "$kind" ] || [ -z "$branch" ] || [ -z "$title" ] || [ -z "$release" ] || [ -z "$from" ]; then + echo "::error ::Malformed plan-updates line: kind='${kind}' branch='${branch}' title='${title}' release='${release}' from='${from}'." + exit 1 + fi + + git checkout -B "$branch" "$BASE_SHA" + + bun .github/scripts/sync-artifacts-catalog.ts --service "$SERVICE" --release "$release" + bun x oxfmt@0.65.0 --config .oxfmtrc.json packages/stack/src/Artifacts.ts + bun apps/cli/scripts/render-service-dockerfile.ts + + generated_files=( + packages/stack/src/Artifacts.ts + apps/cli/src/shared/services/Dockerfile + apps/cli-go/pkg/config/templates/Dockerfile + ) + if git diff --quiet -- "${generated_files[@]}"; then + echo "${branch}: catalog and Dockerfiles already match ${release}; skipping." + continue + fi + + git add -- "${generated_files[@]}" + git commit -m "$title" + + if ! git push --force origin "HEAD:refs/heads/${branch}"; then + echo "::error ::Failed to push ${branch}. Run manually: ${manual_prefix} ${release}, regenerate the Dockerfiles with \`bun apps/cli/scripts/render-service-dockerfile.ts\`, then open a pull request by hand." + exit 1 + fi + + if [ "$kind" = "upgrade" ]; then + action="bumps" + else + action="pins" + fi + body="$(printf 'supabase/slim-services published %s. This %s %s from %s to %s.\n\nhttps://github.com/supabase/slim-services/releases/tag/%s-%s\n\nThis branch is rewritten from %s whenever a newer relevant release arrives.\n' \ + "$RELEASE_VERSION" "$action" "$SERVICE" "$from" "$release" "$SERVICE" "$release" "$DEFAULT_BRANCH")" + + existing="$(gh pr list --head "$branch" --base "$DEFAULT_BRANCH" --state open --json number --jq '.[0].number // empty')" + if [ -n "$existing" ]; then + if ! gh pr edit "$existing" --title "$title" --body "$body"; then + echo "::error ::Pushed ${branch} but failed to edit pull request #${existing}. Run manually: ${manual_prefix} ${release}, then edit the pull request from ${branch} by hand." + exit 1 + fi + elif ! gh pr create \ + --title "$title" \ + --body "$body" \ + --assignee jgoux \ + --head "$branch" \ + --base "$DEFAULT_BRANCH"; then + echo "::error ::Pushed ${branch} but failed to open a pull request. Run manually: ${manual_prefix} ${release}, then open a pull request from ${branch} against ${DEFAULT_BRANCH} by hand." + exit 1 + fi + done < "${RUNNER_TEMP}/slim-updates.tsv" diff --git a/.github/workflows/sync-artifacts-catalog.yml b/.github/workflows/sync-artifacts-catalog.yml deleted file mode 100644 index d57b5d7b28..0000000000 --- a/.github/workflows/sync-artifacts-catalog.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: Sync Artifacts catalog - -# The script is the base revision. The pull request only supplies the Dockerfile -# and catalog text, and the job runs for Dependabot so that pull request cannot -# edit this workflow and then execute it with a write token. The catalog commit -# is pushed with the app token so the new head triggers CI. Dependabot's -# GITHUB_TOKEN would not. -on: - pull_request: - paths: - - apps/cli/src/shared/services/Dockerfile - -permissions: - contents: read - -concurrency: - group: sync-artifacts-catalog-${{ github.event.pull_request.number }} - cancel-in-progress: false - -jobs: - sync: - name: Pin workload catalog - if: github.actor == 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest - timeout-minutes: 15 - permissions: - contents: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.event.pull_request.base.sha }} - persist-credentials: true - fetch-depth: 1 - - # Bun only. The script imports no workspace packages. The firewall token - # is empty on a Dependabot run; setup is not required to format one file. - - name: Install toolchains - uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0 - with: - version: 2026.9.0 - - - name: Install regctl - run: | - set -euo pipefail - install -d "${RUNNER_TEMP}/regctl-bin" - curl -fsSLo "${RUNNER_TEMP}/regctl-bin/regctl" \ - https://github.com/regclient/regclient/releases/download/v0.11.5/regctl-linux-amd64 - echo "c93aa7638749f5aaac1a8e01787321889c78f0101809bb2880343478d0ba0467 ${RUNNER_TEMP}/regctl-bin/regctl" | sha256sum -c - - chmod +x "${RUNNER_TEMP}/regctl-bin/regctl" - echo "${RUNNER_TEMP}/regctl-bin" >> "$GITHUB_PATH" - "${RUNNER_TEMP}/regctl-bin/regctl" version - - - name: Generate token - id: app-token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - client-id: ${{ vars.GH_APP_CLIENT_ID }} - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - permission-contents: write - - - name: Pin the catalog to the Dockerfile - env: - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - HEAD_REF: ${{ github.event.pull_request.head.ref }} - PR_NUMBER: ${{ github.event.pull_request.number }} - APP_TOKEN: ${{ steps.app-token.outputs.token }} - run: | - set -euo pipefail - git fetch --no-tags origin "pull/${PR_NUMBER}/head" - fetched="$(git rev-parse FETCH_HEAD)" - if [ "$fetched" != "$HEAD_SHA" ]; then - echo "::error ::pull/${PR_NUMBER} head is ${fetched}, expected ${HEAD_SHA}." - exit 1 - fi - dockerfile="$(mktemp)" - catalog="$(mktemp)" - original="$(mktemp)" - git show "$HEAD_SHA:apps/cli/src/shared/services/Dockerfile" > "$dockerfile" - git show "$HEAD_SHA:packages/stack/src/Artifacts.ts" > "$catalog" - cp "$catalog" "$original" - bun .github/scripts/sync-artifacts-catalog.ts "$dockerfile" "$catalog" apps/cli/src/shared/services/Dockerfile - if cmp -s "$original" "$catalog"; then - echo "Workload catalog already matches the Dockerfile." - exit 0 - fi - git checkout --detach "$HEAD_SHA" - cp "$catalog" packages/stack/src/Artifacts.ts - bun x oxfmt@0.65.0 --config .oxfmtrc.json packages/stack/src/Artifacts.ts - git add -- packages/stack/src/Artifacts.ts - if git diff --cached --quiet -- packages/stack/src/Artifacts.ts; then - echo "Workload catalog already matches the Dockerfile." - exit 0 - fi - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git commit -m "chore(stack): pin the workload catalog to the Dockerfile image tags" - git remote set-url origin "https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git push origin "HEAD:refs/heads/${HEAD_REF}" From 0467c77a1ba20a627a9b9dac2ed491ba9a10f853 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 17:13:16 +0200 Subject: [PATCH 23/45] chore(ci): drop the Dependabot catalog sync The stack catalog is now the single version table; the Dockerfile's slim-capable lines are generated from it and updated by slim-release-published, not by syncing a Dependabot bump back into the catalog. Dependabot now ignores every slim-capable image and keeps bumping only the upstream-only ones. --- .github/dependabot.yml | 14 ++++++++++++++ .github/workflows/mirror-template-images.yml | 7 ++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3b6a0cd801..3e97bc0c96 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -85,11 +85,25 @@ updates: update-types: - minor - patch + # These lines are generated from packages/stack/src/Artifacts.ts (the single version table + # for slim-capable services); updates to them arrive through slim-release-published.yml, not + # Dependabot. Dependabot keeps bumping only the upstream-only images the Dockerfile still + # hand-manages: kong, and the job images (migra, pg_prove, pgadmin-schema-diff). ignore: - dependency-name: "library/kong" - dependency-name: "axllent/mailpit" - dependency-name: "darthsim/imgproxy" - dependency-name: "timberio/vector" + - dependency-name: "supabase/postgres" + - dependency-name: "supabase/gotrue" + - dependency-name: "postgrest/postgrest" + - dependency-name: "supabase/realtime" + - dependency-name: "supabase/storage-api" + - dependency-name: "supabase/edge-runtime" + - dependency-name: "supabase/studio" + - dependency-name: "supabase/postgres-meta" + - dependency-name: "supabase/logflare" + - dependency-name: "supabase/supavisor" cooldown: default-days: 7 exclude: diff --git a/.github/workflows/mirror-template-images.yml b/.github/workflows/mirror-template-images.yml index 3b8cb03676..dabc95430e 100644 --- a/.github/workflows/mirror-template-images.yml +++ b/.github/workflows/mirror-template-images.yml @@ -3,9 +3,10 @@ name: Mirror template images # Keeps the ghcr.io/ECR mirror in sync with the image versions pinned in # apps/cli/src/shared/services/Dockerfile, which `dockerfileServiceImages` # parses as the single source of truth for local service images. When the -# Dockerfile changes on develop — most often via a merged dependabot `docker` -# bump — this workflow detects any tag that is not yet mirrored and backfills -# it the same way `cli-go-mirror-image.yml` does. +# Dockerfile changes on develop — most often via a merged catalog bump or +# hotfix PR from slim-release-published.yml, or a Dependabot `docker` bump for +# an upstream-only image — this workflow detects any tag that is not yet +# mirrored and backfills it the same way `cli-go-mirror-image.yml` does. # # It runs on `push` to develop (not on the PR) on purpose: mirroring needs the # AWS role + packages:write, which a dependabot-triggered `pull_request` run From 4f6e41a71d6d2a752534fb6489b10ae91f0739d1 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 17:13:22 +0200 Subject: [PATCH 24/45] docs: single version table for slim services Rewrite ADR 0026 for the catalog-as-single-version-table model: the Dockerfile's slim-capable lines are generated with a drift check, and hotfix/upgrade pull requests arrive through slim-release-published instead of a Dependabot-driven sync. --- docs/adr/0026-slim-artifact-mirrors.md | 93 +++++++++++++++++++------- 1 file changed, 70 insertions(+), 23 deletions(-) diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index b478613500..e304a5d9c4 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -34,12 +34,33 @@ semver library — ordering is the service's upstream version key, then revision **Legacy tags** (without `-rN`) are frozen: nothing parses, audits, rewrites, or unpublishes them, so CLIs pinned to a legacy tag keep working. +### The catalog is the single version table + +The CLI's stack catalog (`packages/stack/src/Artifacts.ts`) is the single version table for +every consumer of a slim-capable service: the new stack, legacy `supabase start`, and legacy +slim mode. Each slim-capable service pins exactly one committed slim-services release per +release line it carries (`ArtifactPin`, keyed by upstream version; only postgres carries more +than one line, its default plus an additional Postgres 15 pin). Everything else is derived from +that pin — never the other way around. + +`apps/cli/src/shared/services/Dockerfile`, and its byte-identical Go copy +(`apps/cli-go/pkg/config/templates/Dockerfile`), keep every existing consumer (the legacy TS +stack, the Go CLI embed, `mirror-template-images.yml`, `detect-unmirrored-images.ts`) unchanged, +but for a slim-capable alias its `FROM` line is now a **generated view** of the catalog's +`ArtifactPin.upstreamImage`: `apps/cli/scripts/render-service-dockerfile.ts` rewrites those lines +in place, and a CI check (`render-service-dockerfile.unit.test.ts`) fails on drift between the +Dockerfile and the catalog. Kong, Postgres 14, and the one-shot job images (migra, pg_prove, +pgadmin-schema-diff) have no slim build and stay hand- or Dependabot-managed, as today. + ### Content pins, not runtime checksums -The CLI's stack catalog (`packages/stack/src/Artifacts.ts`) pins every artifact by content: +Every `ArtifactPin` pins its release by content: - the slim image, as `ghcr.io/supabase/cli/:@sha256:`; -- each target's archive sha256 and manifest sha256 (`ArtifactPin.natives`). +- each target's archive sha256 and manifest sha256 (`ArtifactPin.natives`); +- `upstreamImage`, the exact upstream image the release was built or mirrored from, normalized to + the Dockerfile's `FROM` form — this is what legacy non-slim mode resolves against, and what the + generator reads to render the Dockerfile. `SlimServicesSource` hash-checks the manifest against its pin before parsing it (its `version` field must equal `R`) and hash-checks the archive against its pin while it streams. There is no @@ -49,30 +70,41 @@ hash — that comes only from the pin already committed to the catalog. ### Sync and the S3-staleness check -`.github/scripts/sync-artifacts-catalog.ts` writes those pins. On a Dependabot upstream bump to -`U`, it picks the highest committed revision `N` of `-U-rN`, reads that release's -`SHA256SUMS` for the archive and manifest sha256 per target, and resolves the image digest with -`regctl manifest head`. Before writing the pin, it downloads each target's S3 archive and -manifest and hashes them against those same release sums; a mismatch fails the sync instead of -pinning bytes that don't match GitHub. This exists because `publish-release` does not wait for +`.github/scripts/sync-artifacts-catalog.ts` writes those pins, in manual mode: given a service +and either `--upstream` (the highest committed revision of that upstream version) or `--release` +(exactly that committed `-r`, never "highest at apply time"), it reads the target +release's `SHA256SUMS` for the archive and manifest sha256 per target, and resolves the image +digest with `regctl manifest head`. Before writing the pin, it downloads each target's S3 archive +and manifest and hashes them against those same release sums; a mismatch fails the sync instead +of pinning bytes that don't match GitHub. This exists because `publish-release` does not wait for the ECR/S3 mirror to finish, so a freshly committed revision's S3 copy can briefly lag or hold bytes from an earlier failed attempt. Hosts that reach GitHub are unaffected — GitHub is the primary mirror — but a host that can only reach S3 would otherwise fail verification with no -fallback. A mismatch here means "run the mirror backfill", not "the CLI is broken". A Dependabot -bump to a version _older_ than the catalog's current pin is a non-blocking skip: the catalog is -allowed to lead the Dockerfile, and the skip still prevents a downgrade. - -### Hotfix pickup - -`mirror-ecr` runs before `publish-release`, so the mirror dispatch fires before a revision is -committed and can't trigger a catalog update. Instead, the last step of slim-services' -`publish-release` sends a `repository_dispatch` (`slim-release-published`) to the CLI repo with -`{service, upstream_version, revision}`. A CLI workflow handles it: when `upstream_version` -matches any pinned upstream version for that service and `revision` is newer than the pinned -one, it runs the sync for that entry and opens a pull request. The fallback, if that dispatch or -the PR step fails, is a documented manual `bun .github/scripts/sync-artifacts-catalog.ts` -invocation — the release itself is already committed by then, so a failure here means "notify -the CLI by hand", not "republish". +fallback. A mismatch here means "run the mirror backfill", not "the CLI is broken". + +### Hotfix and upgrade pickup + +The last step of slim-services' `publish-release` sends a `repository_dispatch` +(`slim-release-published`) to the CLI repo with `{service, upstream_version, revision, +release_version}`. `slim-release-published.yml` treats the dispatch as a trigger, not as the +payload to apply: it reconciles the named service against every committed (published, non-draft) +`-...-r` release it can currently see (`planSlimUpdates`), and opens or updates, per +release line the service carries: + +- a **hotfix**, when the pinned upstream version has a higher committed revision — branch + `slim-hotfix/[-]`, title `chore(stack): pin `; +- an **upgrade**, when the newest committed upstream on that line is newer than the pinned one — + branch `slim-bump/[-]`, title `chore(stack): bump to `. + +Both can be planned in the same run. Plan and apply run from the same checkout of the default +branch in one job, so a re-run always recomputes from the latest develop: a stale plan can never +be applied, and a superseded PR's branch is rewritten (force-pushed) in place rather than raced +by a new one — the workflow deliberately never auto-closes a superseded PR. A backlog republish +of an older upstream version naturally plans nothing. The fallback, if the push or PR step fails, +is a documented manual `bun .github/scripts/sync-artifacts-catalog.ts --service --release +-r` invocation, followed by `apps/cli/scripts/render-service-dockerfile.ts` — the release +itself is already committed by then, so a failure here means "open the pull request by hand", not +"republish". ### Registry and bucket mirrors @@ -117,6 +149,21 @@ or considered stale and backfilled — never rewritten in place. and the mirror backfill are what keep them converging. - Old CLI releases keep pulling their originally pinned image digest and native bytes forever. +### Tradeoffs of moving updates onto the catalog + +- Upstream and security fixes for slim-capable images now reach the CLI only once + slim-services publishes a release. Direct Dependabot discovery is gone for them; Dependabot + still bumps only the upstream-only images `.github/dependabot.yml`'s docker `ignore` list + excludes (kong, and the job images: migra, pg_prove, pgadmin-schema-diff). +- Dependabot's 7-day cooldown no longer governs those images — hotfix and upgrade PRs land as + soon as `slim-release-published` fires, on whatever cadence slim-services publishes. +- `pg14` has no slim build and was already upstream-only; it is bumped by hand (Dependabot's + `docker` ecosystem also ignores `supabase/postgres` entirely now, since Dependabot cannot tell + `pg14`'s `FROM` line apart from `pg`'s and `pg15`'s by repository name alone). +- The Deno 1 edge-runtime override (`apps/cli/src/shared/functions/functions.shared.ts`, and the + Go `deno1` constant) stays a separately pinned, upstream-only exception — it never goes through + the catalog. + ## Alternatives considered 1. **Google Artifact Registry / GCS** — blob host `storage.googleapis.com` is on at least one From 835e8972a3d14b26d40d3f93c5b8ed43225a6b4b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 17:24:39 +0200 Subject: [PATCH 25/45] refactor(cli): stop exporting the OrioleDB image check --- apps/cli/src/shared/services/slim-images.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index 4e9dabbb92..f99ea85ce3 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -70,7 +70,7 @@ export interface SlimCatalogPin { } /** OrioleDB tags are docker.io-only; slim-services does not publish them. */ -export function isOrioleImage(image: string): boolean { +function isOrioleImage(image: string): boolean { const tag = imageTag(image); return tag !== undefined && tag.toLowerCase().includes("orioledb"); } From 9279d6c479f82d98eb975c6d66f5e28b7f8d1d6f Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 17:42:17 +0200 Subject: [PATCH 26/45] fix(ci): keep planner warnings out of the plan records planSlimUpdates logged ::warning:: lines with console.log onto the same stdout plan-updates wrote its records to, so one ignored release tag corrupted the file and blocked every valid update in the run. The planner is now pure and returns { updates, warnings } instead of logging; records go only to a required --output file, warnings only to stdout. Also fixes services with no additional pins being filtered by releaseLine even though they carry only one line, which wrongly ignored a Studio year rollover or a postgrest major bump, and validates --upstream the same way --release already is. --- .../scripts/sync-artifacts-catalog.test.ts | 133 ++++++++++++++++-- .github/scripts/sync-artifacts-catalog.ts | 111 +++++++++++---- 2 files changed, 200 insertions(+), 44 deletions(-) diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index dd732a71ca..48cbaa2b80 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -7,6 +7,7 @@ import { InvalidPayloadError, nativeFileNames, nativeObjectUrl } from "./slim-mi import { CATALOG_PATH, planSlimUpdates, + planUpdatesForService, refreshCatalogPin, validateSlimReleasePublishedPayload, type RevisionIo, @@ -205,8 +206,11 @@ describe("validateSlimReleasePublishedPayload", () => { describe("planSlimUpdates", () => { test("hotfix only: a higher committed revision of the pinned upstream", () => { - const updates = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.2-r1"]); + const { updates, warnings } = planSlimUpdates(plannerFixture, "postgrest", [ + "postgrest-v16.2-r1", + ]); + expect(warnings).toEqual([]); expect(updates).toEqual([ { kind: "hotfix", @@ -221,7 +225,7 @@ describe("planSlimUpdates", () => { }); test("upgrade only: a newer committed upstream on the same line", () => { - const updates = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.4-r0"]); + const { updates } = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.4-r0"]); expect(updates).toEqual([ { @@ -237,7 +241,7 @@ describe("planSlimUpdates", () => { }); test("both a hotfix and an upgrade can be planned in the same run", () => { - const updates = planSlimUpdates(plannerFixture, "postgrest", [ + const { updates } = planSlimUpdates(plannerFixture, "postgrest", [ "postgrest-v16.2-r1", "postgrest-v16.4-r0", ]); @@ -265,18 +269,26 @@ describe("planSlimUpdates", () => { }); test("an older committed upstream is a backlog republish: it plans nothing", () => { - const updates = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.1-r0"]); + const { updates, warnings } = planSlimUpdates(plannerFixture, "postgrest", [ + "postgrest-v16.1-r0", + ]); expect(updates).toEqual([]); + expect(warnings).toEqual([]); }); - test("postgres: upgrade on the 17 line, hotfix on the 15 line, a major-18 release ignored", () => { - const updates = planSlimUpdates(postgresPlannerFixture, "postgres", [ + test("postgres: upgrade on the 17 line, hotfix on the 15 line, a major-18 release ignored and warned about", () => { + const { updates, warnings } = planSlimUpdates(postgresPlannerFixture, "postgres", [ "postgres-17.11.0.002-r0", "postgres-15.14.1.168-r4", "postgres-18.0.0.001-r0", ]); + // The ignored major-18 tag is warned about, but doesn't block the two valid updates + // alongside it (P1: a warning must never corrupt or swallow the plan). + expect(warnings).toEqual([ + "::warning ::postgres 18.0.0.001 is not on a release line packages/stack/src/Artifacts.ts carries for it; ignoring postgres-18.0.0.001-r0.", + ]); expect(updates).toEqual([ { kind: "upgrade", @@ -315,12 +327,12 @@ describe("planSlimUpdates", () => { `; const sameDate = planSlimUpdates(studioFixture, "studio", ["studio-2026.09.14-sha-bbbbbbb-r0"]); - expect(sameDate).toEqual([]); + expect(sameDate.updates).toEqual([]); const newerDate = planSlimUpdates(studioFixture, "studio", [ "studio-2026.09.28-sha-ccccccc-r0", ]); - expect(newerDate).toEqual([ + expect(newerDate.updates).toEqual([ { kind: "upgrade", line: undefined, @@ -333,20 +345,78 @@ describe("planSlimUpdates", () => { ]); }); - test("a non-comparable version (OrioleDB-style suffix) is ignored", () => { - const updates = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.2-orioledb-r0"]); + test("Studio: a year rollover upgrades even though releaseLine differs (single-pin services accept any upstream)", () => { + const studioFixture = `const workloadCatalog = { + studio: definition( + "studio", + { + upstreamVersion: "2026.09.28-sha-5e59b60", + revision: 0, + image: "ghcr.io/supabase/cli/studio:2026.09.28-sha-5e59b60-r0@sha256:5555555555555555555555555555555555555555555555555555555555555", + natives: {}, + }, + "bin/studio", + ), +}; +`; + + const { updates, warnings } = planSlimUpdates(studioFixture, "studio", [ + "studio-2027.01.01-sha-abcdef0-r0", + ]); + + expect(warnings).toEqual([]); + expect(updates).toEqual([ + { + kind: "upgrade", + line: undefined, + branch: "slim-bump/studio", + title: "chore(stack): bump studio to 2027.01.01-sha-abcdef0-r0", + fromRelease: "2026.09.28-sha-5e59b60-r0", + toUpstream: "2027.01.01-sha-abcdef0", + toRelease: "2027.01.01-sha-abcdef0-r0", + }, + ]); + }); + + test("postgrest: a major-version bump upgrades even though releaseLine differs (single-pin services accept any upstream)", () => { + const { updates, warnings } = planSlimUpdates(plannerFixture, "postgrest", [ + "postgrest-v17.0-r0", + ]); + + expect(warnings).toEqual([]); + expect(updates).toEqual([ + { + kind: "upgrade", + line: undefined, + branch: "slim-bump/postgrest", + title: "chore(stack): bump postgrest to v17.0-r0", + fromRelease: "v16.2-r0", + toUpstream: "v17.0", + toRelease: "v17.0-r0", + }, + ]); + }); + + test("a non-comparable version (OrioleDB-style suffix) is ignored and warned about", () => { + const { updates, warnings } = planSlimUpdates(plannerFixture, "postgrest", [ + "postgrest-v16.2-orioledb-r0", + ]); expect(updates).toEqual([]); + expect(warnings).toEqual([ + "::warning ::postgrest v16.2-orioledb is not a comparable version; ignoring.", + ]); }); test("a legacy tag with no -rN is ignored", () => { - const updates = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.4"]); + const { updates, warnings } = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.4"]); expect(updates).toEqual([]); + expect(warnings).toEqual([]); }); test("postgrest tags never match postgres, even as a prefix", () => { - const updates = planSlimUpdates(postgresPlannerFixture, "postgres", [ + const { updates } = planSlimUpdates(postgresPlannerFixture, "postgres", [ "postgrest-v16.4-r0", "postgres-17.11.0.002-r0", ]); @@ -365,7 +435,7 @@ describe("planSlimUpdates", () => { }); test("branch and title carry no - suffix for a service with a single line", () => { - const updates = planSlimUpdates(plannerFixture, "storage", ["storage-v1.74.0-r0"]); + const { updates } = planSlimUpdates(plannerFixture, "storage", ["storage-v1.74.0-r0"]); expect(updates).toEqual([ { @@ -405,10 +475,43 @@ describe("planSlimUpdates", () => { ).toThrow(InvalidPayloadError); }); - test("plans nothing for a service the catalog does not model", () => { - const updates = planSlimUpdates(plannerFixture, "no-such-service", ["no-such-service-v1.0-r0"]); + test("plans nothing for a service the catalog does not model, and warns about it", () => { + const { updates, warnings } = planSlimUpdates(plannerFixture, "no-such-service", [ + "no-such-service-v1.0-r0", + ]); expect(updates).toEqual([]); + expect(warnings).toEqual([ + "::warning ::packages/stack/src/Artifacts.ts has no slim entry for no-such-service; nothing to plan.", + ]); + }); +}); + +describe("planUpdatesForService (the plan-updates transport's IO seam)", () => { + test("an ignored tag alongside a valid update: the valid record comes back, plus a separate warning", async () => { + const result = await planUpdatesForService({ + catalog: postgresPlannerFixture, + service: "postgres", + listReleaseTags: async () => [ + "postgres-17.11.0.002-r0", + "postgres-18.0.0.001-r0", // ignored: no line 18 + ], + }); + + expect(result.warnings).toEqual([ + "::warning ::postgres 18.0.0.001 is not on a release line packages/stack/src/Artifacts.ts carries for it; ignoring postgres-18.0.0.001-r0.", + ]); + expect(result.updates).toEqual([ + { + kind: "upgrade", + line: "17", + branch: "slim-bump/postgres-17", + title: "chore(stack): bump postgres to 17.11.0.002-r0", + fromRelease: "17.6.1.168-r1", + toUpstream: "17.11.0.002", + toRelease: "17.11.0.002-r0", + }, + ]); }); }); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index d0a94b0237..a3ad81bedc 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -18,10 +18,14 @@ * Plan-updates mode (used by the `slim-release-published` dispatch workflow): * lists a service's committed slim-services releases and computes, per * release line, the hotfix and/or upgrade a workflow should apply. Pure - * planning: `planSlimUpdates` takes the release tag list as an argument and - * makes no network calls. + * planning: `planSlimUpdates` takes the release tag list as an argument, + * makes no network calls, and never logs — it returns `{ updates, warnings }`. + * Records go only to `--output ` (never stdout); warnings print to + * stdout as `::warning ::…` lines, so the two channels can't corrupt one + * another when a caller redirects stdout separately from the records file. * - * bun .github/scripts/sync-artifacts-catalog.ts plan-updates --service [--format lines] + * bun .github/scripts/sync-artifacts-catalog.ts plan-updates --service \ + * --output [--format lines] * * Validate-payload mode (used by the same workflow, before anything else): * checks an untrusted `slim-release-published` dispatch payload against @@ -698,29 +702,45 @@ function buildUpdate( }; } +export interface PlanSlimUpdatesResult { + readonly updates: ReadonlyArray; + /** `::warning ::…` workflow-command lines, for the caller to print to stdout. */ + readonly warnings: ReadonlyArray; +} + +/** Groups every candidate release tag under one bucket, for a service with no additional pins. */ +const SINGLE_LINE_KEY = "*"; + /** * The hotfix and/or upgrade a `slim-release-published` run should apply for `service`, computed * against `catalog`'s current pins and `releaseTags` (every committed — published, non-draft — * slim-services release tag; the caller filters drafts before calling this). Pure: no network, no - * file I/O. + * file I/O, no logging — every diagnostic comes back in `warnings` instead, so a caller (the CLI, + * or a test) decides where it goes. This matters because `plan-updates` writes `updates` straight + * into a file the workflow parses as records; a `console.log`'d warning on the same stdout the + * workflow captures would corrupt that file instead of just being informational. * * Per release line (ruling: the default pin's line, plus one per additional pin — only postgres - * has more than one): a **hotfix** fires when the pinned upstream has a committed revision higher + * has more than one) a **hotfix** fires when the pinned upstream has a committed revision higher * than the pinned one; an **upgrade** fires when the newest committed upstream on the line is * newer than the pinned one (ties, e.g. two Studio builds dated the same day, are not newer). - * Both can fire in the same run. A release tag whose upstream sits on no line the catalog carries - * for `service`, or whose version isn't comparable, is logged and ignored — it never causes a - * plan-updates run to fail. + * Both can fire in the same run. A service with no additional pins has exactly one line and + * accepts any comparable upstream on it — a Studio year rollover or a postgrest major bump is + * the same line moving forward, not a different one, so it is never filtered by `releaseLine`. + * Only a service that does carry additional pins (postgres) filters a release tag to the line its + * `releaseLine` names; a tag on no such line, or whose version isn't comparable, is warned about + * and ignored — it never causes a plan-updates run to fail. */ export function planSlimUpdates( catalog: string, service: string, releaseTags: ReadonlyArray, -): ReadonlyArray { +): PlanSlimUpdatesResult { + const warnings: string[] = []; const pins = collectServicePins(catalog, service); if (pins === undefined) { - console.log(`::warning ::${CATALOG_PATH} has no slim entry for ${service}; nothing to plan.`); - return []; + warnings.push(`::warning ::${CATALOG_PATH} has no slim entry for ${service}; nothing to plan.`); + return { updates: [], warnings }; } const allPins = [pins.defaultPin, ...pins.additional]; @@ -728,7 +748,7 @@ export function planSlimUpdates( const pinnedByLine = new Map(); for (const span of allPins) { - const line = releaseLine(span.version); + const line = hasLines ? releaseLine(span.version) : SINGLE_LINE_KEY; const text = catalog.slice(span.start, span.end); const revisionMatch = PIN_REVISION.exec(text); const revision = revisionMatch === null ? -1 : Number(revisionMatch[1]); @@ -742,12 +762,17 @@ export function planSlimUpdates( if (match === null) continue; // not this service, or a legacy tag with no `-rN`: ignored. const upstream = match[1] as string; const revision = Number(match[2]); - const line = releaseLine(upstream); - if (!pinnedByLine.has(line)) { - console.log( - `::warning ::${service} ${upstream} is not on a release line ${CATALOG_PATH} carries for it; ignoring ${tag}.`, - ); - continue; + let line: string; + if (hasLines) { + line = releaseLine(upstream); + if (!pinnedByLine.has(line)) { + warnings.push( + `::warning ::${service} ${upstream} is not on a release line ${CATALOG_PATH} carries for it; ignoring ${tag}.`, + ); + continue; + } + } else { + line = SINGLE_LINE_KEY; } const list = candidatesByLine.get(line) ?? []; list.push({ upstream, revision }); @@ -782,7 +807,7 @@ export function planSlimUpdates( for (const upstream of highestRevisionByUpstream.keys()) { const comparedToPinned = compareVersions(upstream, pinned.upstream); if (comparedToPinned === undefined) { - console.log(`::warning ::${service} ${upstream} is not a comparable version; ignoring.`); + warnings.push(`::warning ::${service} ${upstream} is not a comparable version; ignoring.`); continue; } if (bestUpstream === undefined || (compareVersions(upstream, bestUpstream) ?? 0) > 0) { @@ -795,7 +820,22 @@ export function planSlimUpdates( } } - return updates; + return { updates, warnings }; +} + +/** + * Reads every committed release tag through `listReleaseTags` and plans against `catalog`. The + * injectable lister is the seam a dry run or an end-to-end test uses to exercise the whole + * `plan-updates` transport — this function plus the CLI's file/stdout wiring — without a network + * call, the same pattern `RevisionIo.listReleaseTags` already uses. + */ +export async function planUpdatesForService(input: { + readonly catalog: string; + readonly service: string; + readonly listReleaseTags: () => Promise>; +}): Promise { + const releaseTags = await input.listReleaseTags(); + return planSlimUpdates(input.catalog, input.service, releaseTags); } export interface CatalogRefreshResult { @@ -820,6 +860,9 @@ export async function refreshCatalogPin(input: { if (input.upstream !== undefined && input.release !== undefined) { throw new InvalidPayloadError("--upstream and --release are mutually exclusive."); } + if (input.upstream !== undefined && !UPSTREAM_VERSION_PATTERN.test(input.upstream)) { + throw new InvalidPayloadError(`invalid --upstream '${input.upstream}'`); + } let upstream = input.upstream; let requiredRevision: number | undefined; @@ -1050,12 +1093,18 @@ function updateLine(update: SlimUpdate): string { ); } +/** + * `plan-updates`' records go only into `--output `, never stdout: a caller (the workflow) + * reads warnings from stdout as `::warning ::…` lines, and would otherwise mistake one for a + * malformed record and abort a run that had valid updates alongside it. + */ async function runPlanUpdates(argv: ReadonlyArray): Promise { const flags = parseFlags(argv); const service = flags.get("service"); - if (service === undefined) { + const output = flags.get("output"); + if (service === undefined || output === undefined) { throw new InvalidPayloadError( - "Usage: sync-artifacts-catalog.ts plan-updates --service [--format lines]", + "Usage: sync-artifacts-catalog.ts plan-updates --service --output [--format lines]", ); } const format = flags.get("format") ?? "json"; @@ -1063,13 +1112,17 @@ async function runPlanUpdates(argv: ReadonlyArray): Promise { throw new InvalidPayloadError(`invalid --format '${format}' (expected 'json' or 'lines')`); } const catalog = await Bun.file(CATALOG_PATH).text(); - const releaseTags = await listReleaseTags(); - const updates = planSlimUpdates(catalog, service, releaseTags); - if (format === "lines") { - for (const update of updates) console.log(updateLine(update)); - return; - } - console.log(JSON.stringify(updates)); + const { updates, warnings } = await planUpdatesForService({ + catalog, + service, + listReleaseTags, + }); + for (const warning of warnings) console.log(warning); + const content = + format === "lines" + ? updates.map((update) => `${updateLine(update)}\n`).join("") + : `${JSON.stringify(updates)}\n`; + await Bun.write(output, content); } async function runValidatePayload(argv: ReadonlyArray): Promise { From f5bb72e0c8219e88911c761491260be7445441e7 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 17:42:25 +0200 Subject: [PATCH 27/45] fix(ci): pass GITHUB_TOKEN and read the plan on its own fd in the apply step The apply step set GH_TOKEN for gh and git but not GITHUB_TOKEN, so the committed-release check behind --release listed releases anonymously. Also read the plan file on fd 3 instead of stdin, since every command the loop body runs (bun, gh, git) otherwise shares stdin with the read and any of them consuming a byte would swallow the remaining records; and name the release a PR actually pins as its body's subject, only mentioning the dispatch's triggering release separately, since the two can differ. Wires the plan-updates step to the new required --output flag. --- .github/workflows/slim-release-published.yml | 24 ++++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/slim-release-published.yml b/.github/workflows/slim-release-published.yml index ed004a0474..6c414216d2 100644 --- a/.github/workflows/slim-release-published.yml +++ b/.github/workflows/slim-release-published.yml @@ -79,8 +79,10 @@ jobs: # Reconciles the service against every committed slim-services release (`planSlimUpdates`), # not just the release that triggered this dispatch — this run is idempotent and safe to - # receive out of order. Written to a file, not a $GITHUB_OUTPUT multi-line value: each line - # is field-separated with `\x1f` (see `updateLine`), which a later step reads directly. + # receive out of order. Records go only to `--output` (never stdout): each line is + # field-separated with `\x1f` (see `updateLine`), which a later step reads directly. Any + # `::warning ::…` the planner emits (an ignored tag, say) prints to this step's own stdout + # instead, so it can never be mistaken for a malformed record. - name: Plan updates id: plan env: @@ -89,7 +91,7 @@ jobs: run: | set -euo pipefail bun .github/scripts/sync-artifacts-catalog.ts plan-updates --service "$SERVICE" --format lines \ - > "${RUNNER_TEMP}/slim-updates.tsv" + --output "${RUNNER_TEMP}/slim-updates.tsv" count="$(wc -l < "${RUNNER_TEMP}/slim-updates.tsv" | tr -d ' ')" echo "count=${count}" >>"$GITHUB_OUTPUT" @@ -143,6 +145,7 @@ jobs: if: steps.plan.outputs.count != '0' env: GH_TOKEN: ${{ steps.app-token.outputs.token }} + GITHUB_TOKEN: ${{ github.token }} SERVICE: ${{ steps.validate.outputs.service }} RELEASE_VERSION: ${{ steps.validate.outputs.release_version }} BASE_SHA: ${{ steps.base.outputs.sha }} @@ -151,7 +154,10 @@ jobs: set -euo pipefail manual_prefix="bun .github/scripts/sync-artifacts-catalog.ts --service ${SERVICE} --release" - while IFS=$'\x1f' read -r kind branch title release from; do + # Read the records on fd 3, not stdin: every command the loop body runs (`bun`, `gh`, + # `git`) otherwise shares stdin with the `read`, and any of them consuming a byte of it + # would swallow the rest of the file's records. + while IFS=$'\x1f' read -r -u 3 kind branch title release from; do if [ -z "$kind" ] || [ -z "$branch" ] || [ -z "$title" ] || [ -z "$release" ] || [ -z "$from" ]; then echo "::error ::Malformed plan-updates line: kind='${kind}' branch='${branch}' title='${title}' release='${release}' from='${from}'." exit 1 @@ -186,8 +192,12 @@ jobs: else action="pins" fi - body="$(printf 'supabase/slim-services published %s. This %s %s from %s to %s.\n\nhttps://github.com/supabase/slim-services/releases/tag/%s-%s\n\nThis branch is rewritten from %s whenever a newer relevant release arrives.\n' \ - "$RELEASE_VERSION" "$action" "$SERVICE" "$from" "$release" "$SERVICE" "$release" "$DEFAULT_BRANCH")" + # Names the release this PR actually pins as the subject; the dispatch's triggering + # release (RELEASE_VERSION) can differ from it (a hotfix dispatch commonly also + # yields an unrelated upgrade on the same line), so it's only mentioned, not implied + # to be what changed. + body="$(printf 'This %s %s from %s to %s.\n\nhttps://github.com/supabase/slim-services/releases/tag/%s-%s\n\nPlanned after supabase/slim-services published %s. This branch is rewritten from %s whenever a newer relevant release arrives.\n' \ + "$action" "$SERVICE" "$from" "$release" "$SERVICE" "$release" "$RELEASE_VERSION" "$DEFAULT_BRANCH")" existing="$(gh pr list --head "$branch" --base "$DEFAULT_BRANCH" --state open --json number --jq '.[0].number // empty')" if [ -n "$existing" ]; then @@ -204,4 +214,4 @@ jobs: echo "::error ::Pushed ${branch} but failed to open a pull request. Run manually: ${manual_prefix} ${release}, then open a pull request from ${branch} against ${DEFAULT_BRANCH} by hand." exit 1 fi - done < "${RUNNER_TEMP}/slim-updates.tsv" + done 3< "${RUNNER_TEMP}/slim-updates.tsv" From 62ee1dc685ee78bd5115c0fb15684bf42d6c17f8 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 17:42:32 +0200 Subject: [PATCH 28/45] docs(ci): correct kong and pg14's update ownership The ADR and the dependabot.yml comment listed kong among the images Dependabot keeps bumping, but library/kong is in dependabot's ignore list. Kong and pg14 (supabase/postgres) have no slim build and are bumped by hand; Dependabot bumps only the images that remain genuinely upstream-only: migra, pg_prove, pgadmin-schema-diff. --- .github/dependabot.yml | 5 +++-- docs/adr/0026-slim-artifact-mirrors.md | 25 +++++++++++++++---------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3e97bc0c96..b66df12186 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -87,8 +87,9 @@ updates: - patch # These lines are generated from packages/stack/src/Artifacts.ts (the single version table # for slim-capable services); updates to them arrive through slim-release-published.yml, not - # Dependabot. Dependabot keeps bumping only the upstream-only images the Dockerfile still - # hand-manages: kong, and the job images (migra, pg_prove, pgadmin-schema-diff). + # Dependabot. Kong and pg14 (supabase/postgres) have no slim build either, but are bumped by + # hand, not by Dependabot. Dependabot keeps bumping only the images that remain genuinely + # upstream-only: migra, pg_prove, pgadmin-schema-diff. ignore: - dependency-name: "library/kong" - dependency-name: "axllent/mailpit" diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index e304a5d9c4..b56be1a060 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -49,8 +49,10 @@ stack, the Go CLI embed, `mirror-template-images.yml`, `detect-unmirrored-images but for a slim-capable alias its `FROM` line is now a **generated view** of the catalog's `ArtifactPin.upstreamImage`: `apps/cli/scripts/render-service-dockerfile.ts` rewrites those lines in place, and a CI check (`render-service-dockerfile.unit.test.ts`) fails on drift between the -Dockerfile and the catalog. Kong, Postgres 14, and the one-shot job images (migra, pg_prove, -pgadmin-schema-diff) have no slim build and stay hand- or Dependabot-managed, as today. +Dockerfile and the catalog. Kong and Postgres 14 have no slim build and are bumped by hand (both +share `library/kong`'s or `supabase/postgres`'s Dependabot `ignore` entry — see "Tradeoffs" below). +The one-shot job images (migra, pg_prove, pgadmin-schema-diff) also have no slim build, but are +upstream-only: they are the images Dependabot still bumps. ### Content pins, not runtime checksums @@ -152,14 +154,17 @@ or considered stale and backfilled — never rewritten in place. ### Tradeoffs of moving updates onto the catalog - Upstream and security fixes for slim-capable images now reach the CLI only once - slim-services publishes a release. Direct Dependabot discovery is gone for them; Dependabot - still bumps only the upstream-only images `.github/dependabot.yml`'s docker `ignore` list - excludes (kong, and the job images: migra, pg_prove, pgadmin-schema-diff). -- Dependabot's 7-day cooldown no longer governs those images — hotfix and upgrade PRs land as - soon as `slim-release-published` fires, on whatever cadence slim-services publishes. -- `pg14` has no slim build and was already upstream-only; it is bumped by hand (Dependabot's - `docker` ecosystem also ignores `supabase/postgres` entirely now, since Dependabot cannot tell - `pg14`'s `FROM` line apart from `pg`'s and `pg15`'s by repository name alone). + slim-services publishes a release. Direct Dependabot discovery is gone for them; Dependabot's + `docker` ecosystem `ignore` list (`.github/dependabot.yml`) now excludes every slim-capable + image plus `library/kong` and `supabase/postgres`, so it bumps only the three images that + remain genuinely upstream-only: migra, pg_prove, pgadmin-schema-diff. +- Dependabot's 7-day cooldown no longer governs the excluded images — hotfix and upgrade PRs for + slim-capable ones land as soon as `slim-release-published` fires, on whatever cadence + slim-services publishes; kong and `pg14` are simply bumped by hand. +- `pg14` has no slim build and was already upstream-only; it is bumped by hand alongside kong, + not by Dependabot — Dependabot's `docker` ecosystem ignores `supabase/postgres` entirely now, + since it cannot tell `pg14`'s `FROM` line apart from `pg`'s and `pg15`'s by repository name + alone. - The Deno 1 edge-runtime override (`apps/cli/src/shared/functions/functions.shared.ts`, and the Go `deno1` constant) stays a separately pinned, upstream-only exception — it never goes through the catalog. From e2244ac8109bd908ec79c848980dbaac0a7ccd9b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 17:42:43 +0200 Subject: [PATCH 29/45] chore(cli): describe toSlimImage's fallback for the single-table model These comments explained the upstream/slim mismatch as Dependabot bumping the Dockerfile ahead of the catalog, which no longer happens: the Dockerfile's slim-capable lines are generated from the catalog now, so they never disagree with it. Comment-only; no behavior change. --- apps/cli/src/shared/services/services.shared.unit.test.ts | 7 ++++--- apps/cli/src/shared/services/slim-images.ts | 7 ++++--- apps/cli/src/shared/services/slim-images.unit.test.ts | 8 +++++--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/apps/cli/src/shared/services/services.shared.unit.test.ts b/apps/cli/src/shared/services/services.shared.unit.test.ts index ef4bdc2be5..71f41344fb 100644 --- a/apps/cli/src/shared/services/services.shared.unit.test.ts +++ b/apps/cli/src/shared/services/services.shared.unit.test.ts @@ -18,9 +18,10 @@ import { // `v2.197.0-r0`, with a realistic (non-placeholder) fixture digest built to the real // `ArtifactPin`/`NativePin` shape from `@supabase/stack/internal/artifacts`. Every other service // is deliberately absent, so `toSlimImage` falls through to the upstream image for them — the -// expected state until Dependabot catches the Dockerfile up. `vi.mock` factories are hoisted -// above every other top-level statement, so the fixture is inlined rather than referencing an -// outer const. +// permanent state for a non-slim-capable alias (kong, `pg14`, the job images): the Dockerfile's +// slim-capable lines are generated from the catalog now, so they never disagree with it. +// `vi.mock` factories are hoisted above every other top-level statement, so the fixture is +// inlined rather than referencing an outer const. vi.mock("@supabase/stack/internal/artifacts", () => { const digest = "260e94edb8d402555791146fcf70b8e90efdc6a81877a04e5aa26f0f416a5dd7"; const nativePin = { archive: digest, manifest: digest }; diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index f99ea85ce3..f2be119f10 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -118,9 +118,10 @@ function catalogImageFor(service: SlimServiceName, upstreamVersion: string): str * This owns tag normalization (`v`-prefixing, `tagPrefix`, vector's `-alpine` * strip) via {@link slimCatalogPin}, so pins that differ only in prefix * between the two registries (`supavisor`, `logflare`) still match. Returns - * `undefined` when no catalog entry's `upstreamVersion` matches — including - * when Dependabot has bumped the Dockerfile ahead of the catalog — so callers - * keep the upstream (non-slim) image instead of guessing a slim tag. + * `undefined` when no catalog entry's `upstreamVersion` matches — the catalog is the single + * version table now, so this only happens for an alias with no slim build (kong, `pg14`, the + * one-shot job images) — so callers keep the upstream (non-slim) image instead of guessing a + * slim tag. */ export function toSlimImage(alias: string, image: string): string | undefined { const pin = slimCatalogPin(alias, image); diff --git a/apps/cli/src/shared/services/slim-images.unit.test.ts b/apps/cli/src/shared/services/slim-images.unit.test.ts index 9122588ec3..2c70a084c6 100644 --- a/apps/cli/src/shared/services/slim-images.unit.test.ts +++ b/apps/cli/src/shared/services/slim-images.unit.test.ts @@ -15,9 +15,11 @@ import { // Only `auth` is pinned in this fixture catalog, at `v2.197.0-r0`, with a realistic // (non-placeholder) fixture digest built to the real `ArtifactPin`/`NativePin` shape from // `@supabase/stack/internal/artifacts`. Every other service is deliberately absent, so -// `toSlimImage` falls through to the upstream image for them — the expected state until -// Dependabot catches the Dockerfile up. `vi.mock` factories are hoisted above every other -// top-level statement, so the fixture is inlined rather than referencing an outer const. +// `toSlimImage` falls through to the upstream image for them — the permanent state for a +// non-slim-capable alias (kong, `pg14`, the job images): the Dockerfile's slim-capable lines +// are generated from the catalog now, so they never disagree with it. `vi.mock` factories are +// hoisted above every other top-level statement, so the fixture is inlined rather than +// referencing an outer const. vi.mock("@supabase/stack/internal/artifacts", () => { const digest = "d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c"; const nativePin = { archive: digest, manifest: digest }; From 480b070151cf3b0336203b5edfb229aa1ba0f3f8 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 18:09:52 +0200 Subject: [PATCH 30/45] test(cli): make release-asset, shadow-stack and sweep tests robust under load --- .../sweep-live-projects.integration.test.ts | 24 ++++++++++++++++--- .../upload-release-assets.integration.test.ts | 20 +++++++++------- .../stack-shadow.integration.test.ts | 1 + 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/apps/cli/scripts/sweep-live-projects.integration.test.ts b/apps/cli/scripts/sweep-live-projects.integration.test.ts index ac124dce97..1b368a6e23 100644 --- a/apps/cli/scripts/sweep-live-projects.integration.test.ts +++ b/apps/cli/scripts/sweep-live-projects.integration.test.ts @@ -6,6 +6,11 @@ import path from "node:path"; const script = path.resolve(import.meta.dirname, "sweep-live-projects.sh"); const directories: string[] = []; +// The describe budget below; the hang safety net fires well before it so vitest +// still has time to report a clear failure instead of a bare test-timeout. +const TEST_BUDGET_MS = 40_000; +const HANG_KILL_MS = TEST_BUDGET_MS - 10_000; + afterEach(async () => { await Promise.all(directories.splice(0).map((directory) => rm(directory, { recursive: true }))); }); @@ -73,13 +78,22 @@ async function runSweep(scenario: Scenario) { stdout: "pipe", stderr: "pipe", }); - const timeout = setTimeout(() => child.kill(), 10_000); + let hung = false; + const timeout = setTimeout(() => { + hung = true; + child.kill(); + }, HANG_KILL_MS); const [exitCode, stdout, stderr] = await Promise.all([ child.exited, new Response(child.stdout).text(), new Response(child.stderr).text(), ]); clearTimeout(timeout); + if (hung) { + throw new Error( + `sweep script hung: killed after ${HANG_KILL_MS}ms without exiting (stdout: ${stdout}, stderr: ${stderr})`, + ); + } return { exitCode, stdout, stderr, deletes }; } finally { await server.stop(true); @@ -88,7 +102,10 @@ async function runSweep(scenario: Scenario) { const active = (ref: string, name = `e2e-${ref}`) => ({ ref, name, status: "ACTIVE" }); -describe.skipIf(process.platform === "win32")("sweep-live-projects.sh", { timeout: 40_000 }, () => { +describe.skipIf(process.platform === "win32")( + "sweep-live-projects.sh", + { timeout: TEST_BUDGET_MS }, + () => { test("accepts refused deletion when a fresh authenticated list shows absence", async () => { const result = await runSweep({ lists: [[active("gone")], []], @@ -159,4 +176,5 @@ describe.skipIf(process.platform === "win32")("sweep-live-projects.sh", { timeou "delete retry completed for project flaky (HTTP 204 request delete-flaky)", ); }); -}); + }, +); diff --git a/apps/cli/scripts/upload-release-assets.integration.test.ts b/apps/cli/scripts/upload-release-assets.integration.test.ts index 1866370852..699cae4bf1 100644 --- a/apps/cli/scripts/upload-release-assets.integration.test.ts +++ b/apps/cli/scripts/upload-release-assets.integration.test.ts @@ -91,20 +91,22 @@ describe("upload-release-assets against a fake gh", () => { await uploadAssets("v1.0.0", [asset], io, { maxAttempts: 2, - timeoutMs: 500, + timeoutMs: 5_000, backoffMs: () => 0, }); - expect(Date.now() - startedAt).toBeLessThan(10_000); + const elapsed = Date.now() - startedAt; + expect(elapsed).toBeGreaterThanOrEqual(5_000); + expect(elapsed).toBeLessThan(25_000); expect(await calls()).toEqual([ `release upload v1.0.0 ${asset.path} --clobber`, `release upload v1.0.0 ${asset.path} --clobber`, ]); expect(logs).toEqual([ - `Upload of ${asset.name} failed on attempt 1 (timed out after 0.5s); retrying in 0s.`, + `Upload of ${asset.name} failed on attempt 1 (timed out after 5s); retrying in 0s.`, `Uploaded ${asset.name} (attempt 2).`, ]); - }, 20_000); + }, 40_000); test("kills an upload that ignores SIGTERM once the grace period passes", async () => { const { directory, env, calls } = await fakeGhOnPath(); @@ -114,18 +116,18 @@ describe("upload-release-assets against a fake gh", () => { await uploadAssets("v1.0.0", [asset], io, { maxAttempts: 2, - timeoutMs: 500, + timeoutMs: 5_000, backoffMs: () => 0, }); const elapsed = Date.now() - startedAt; - expect(elapsed).toBeGreaterThanOrEqual(500 + KILL_GRACE_MS); - expect(elapsed).toBeLessThan(20_000); + expect(elapsed).toBeGreaterThanOrEqual(5_000 + KILL_GRACE_MS); + expect(elapsed).toBeLessThan(25_000); expect(await calls()).toHaveLength(2); expect(logs[0]).toBe( - `Upload of ${asset.name} failed on attempt 1 (timed out after 0.5s); retrying in 0s.`, + `Upload of ${asset.name} failed on attempt 1 (timed out after 5s); retrying in 0s.`, ); - }, 30_000); + }, 40_000); test("surfaces gh's stderr for a failed attempt and retries it", async () => { const { directory, env, calls } = await fakeGhOnPath(); diff --git a/apps/cli/src/command-internal/stack-shadow.integration.test.ts b/apps/cli/src/command-internal/stack-shadow.integration.test.ts index 8fe9849ca3..7b1b7c78bf 100644 --- a/apps/cli/src/command-internal/stack-shadow.integration.test.ts +++ b/apps/cli/src/command-internal/stack-shadow.integration.test.ts @@ -511,5 +511,6 @@ describe("stack shadow databases", () => { /Warning: Docker was unavailable, so Docker resources for shadow stack [0-9a-f]{64} were not removed\. Once it is running, remove them with:\n {2}docker rm --force \$\(docker ps --all --quiet\)\n/u, ); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + 120_000, ); }); From 6cc74dbe3149c43bf8c2f6b0b0cb649580e7331c Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 18:11:16 +0200 Subject: [PATCH 31/45] style(cli): format the load-hardened integration tests --- .../sweep-live-projects.integration.test.ts | 118 +++++++++--------- .../stack-shadow.integration.test.ts | 104 +++++++-------- 2 files changed, 114 insertions(+), 108 deletions(-) diff --git a/apps/cli/scripts/sweep-live-projects.integration.test.ts b/apps/cli/scripts/sweep-live-projects.integration.test.ts index 1b368a6e23..1d8fe1b748 100644 --- a/apps/cli/scripts/sweep-live-projects.integration.test.ts +++ b/apps/cli/scripts/sweep-live-projects.integration.test.ts @@ -106,75 +106,79 @@ describe.skipIf(process.platform === "win32")( "sweep-live-projects.sh", { timeout: TEST_BUDGET_MS }, () => { - test("accepts refused deletion when a fresh authenticated list shows absence", async () => { - const result = await runSweep({ - lists: [[active("gone")], []], - deletes: { gone: { status: 403, body: { message: "already removed" } } }, + test("accepts refused deletion when a fresh authenticated list shows absence", async () => { + const result = await runSweep({ + lists: [[active("gone")], []], + deletes: { gone: { status: 403, body: { message: "already removed" } } }, + }); + expect(result.exitCode).toBe(0); + expect(result.deletes).toEqual(["gone"]); + expect(`${result.stdout}\n${result.stderr}`).not.toContain("test-token"); }); - expect(result.exitCode).toBe(0); - expect(result.deletes).toEqual(["gone"]); - expect(`${result.stdout}\n${result.stderr}`).not.toContain("test-token"); - }); - test("accepts a terminal status and converges after a stale list", async () => { - const result = await runSweep({ - lists: [[active("stale")], [active("stale")], [{ ...active("stale"), status: "REMOVED" }]], - deletes: { stale: { status: 202 } }, + test("accepts a terminal status and converges after a stale list", async () => { + const result = await runSweep({ + lists: [[active("stale")], [active("stale")], [{ ...active("stale"), status: "REMOVED" }]], + deletes: { stale: { status: 202 } }, + }); + expect(result.exitCode).toBe(0); }); - expect(result.exitCode).toBe(0); - }); - test("retries a transient read failure but rejects malformed evidence", async () => { - const transient = await runSweep({ - lists: [503, [active("retry")], []], - deletes: { retry: { status: 202 } }, - }); - expect(transient.exitCode).toBe(0); + test("retries a transient read failure but rejects malformed evidence", async () => { + const transient = await runSweep({ + lists: [503, [active("retry")], []], + deletes: { retry: { status: 202 } }, + }); + expect(transient.exitCode).toBe(0); - const malformed = await runSweep({ lists: [{ projects: [] }], deletes: {} }); - expect(malformed.exitCode).not.toBe(0); - expect(malformed.deletes).toEqual([]); - }); + const malformed = await runSweep({ lists: [{ projects: [] }], deletes: {} }); + expect(malformed.exitCode).not.toBe(0); + expect(malformed.deletes).toEqual([]); + }); - test("fails terminal listing errors without retrying or deleting", async () => { - const forbidden = await runSweep({ lists: [403, []], deletes: {} }); - expect(forbidden.exitCode).not.toBe(0); - expect(forbidden.deletes).toEqual([]); + test("fails terminal listing errors without retrying or deleting", async () => { + const forbidden = await runSweep({ lists: [403, []], deletes: {} }); + expect(forbidden.exitCode).not.toBe(0); + expect(forbidden.deletes).toEqual([]); - const unavailable = await runSweep({ lists: [503, 503, 503], deletes: {} }); - expect(unavailable.exitCode).not.toBe(0); - expect(unavailable.deletes).toEqual([]); + const unavailable = await runSweep({ lists: [503, 503, 503], deletes: {} }); + expect(unavailable.exitCode).not.toBe(0); + expect(unavailable.deletes).toEqual([]); - const malformedReconciliation = await runSweep({ - lists: [[active("bad-evidence")], { projects: [] }], - deletes: { "bad-evidence": { status: 403 } }, + const malformedReconciliation = await runSweep({ + lists: [[active("bad-evidence")], { projects: [] }], + deletes: { "bad-evidence": { status: 403 } }, + }); + expect(malformedReconciliation.exitCode).not.toBe(0); }); - expect(malformedReconciliation.exitCode).not.toBe(0); - }); - test("attempts every owned project and fails if one remains active", async () => { - const result = await runSweep({ - lists: [ - [active("stuck"), active("other"), { ref: "foreign", name: "unrelated", status: "ACTIVE" }], - [active("stuck"), { ref: "foreign", name: "unrelated", status: "ACTIVE" }], - ], - deletes: { stuck: { status: 403 }, other: { status: 204 } }, + test("attempts every owned project and fails if one remains active", async () => { + const result = await runSweep({ + lists: [ + [ + active("stuck"), + active("other"), + { ref: "foreign", name: "unrelated", status: "ACTIVE" }, + ], + [active("stuck"), { ref: "foreign", name: "unrelated", status: "ACTIVE" }], + ], + deletes: { stuck: { status: 403 }, other: { status: 204 } }, + }); + expect(result.exitCode).not.toBe(0); + expect(result.deletes).toEqual(["stuck", "other"]); }); - expect(result.exitCode).not.toBe(0); - expect(result.deletes).toEqual(["stuck", "other"]); - }); - test("retries a transient deletion after fresh evidence still shows the project", async () => { - const result = await runSweep({ - lists: (deletes) => (deletes.length >= 2 ? [] : [active("flaky")]), - deletes: {}, - deleteStatuses: { flaky: [503, 204] }, + test("retries a transient deletion after fresh evidence still shows the project", async () => { + const result = await runSweep({ + lists: (deletes) => (deletes.length >= 2 ? [] : [active("flaky")]), + deletes: {}, + deleteStatuses: { flaky: [503, 204] }, + }); + expect(result.exitCode).toBe(0); + expect(result.deletes).toEqual(["flaky", "flaky"]); + expect(result.stderr).toContain( + "delete retry completed for project flaky (HTTP 204 request delete-flaky)", + ); }); - expect(result.exitCode).toBe(0); - expect(result.deletes).toEqual(["flaky", "flaky"]); - expect(result.stderr).toContain( - "delete retry completed for project flaky (HTTP 204 request delete-flaky)", - ); - }); }, ); diff --git a/apps/cli/src/command-internal/stack-shadow.integration.test.ts b/apps/cli/src/command-internal/stack-shadow.integration.test.ts index 7b1b7c78bf..ca7dae1a4b 100644 --- a/apps/cli/src/command-internal/stack-shadow.integration.test.ts +++ b/apps/cli/src/command-internal/stack-shadow.integration.test.ts @@ -457,60 +457,62 @@ describe("stack shadow databases", () => { 120_000, ); - it.live("prints the cleanup commands when a shadow's destroy skips its engine", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-skipped-" }); - const output = mockOutput(); - const cleanupCommand = "docker rm --force $(docker ps --all --quiet)"; - // A real registration whose database creation fails fast and whose destroy reports skipped cleanup. - const api = Layer.effect( - StackApi, - Effect.gen(function* () { - const real = yield* StackApi; - return StackApi.of({ - ...real, - create: (options) => - real.create(options).pipe( - Effect.map((stack) => ({ - ...stack, - services: { - ...stack.services, - create: () => - Effect.fail(new StackError({ operation: "create", message: "injected" })), - }, - destroy: Effect.succeed({ - runtimeCleanup: "skipped", - engine: "docker", - cleanupCommands: [cleanupCommand], - } as const), - })), - ), - }); - }), - ).pipe(Layer.provide(stackApiLayer), Layer.provide(BunServices.layer)); + it.live( + "prints the cleanup commands when a shadow's destroy skips its engine", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-skipped-" }); + const output = mockOutput(); + const cleanupCommand = "docker rm --force $(docker ps --all --quiet)"; + // A real registration whose database creation fails fast and whose destroy reports skipped cleanup. + const api = Layer.effect( + StackApi, + Effect.gen(function* () { + const real = yield* StackApi; + return StackApi.of({ + ...real, + create: (options) => + real.create(options).pipe( + Effect.map((stack) => ({ + ...stack, + services: { + ...stack.services, + create: () => + Effect.fail(new StackError({ operation: "create", message: "injected" })), + }, + destroy: Effect.succeed({ + runtimeCleanup: "skipped", + engine: "docker", + cleanupCommands: [cleanupCommand], + } as const), + })), + ), + }); + }), + ).pipe(Layer.provide(stackApiLayer), Layer.provide(BunServices.layer)); - yield* stackWithShadowDatabase(input(fs, path, root), () => Effect.void, { - runtime: "native", - }).pipe( - Effect.provide( - Layer.mergeAll( - api, - stackCatalogSetupLayer, - dbConnectionLayer, - runtimeInfoLayer, - mockCommandSettings({ workdir: root, supabaseHome: root }), - output.layer, + yield* stackWithShadowDatabase(input(fs, path, root), () => Effect.void, { + runtime: "native", + }).pipe( + Effect.provide( + Layer.mergeAll( + api, + stackCatalogSetupLayer, + dbConnectionLayer, + runtimeInfoLayer, + mockCommandSettings({ workdir: root, supabaseHome: root }), + output.layer, + ), ), - ), - Effect.flip, - ); + Effect.flip, + ); - expect(output.stderrText).toMatch( - /Warning: Docker was unavailable, so Docker resources for shadow stack [0-9a-f]{64} were not removed\. Once it is running, remove them with:\n {2}docker rm --force \$\(docker ps --all --quiet\)\n/u, - ); - }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + expect(output.stderrText).toMatch( + /Warning: Docker was unavailable, so Docker resources for shadow stack [0-9a-f]{64} were not removed\. Once it is running, remove them with:\n {2}docker rm --force \$\(docker ps --all --quiet\)\n/u, + ); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), 120_000, ); }); From dfb00525e15fd0ecbd5a8e1ba80555677d5160ee Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 18:19:47 +0200 Subject: [PATCH 32/45] test(cli): budget sweep tests by their sweep count --- .../sweep-live-projects.integration.test.ts | 76 +++++++++++-------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/apps/cli/scripts/sweep-live-projects.integration.test.ts b/apps/cli/scripts/sweep-live-projects.integration.test.ts index 1d8fe1b748..faaaac4c2b 100644 --- a/apps/cli/scripts/sweep-live-projects.integration.test.ts +++ b/apps/cli/scripts/sweep-live-projects.integration.test.ts @@ -6,10 +6,12 @@ import path from "node:path"; const script = path.resolve(import.meta.dirname, "sweep-live-projects.sh"); const directories: string[] = []; -// The describe budget below; the hang safety net fires well before it so vitest -// still has time to report a clear failure instead of a bare test-timeout. -const TEST_BUDGET_MS = 40_000; -const HANG_KILL_MS = TEST_BUDGET_MS - 10_000; +// Per-sweep hang guard: a spawned run gets this long before the safety net kills +// it. A test's own timeout must cover every sweep it runs plus report margin, so +// derive it from the sweep count rather than a single shared budget. +const SWEEP_GUARD_MS = 30_000; +const REPORT_MARGIN_MS = 10_000; +const sweepBudget = (sweeps: number) => sweeps * SWEEP_GUARD_MS + REPORT_MARGIN_MS; afterEach(async () => { await Promise.all(directories.splice(0).map((directory) => rm(directory, { recursive: true }))); @@ -82,7 +84,7 @@ async function runSweep(scenario: Scenario) { const timeout = setTimeout(() => { hung = true; child.kill(); - }, HANG_KILL_MS); + }, SWEEP_GUARD_MS); const [exitCode, stdout, stderr] = await Promise.all([ child.exited, new Response(child.stdout).text(), @@ -91,7 +93,7 @@ async function runSweep(scenario: Scenario) { clearTimeout(timeout); if (hung) { throw new Error( - `sweep script hung: killed after ${HANG_KILL_MS}ms without exiting (stdout: ${stdout}, stderr: ${stderr})`, + `sweep script hung: killed after ${SWEEP_GUARD_MS}ms without exiting (stdout: ${stdout}, stderr: ${stderr})`, ); } return { exitCode, stdout, stderr, deletes }; @@ -104,7 +106,7 @@ const active = (ref: string, name = `e2e-${ref}`) => ({ ref, name, status: "ACTI describe.skipIf(process.platform === "win32")( "sweep-live-projects.sh", - { timeout: TEST_BUDGET_MS }, + { timeout: sweepBudget(1) }, () => { test("accepts refused deletion when a fresh authenticated list shows absence", async () => { const result = await runSweep({ @@ -124,33 +126,41 @@ describe.skipIf(process.platform === "win32")( expect(result.exitCode).toBe(0); }); - test("retries a transient read failure but rejects malformed evidence", async () => { - const transient = await runSweep({ - lists: [503, [active("retry")], []], - deletes: { retry: { status: 202 } }, - }); - expect(transient.exitCode).toBe(0); - - const malformed = await runSweep({ lists: [{ projects: [] }], deletes: {} }); - expect(malformed.exitCode).not.toBe(0); - expect(malformed.deletes).toEqual([]); - }); - - test("fails terminal listing errors without retrying or deleting", async () => { - const forbidden = await runSweep({ lists: [403, []], deletes: {} }); - expect(forbidden.exitCode).not.toBe(0); - expect(forbidden.deletes).toEqual([]); - - const unavailable = await runSweep({ lists: [503, 503, 503], deletes: {} }); - expect(unavailable.exitCode).not.toBe(0); - expect(unavailable.deletes).toEqual([]); + test( + "retries a transient read failure but rejects malformed evidence", + async () => { + const transient = await runSweep({ + lists: [503, [active("retry")], []], + deletes: { retry: { status: 202 } }, + }); + expect(transient.exitCode).toBe(0); - const malformedReconciliation = await runSweep({ - lists: [[active("bad-evidence")], { projects: [] }], - deletes: { "bad-evidence": { status: 403 } }, - }); - expect(malformedReconciliation.exitCode).not.toBe(0); - }); + const malformed = await runSweep({ lists: [{ projects: [] }], deletes: {} }); + expect(malformed.exitCode).not.toBe(0); + expect(malformed.deletes).toEqual([]); + }, + sweepBudget(2), + ); + + test( + "fails terminal listing errors without retrying or deleting", + async () => { + const forbidden = await runSweep({ lists: [403, []], deletes: {} }); + expect(forbidden.exitCode).not.toBe(0); + expect(forbidden.deletes).toEqual([]); + + const unavailable = await runSweep({ lists: [503, 503, 503], deletes: {} }); + expect(unavailable.exitCode).not.toBe(0); + expect(unavailable.deletes).toEqual([]); + + const malformedReconciliation = await runSweep({ + lists: [[active("bad-evidence")], { projects: [] }], + deletes: { "bad-evidence": { status: 403 } }, + }); + expect(malformedReconciliation.exitCode).not.toBe(0); + }, + sweepBudget(3), + ); test("attempts every owned project and fails if one remains active", async () => { const result = await runSweep({ From 677b087455c3deabe4196ac55e18a7ef4ddf79b2 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 18:21:05 +0200 Subject: [PATCH 33/45] test(ci): cover the plan-updates output transport The prior test only checked planSlimUpdates's returned arrays; nothing exercised the plan-updates CLI mode's own file/stdout wiring, which is where the original P1 bug (a warning corrupting the records file) actually lived. runPlanUpdates now takes an injectable listReleaseTags, the same seam RevisionIo already uses, so a test can drive it directly: given an ignored tag alongside a valid update, the --output file gets exactly the valid record and the warning reaches stdout. A missing --output exits non-zero before any network call. --- .../scripts/sync-artifacts-catalog.test.ts | 61 +++++++++++++++++++ .github/scripts/sync-artifacts-catalog.ts | 13 +++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index 48cbaa2b80..d782575858 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -9,6 +9,7 @@ import { planSlimUpdates, planUpdatesForService, refreshCatalogPin, + runPlanUpdates, validateSlimReleasePublishedPayload, type RevisionIo, } from "./sync-artifacts-catalog.ts"; @@ -515,6 +516,66 @@ describe("planUpdatesForService (the plan-updates transport's IO seam)", () => { }); }); +describe("runPlanUpdates (the actual plan-updates CLI mode, not just the pure planner)", () => { + test("an ignored tag plus a valid update: --output gets exactly the valid record, and the warning reaches stdout", async () => { + // Real catalog, real "auth" pin — `io.listReleaseTags` is the seam this CLI mode injects, so + // this exercises its own file/stdout wiring (P1) without a network call. + const catalog = await Bun.file(CATALOG_PATH).text(); + const pinMatch = + /definition\(\s*"auth",\s*\{\s*upstreamVersion:\s*"([^"]+)",\s*revision:\s*(\d+)/.exec( + catalog, + ); + if (pinMatch === null) throw new Error("auth pin not found in the real catalog"); + const pinnedUpstream = pinMatch[1] as string; + const pinnedRevision = Number(pinMatch[2]); + const hotfixRelease = `${pinnedUpstream}-r${pinnedRevision + 1}`; + + const dir = await mkdtemp(join(tmpdir(), "plan-updates-cli-")); + const outputPath = join(dir, "slim-updates.tsv"); + const logs: string[] = []; + const originalLog = console.log; + let content: string; + try { + console.log = (...args: unknown[]) => { + logs.push(args.map(String).join(" ")); + }; + // No exception reaching past this call is this mode's own success condition — `main()`'s + // wrapper only calls `process.exit(1)` when `runPlanUpdates` rejects, so resolving here is + // the in-process analogue of "the exit status is 0". + await runPlanUpdates(["--service", "auth", "--format", "lines", "--output", outputPath], { + listReleaseTags: async () => [ + `auth-${hotfixRelease}`, + `auth-${pinnedUpstream}-orioledb-r0`, // ignored: not a comparable version + ], + }); + content = await readFile(outputPath, "utf8"); + } finally { + console.log = originalLog; + await rm(dir, { recursive: true, force: true }); + } + + expect(logs).toEqual([ + `::warning ::auth ${pinnedUpstream}-orioledb is not a comparable version; ignoring.`, + ]); + expect(content).toBe( + `hotfix\x1fslim-hotfix/auth\x1fchore(stack): pin auth ${hotfixRelease}\x1f${hotfixRelease}\x1f${pinnedUpstream}-r${pinnedRevision}\n`, + ); + }); + + test("a missing --output exits non-zero before any network call", async () => { + const proc = Bun.spawn( + ["bun", ".github/scripts/sync-artifacts-catalog.ts", "plan-updates", "--service", "auth"], + { stdout: "pipe", stderr: "pipe" }, + ); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + + expect(exitCode).toBe(1); + expect(stdout).toContain( + "Usage: sync-artifacts-catalog.ts plan-updates --service --output ", + ); + }); +}); + describe("refreshCatalogPin", () => { test("refreshes a service to the highest revision of its currently pinned upstream", async () => { const digests = nativeDigests("g"); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index a3ad81bedc..506cb03856 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -1097,8 +1097,17 @@ function updateLine(update: SlimUpdate): string { * `plan-updates`' records go only into `--output `, never stdout: a caller (the workflow) * reads warnings from stdout as `::warning ::…` lines, and would otherwise mistake one for a * malformed record and abort a run that had valid updates alongside it. + * + * `io.listReleaseTags` defaults to the real network lister but is overridable — the minimal seam + * a test uses to exercise this CLI mode's own file/stdout wiring (not just the pure planner) + * without a network call, the same pattern `RevisionIo` already uses. The `--service`/`--output` + * usage check runs before either the catalog read or the lister, so a test can also assert this + * mode fails fast on a missing flag without touching the network. */ -async function runPlanUpdates(argv: ReadonlyArray): Promise { +export async function runPlanUpdates( + argv: ReadonlyArray, + io: { readonly listReleaseTags: () => Promise> } = { listReleaseTags }, +): Promise { const flags = parseFlags(argv); const service = flags.get("service"); const output = flags.get("output"); @@ -1115,7 +1124,7 @@ async function runPlanUpdates(argv: ReadonlyArray): Promise { const { updates, warnings } = await planUpdatesForService({ catalog, service, - listReleaseTags, + listReleaseTags: io.listReleaseTags, }); for (const warning of warnings) console.log(warning); const content = From 403accd47e581c39fd9d678e3dfde0f3267671ac Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 18:21:12 +0200 Subject: [PATCH 34/45] docs(cli): state toSlimImage's fallback contract The comment said the fallback "only happens for an alias with no slim build", but toSlimImage also returns undefined for an OrioleDB tag (an alias that does have a slim build) and for a tag the catalog simply doesn't pin. State the general contract instead: undefined whenever no catalog pin matches the alias and tag, with all three cases as examples. Comment-only; no behavior change. --- apps/cli/src/shared/services/slim-images.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index f2be119f10..401c130ddc 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -117,11 +117,13 @@ function catalogImageFor(service: SlimServiceName, upstreamVersion: string): str * digest) whose `upstreamVersion` normalizes to `image`'s tag for `alias`. * This owns tag normalization (`v`-prefixing, `tagPrefix`, vector's `-alpine` * strip) via {@link slimCatalogPin}, so pins that differ only in prefix - * between the two registries (`supavisor`, `logflare`) still match. Returns - * `undefined` when no catalog entry's `upstreamVersion` matches — the catalog is the single - * version table now, so this only happens for an alias with no slim build (kong, `pg14`, the - * one-shot job images) — so callers keep the upstream (non-slim) image instead of guessing a - * slim tag. + * between the two registries (`supavisor`, `logflare`) still match. Returns `undefined` whenever + * no catalog pin matches `alias` and `image`'s tag — callers then keep the upstream (non-slim) + * image instead of guessing a slim tag. That covers more than "no slim build": an alias with no + * slim build at all (kong, `pg14`, the one-shot job images); an excluded tag on an alias that + * does have one (an OrioleDB `pg` tag, which {@link slimCatalogPin} always excludes); and a tag + * the catalog simply doesn't pin (an upstream version the catalog hasn't caught up to yet, or a + * hosted-project override from `supabase link` that doesn't match the pinned upstream version). */ export function toSlimImage(alias: string, image: string): string | undefined { const pin = slimCatalogPin(alias, image); From 5d2da47626e3a04d82a6cf3c8d780c9364c2a643 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 18:51:23 +0200 Subject: [PATCH 35/45] fix(ci): wait for a dispatched release before planning catalog updates Guards slim-release-published against a releases API that hasn't caught up with the release that triggered the dispatch: plan-updates now requires --expect-release to be listed before planning, retrying a bounded number of times first. Also drops the dead placeholderPin parsing path now that the catalog only ever holds resolved pins, and stops embedding raw dispatch values in error messages so a crafted payload can't inject workflow commands. --- .../scripts/sync-artifacts-catalog.test.ts | 265 +++++++++++++++--- .github/scripts/sync-artifacts-catalog.ts | 170 ++++++++--- .github/workflows/slim-release-published.yml | 12 +- 3 files changed, 358 insertions(+), 89 deletions(-) diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index d782575858..b621fe5b66 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -9,8 +9,8 @@ import { planSlimUpdates, planUpdatesForService, refreshCatalogPin, - runPlanUpdates, validateSlimReleasePublishedPayload, + waitForExpectedRelease, type RevisionIo, } from "./sync-artifacts-catalog.ts"; @@ -72,33 +72,54 @@ async function formatWithOxfmt(source: string): Promise { } } +/** + * A resolved `ArtifactPin` literal, for fixtures that exercise refreshing an already-resolved + * catalog entry to a new revision (design B: a committed catalog is never an unresolved + * placeholder). `seed` keys a real, distinct digest via the module's `digest` helper. + */ +function resolvedPinLiteral( + service: string, + version: string, + revision: number, + seed: string, +): string { + return `{ upstreamVersion: "${version}", revision: ${revision}, image: "ghcr.io/supabase/cli/${service}:${version}-r${revision}@${digest(seed)}", natives: {} }`; +} + +const postgres17Pin = resolvedPinLiteral("postgres", "17.6.1.168", 0, "postgres-17"); +/** + * The default pin's `image` field, quoted. A formatter may move a long property value onto its + * own line, but never breaks a string literal's contents — so this survives real formatting the + * way asserting the whole (potentially re-wrapped) `postgres17Pin` literal would not. + */ +const postgres17Image = `"ghcr.io/supabase/cli/postgres:17.6.1.168-r0@${digest("postgres-17")}"`; + const fixture = `const workloadCatalog = { database: definition( "postgres", - placeholderPin("postgres", "17.6.1.168"), + ${postgres17Pin}, "bin/supabase-postgres-start", ["bin/supabase-postgres-start"], - { "15.14.1.168": placeholderPin("postgres", "15.14.1.168") }, + { "15.14.1.168": ${resolvedPinLiteral("postgres", "15.14.1.168", 0, "postgres-15")} }, ), - rest: definition("postgrest", placeholderPin("postgrest", "v16.2"), "bin/postgrest"), - storage: definition("storage", placeholderPin("storage", "v1.73.0"), "bin/storage", ["bin/storage"]), + rest: definition("postgrest", ${resolvedPinLiteral("postgrest", "v16.2", 0, "postgrest")}, "bin/postgrest"), + storage: definition("storage", ${resolvedPinLiteral("storage", "v1.73.0", 0, "storage")}, "bin/storage", ["bin/storage"]), analytics: definition( "analytics", - placeholderPin("analytics", "v1.50.9"), + ${resolvedPinLiteral("analytics", "v1.50.9", 0, "analytics")}, "bin/logflare", ), }; `; const vectorFixture = `const workloadCatalog = { - vector: definition("vector", placeholderPin("vector", "0.53.0"), "bin/vector"), + vector: definition("vector", ${resolvedPinLiteral("vector", "0.53.0", 0, "vector")}, "bin/vector"), }; `; /** - * Fully resolved (not `placeholderPin`) fixtures for `planSlimUpdates`: a committed catalog - * always carries resolved pins (see the design doc's one-time alignment), so these give every - * pin a real revision instead of the `-1` an unresolved placeholder would carry. + * Fixtures for `planSlimUpdates`: a committed catalog always carries resolved pins, so these + * give every pin a real revision. */ const plannerFixture = `const workloadCatalog = { rest: definition( @@ -161,6 +182,25 @@ describe("validateSlimReleasePublishedPayload", () => { ).toThrow(InvalidPayloadError); }); + test("a newline-bearing service value produces a single-line error", () => { + let caught: unknown; + try { + validateSlimReleasePublishedPayload({ + service: "postgres\n::error ::injected", + upstream_version: "15.14.1.168", + revision: "0", + release_version: "15.14.1.168-r0", + }); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(InvalidPayloadError); + const message = (caught as InvalidPayloadError).message; + expect(message.split("\n")).toHaveLength(1); + expect(message).toContain(JSON.stringify("postgres\n::error ::injected")); + }); + test("accepts a Studio-style calendar-versioned upstream", () => { const payload = validateSlimReleasePublishedPayload({ service: "studio", @@ -516,50 +556,189 @@ describe("planUpdatesForService (the plan-updates transport's IO seam)", () => { }); }); +describe("waitForExpectedRelease (item A's eventual-consistency wait, call counts a subprocess can't assert)", () => { + test("visible on the first listing: returns immediately without waiting", async () => { + let calls = 0; + const waits: number[] = []; + const result = await waitForExpectedRelease( + "postgrest", + "v16.4-r1", + { + listReleaseTags: async () => { + calls += 1; + return ["postgrest-v16.4-r1"]; + }, + wait: async (ms) => { + waits.push(ms); + }, + }, + 6, + 10_000, + ); + + expect(result).toEqual({ visible: true, tags: ["postgrest-v16.4-r1"] }); + expect(calls).toBe(1); + expect(waits).toEqual([]); + }); + + test("visible on a later listing: re-lists and waits between attempts until it appears", async () => { + let calls = 0; + const waits: number[] = []; + const result = await waitForExpectedRelease( + "postgrest", + "v16.4-r1", + { + listReleaseTags: async () => { + calls += 1; + return calls < 3 ? ["postgrest-v16.4-r0"] : ["postgrest-v16.4-r0", "postgrest-v16.4-r1"]; + }, + wait: async (ms) => { + waits.push(ms); + }, + }, + 6, + 10_000, + ); + + expect(result).toEqual({ visible: true, tags: ["postgrest-v16.4-r0", "postgrest-v16.4-r1"] }); + expect(calls).toBe(3); + expect(waits).toEqual([10_000, 10_000]); + }); + + test("never visible: exhausts every bounded attempt, waiting between but not after the last", async () => { + let calls = 0; + const waits: number[] = []; + const result = await waitForExpectedRelease( + "postgrest", + "v16.4-r1", + { + listReleaseTags: async () => { + calls += 1; + return ["postgrest-v16.4-r0"]; + }, + wait: async (ms) => { + waits.push(ms); + }, + }, + 3, + 10, + ); + + expect(result).toEqual({ visible: false, tags: ["postgrest-v16.4-r0"] }); + expect(calls).toBe(3); + expect(waits).toEqual([10, 10]); + }); +}); + describe("runPlanUpdates (the actual plan-updates CLI mode, not just the pure planner)", () => { - test("an ignored tag plus a valid update: --output gets exactly the valid record, and the warning reaches stdout", async () => { - // Real catalog, real "auth" pin — `io.listReleaseTags` is the seam this CLI mode injects, so - // this exercises its own file/stdout wiring (P1) without a network call. + test("an ignored tag, a valid upgrade, and the expected dispatched release: --output gets exactly the valid record, the warning reaches stdout, and the process exits 0", async () => { + // Real catalog, real "postgrest" pin — a local fixture server stands in for the releases API + // (`SLIM_SERVICES_RELEASES_API`), so this exercises the real subprocess: the CLI's argv + // parsing, `--expect-release`'s visibility wait, the network lister, and the file/stdout + // wiring together, not just the pure planner or an in-process stub. const catalog = await Bun.file(CATALOG_PATH).text(); const pinMatch = - /definition\(\s*"auth",\s*\{\s*upstreamVersion:\s*"([^"]+)",\s*revision:\s*(\d+)/.exec( + /definition\(\s*"postgrest",\s*\{\s*upstreamVersion:\s*"([^"]+)",\s*revision:\s*(\d+)/.exec( catalog, ); - if (pinMatch === null) throw new Error("auth pin not found in the real catalog"); + if (pinMatch === null) throw new Error("postgrest pin not found in the real catalog"); const pinnedUpstream = pinMatch[1] as string; const pinnedRevision = Number(pinMatch[2]); - const hotfixRelease = `${pinnedUpstream}-r${pinnedRevision + 1}`; - - const dir = await mkdtemp(join(tmpdir(), "plan-updates-cli-")); + // Higher than any real pinned postgrest version, so it always plans as an upgrade. + const dispatchedRelease = "v999.0-r0"; + + const server = Bun.serve({ + port: 0, + fetch: () => + Response.json([ + { tag_name: `postgrest-${pinnedUpstream}-orioledb-r0`, draft: false }, // ignored + { tag_name: `postgrest-${dispatchedRelease}`, draft: false }, // the valid upgrade + ]), + }); + const dir = await mkdtemp(join(tmpdir(), "plan-updates-subprocess-")); const outputPath = join(dir, "slim-updates.tsv"); - const logs: string[] = []; - const originalLog = console.log; - let content: string; try { - console.log = (...args: unknown[]) => { - logs.push(args.map(String).join(" ")); - }; - // No exception reaching past this call is this mode's own success condition — `main()`'s - // wrapper only calls `process.exit(1)` when `runPlanUpdates` rejects, so resolving here is - // the in-process analogue of "the exit status is 0". - await runPlanUpdates(["--service", "auth", "--format", "lines", "--output", outputPath], { - listReleaseTags: async () => [ - `auth-${hotfixRelease}`, - `auth-${pinnedUpstream}-orioledb-r0`, // ignored: not a comparable version + const proc = Bun.spawn( + [ + "bun", + ".github/scripts/sync-artifacts-catalog.ts", + "plan-updates", + "--service", + "postgrest", + "--format", + "lines", + "--expect-release", + dispatchedRelease, + "--output", + outputPath, ], - }); - content = await readFile(outputPath, "utf8"); + { + stdout: "pipe", + stderr: "pipe", + env: { + ...globalThis.process.env, + SLIM_SERVICES_RELEASES_API: `http://127.0.0.1:${server.port}`, + }, + }, + ); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + + expect(exitCode).toBe(0); + expect(stdout).toContain( + `::warning ::postgrest ${pinnedUpstream}-orioledb is not a comparable version; ignoring.`, + ); + const content = await readFile(outputPath, "utf8"); + expect(content).toBe( + `upgrade\x1fslim-bump/postgrest\x1fchore(stack): bump postgrest to ${dispatchedRelease}\x1f${dispatchedRelease}\x1f${pinnedUpstream}-r${pinnedRevision}\n`, + ); } finally { - console.log = originalLog; + await server.stop(true); await rm(dir, { recursive: true, force: true }); } + }); - expect(logs).toEqual([ - `::warning ::auth ${pinnedUpstream}-orioledb is not a comparable version; ignoring.`, - ]); - expect(content).toBe( - `hotfix\x1fslim-hotfix/auth\x1fchore(stack): pin auth ${hotfixRelease}\x1f${hotfixRelease}\x1f${pinnedUpstream}-r${pinnedRevision}\n`, - ); + test("the expected dispatched release never becomes visible: the process exits non-zero with an actionable error, and writes no output file", async () => { + const server = Bun.serve({ + port: 0, + fetch: () => Response.json([{ tag_name: "postgrest-v16.4-r0", draft: false }]), + }); + const dir = await mkdtemp(join(tmpdir(), "plan-updates-subprocess-never-")); + const outputPath = join(dir, "slim-updates.tsv"); + try { + const proc = Bun.spawn( + [ + "bun", + ".github/scripts/sync-artifacts-catalog.ts", + "plan-updates", + "--service", + "postgrest", + "--format", + "lines", + "--expect-release", + "v999.0-r0", + "--output", + outputPath, + ], + { + stdout: "pipe", + stderr: "pipe", + env: { + ...globalThis.process.env, + SLIM_SERVICES_RELEASES_API: `http://127.0.0.1:${server.port}`, + SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS: "5", + }, + }, + ); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + + expect(exitCode).toBe(1); + expect(stdout).toContain("::error ::"); + expect(stdout).toContain("postgrest-v999.0-r0 is not visible yet"); + await expect(readFile(outputPath, "utf8")).rejects.toThrow(); + } finally { + await server.stop(true); + await rm(dir, { recursive: true, force: true }); + } }); test("a missing --output exits non-zero before any network call", async () => { @@ -668,7 +847,7 @@ describe("refreshCatalogPin", () => { target: "additional", }); // The default (17.x) postgres line is untouched. - expect(written.source).toContain('placeholderPin("postgres", "17.6.1.168")'); + expect(written.source).toContain(postgres17Image); const formatted = await formatWithOxfmt(written.source); const nextDigests = nativeDigests("o"); @@ -693,7 +872,7 @@ describe("refreshCatalogPin", () => { previousVersion: "15.14.1.168", target: "additional", }); - expect(refreshed.source).toContain('placeholderPin("postgres", "17.6.1.168")'); + expect(refreshed.source).toContain(postgres17Image); expect(refreshed.source).toContain(nextDigests["linux-arm64"].manifest); }); @@ -723,7 +902,7 @@ describe("refreshCatalogPin", () => { expect(written.source).not.toContain('"15.14.1.168"'); expect(written.source).toContain('"15.19.0.002": { upstreamVersion: "15.19.0.002"'); // The default (17.x) postgres line is untouched. - expect(written.source).toContain('placeholderPin("postgres", "17.6.1.168")'); + expect(written.source).toContain(postgres17Image); // The renamed key resolves the entry again after real formatting, e.g. for a later hotfix. const formatted = await formatWithOxfmt(written.source); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index 506cb03856..aad6b9e514 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -25,7 +25,11 @@ * another when a caller redirects stdout separately from the records file. * * bun .github/scripts/sync-artifacts-catalog.ts plan-updates --service \ - * --output [--format lines] + * --output [--format lines] [--expect-release -r] + * + * `--expect-release` handles the releases API lagging behind the dispatch that triggered this + * run: before planning, the listed committed tags must include `-`, or + * this mode re-lists a bounded number of times before giving up (`waitForExpectedRelease`). * * Validate-payload mode (used by the same workflow, before anything else): * checks an untrusted `slim-release-published` dispatch payload against @@ -54,9 +58,19 @@ const SLIM_IMAGE_PREFIX = `${SOURCE_REGISTRY}/`; const NATIVE_TARGETS = ["darwin-arm64", "linux-amd64", "linux-arm64"] as const; type NativeTargetName = (typeof NATIVE_TARGETS)[number]; -const RELEASES_API = "https://api.github.com/repos/supabase/slim-services/releases"; +/** Overridable so an integration test can point at a local fixture server instead of GitHub. */ +const RELEASES_API = + process.env.SLIM_SERVICES_RELEASES_API ?? + "https://api.github.com/repos/supabase/slim-services/releases"; const RELEASE_DOWNLOAD_BASE = "https://github.com/supabase/slim-services/releases/download"; +/** Bounded retry for `waitForExpectedRelease`: 6 attempts, 10s apart, by default. */ +const EXPECT_RELEASE_ATTEMPTS = 6; +/** Overridable so an integration test can shrink the real delay between attempts. */ +const EXPECT_RELEASE_INTERVAL_MS = Number( + process.env.SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS ?? 10_000, +); + /** Leading numeric component, `v` stripped. Only postgres carries more than one line. */ function releaseLine(version: string): string { const withoutPrefix = version.replace(/^[vV]/, ""); @@ -138,7 +152,7 @@ export type RevisionResolution = /** * Network ports the resolver needs: the release list (for revision allocation), a release's * `SHA256SUMS` body, the GHCR manifest digest, and a byte source's sha256. Tests stub these - * directly, the same way the previous `publication` callback was stubbed. + * directly. */ export interface RevisionIo { /** Tags of every published, non-draft release — a draft is not yet a committed revision. */ @@ -179,7 +193,7 @@ function desiredImage(service: string, releaseVersion: string, digest: string): /** * Resolves `service`'s committed `-r` revision, pinned by content: the GHCR * manifest digest, and every native target's archive and manifest sha256, cross-checked against - * the S3 mirror copy. See module docs for the five-step protocol. + * the S3 mirror copy. * * With `requiredRevision` given, that exact revision must already be committed — this is what * pins exactly a planned release (`--release`), never "highest at apply time". Without it, the @@ -324,7 +338,9 @@ const IMAGE_REFERENCE_PATTERN = /** Validates a normalized `upstreamImage` before it's ever written to `Artifacts.ts`. */ function validateUpstreamImage(image: string, context: string): string { if (!IMAGE_REFERENCE_PATTERN.test(image)) { - throw new InvalidPayloadError(`${context} has an invalid upstreamImage '${image}'.`); + throw new InvalidPayloadError( + `${context} has an invalid upstreamImage ${JSON.stringify(image)}.`, + ); } return image; } @@ -384,7 +400,10 @@ async function resolveUpstreamImage( } if (values.size !== 1) { throw new InvalidPayloadError( - `${service}-${releaseVersion} manifests disagree on the upstream image: ${[...values].sort().join(", ")}.`, + `${service}-${releaseVersion} manifests disagree on the upstream image: ${[...values] + .sort() + .map((value) => JSON.stringify(value)) + .join(", ")}.`, ); } return validateUpstreamImage([...values][0] as string, `${service}-${releaseVersion}`); @@ -445,29 +464,18 @@ function scanBalanced(source: string, openIndex: number, open: string, close: st throw new InvalidPayloadError(`unterminated '${open}' while parsing ${CATALOG_PATH}`); } -const PLACEHOLDER_PREFIX = /placeholderPin\s*\(/y; -/** Loosely finds `placeholderPin`'s second (version) argument anywhere inside its call text. */ -const PLACEHOLDER_VERSION = /"[a-z0-9-]+"\s*,\s*"([^"]+)"/; /** Loosely finds `upstreamVersion` anywhere inside a resolved pin literal's text. */ const PIN_UPSTREAM_VERSION = /upstreamVersion:\s*"([^"]+)"/; -/** Loosely finds `revision` anywhere inside a resolved pin literal's text. Absent for a `placeholderPin`. */ +/** Loosely finds `revision` anywhere inside a resolved pin literal's text. */ const PIN_REVISION = /revision:\s*(\d+)/; /** - * Matches a pin expression (B1's `placeholderPin(...)` call, or a resolved `ArtifactPin` object - * literal) starting exactly at `index`. The span is found by bracket balance, then the version is - * pulled out with a loose field search — so a formatter's whitespace, line breaks, trailing - * commas, or property order never break matching, only the two literal shapes themselves would. + * Matches a resolved `ArtifactPin` object literal starting exactly at `index`. The span is found + * by bracket balance, then the version is pulled out with a loose field search — so a formatter's + * whitespace, line breaks, trailing commas, or property order never break matching, only the + * literal shape itself would. */ function matchPinAt(source: string, index: number): PinSpan | undefined { - PLACEHOLDER_PREFIX.lastIndex = index; - const placeholderPrefix = PLACEHOLDER_PREFIX.exec(source); - if (placeholderPrefix !== null) { - const openParen = index + placeholderPrefix[0].length - 1; - const end = scanBalanced(source, openParen, "(", ")"); - const version = PLACEHOLDER_VERSION.exec(source.slice(index, end))?.[1]; - return version === undefined ? undefined : { start: index, end, version }; - } if (source[index] === "{") { const end = scanBalanced(source, index, "{", "}"); const version = PIN_UPSTREAM_VERSION.exec(source.slice(index, end))?.[1]; @@ -582,22 +590,26 @@ export function validateSlimReleasePublishedPayload(input: { readonly release_version: string; }): SlimReleasePublishedPayload { if (!SERVICE_NAME_PATTERN.test(input.service)) { - throw new InvalidPayloadError(`invalid service: '${input.service}'`); + throw new InvalidPayloadError(`invalid service: ${JSON.stringify(input.service)}`); } if (!UPSTREAM_VERSION_PATTERN.test(input.upstream_version)) { - throw new InvalidPayloadError(`invalid upstream_version: '${input.upstream_version}'`); + throw new InvalidPayloadError( + `invalid upstream_version: ${JSON.stringify(input.upstream_version)}`, + ); } if (!/^(0|[1-9][0-9]*)$/.test(input.revision)) { - throw new InvalidPayloadError(`invalid revision: '${input.revision}'`); + throw new InvalidPayloadError(`invalid revision: ${JSON.stringify(input.revision)}`); } if (!RELEASE_VERSION_PATTERN.test(input.release_version)) { - throw new InvalidPayloadError(`invalid release_version: '${input.release_version}'`); + throw new InvalidPayloadError( + `invalid release_version: ${JSON.stringify(input.release_version)}`, + ); } const revision = Number(input.revision); const expected = `${input.upstream_version}-r${revision}`; if (input.release_version !== expected) { throw new InvalidPayloadError( - `release_version '${input.release_version}' does not match derived '${expected}'`, + `release_version ${JSON.stringify(input.release_version)} does not match derived ${JSON.stringify(expected)}`, ); } return { @@ -672,17 +684,16 @@ function buildUpdate( toRevision: number, ): SlimUpdate { if (!SERVICE_NAME_PATTERN.test(service)) { - throw new InvalidPayloadError(`invalid service: '${service}'`); + throw new InvalidPayloadError(`invalid service: ${JSON.stringify(service)}`); } if (!UPSTREAM_VERSION_PATTERN.test(toUpstream)) { - throw new InvalidPayloadError(`invalid upstream version: '${toUpstream}'`); + throw new InvalidPayloadError(`invalid upstream version: ${JSON.stringify(toUpstream)}`); } const toRelease = `${toUpstream}-r${toRevision}`; if (!RELEASE_VERSION_PATTERN.test(toRelease)) { - throw new InvalidPayloadError(`invalid release version: '${toRelease}'`); + throw new InvalidPayloadError(`invalid release version: ${JSON.stringify(toRelease)}`); } - const fromRelease = - pinned.revision < 0 ? pinned.upstream : `${pinned.upstream}-r${pinned.revision}`; + const fromRelease = `${pinned.upstream}-r${pinned.revision}`; const suffix = hasLines ? `-${line}` : ""; const branch = kind === "hotfix" ? `slim-hotfix/${service}${suffix}` : `slim-bump/${service}${suffix}`; @@ -751,8 +762,12 @@ export function planSlimUpdates( const line = hasLines ? releaseLine(span.version) : SINGLE_LINE_KEY; const text = catalog.slice(span.start, span.end); const revisionMatch = PIN_REVISION.exec(text); - const revision = revisionMatch === null ? -1 : Number(revisionMatch[1]); - pinnedByLine.set(line, { upstream: span.version, revision }); + if (revisionMatch === null) { + throw new InvalidPayloadError( + `${CATALOG_PATH} has no revision for ${service} ${span.version}; a committed catalog always pins a resolved revision.`, + ); + } + pinnedByLine.set(line, { upstream: span.version, revision: Number(revisionMatch[1]) }); } const tagPattern = releaseTagPattern(service); @@ -838,6 +853,40 @@ export async function planUpdatesForService(input: { return planSlimUpdates(input.catalog, input.service, releaseTags); } +export interface WaitForExpectedReleaseResult { + readonly visible: boolean; + /** The last listing `listReleaseTags` returned, whichever attempt it came from. */ + readonly tags: ReadonlyArray; +} + +/** + * Eventual-consistency handling of the releases API lagging behind the `slim-release-published` + * dispatch that triggered this run: re-lists up to `attempts` times, `io.wait`-ing between + * attempts, until `-` appears. Not a test retry — a real, bounded wait + * for an external API to catch up. `io.wait` is the seam a unit test overrides to skip the real + * delay; an integration test instead shrinks `EXPECT_RELEASE_INTERVAL_MS` via + * `SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS`, since a subprocess can't be handed a function. + */ +export async function waitForExpectedRelease( + service: string, + releaseVersion: string, + io: { + readonly listReleaseTags: () => Promise>; + readonly wait: (ms: number) => Promise; + }, + attempts: number = EXPECT_RELEASE_ATTEMPTS, + intervalMs: number = EXPECT_RELEASE_INTERVAL_MS, +): Promise { + const expectedTag = `${service}-${releaseVersion}`; + let tags: ReadonlyArray = []; + for (let attempt = 0; attempt < attempts; attempt++) { + tags = await io.listReleaseTags(); + if (tags.includes(expectedTag)) return { visible: true, tags }; + if (attempt < attempts - 1) await io.wait(intervalMs); + } + return { visible: false, tags }; +} + export interface CatalogRefreshResult { readonly source: string; readonly update?: CatalogPinUpdate; @@ -1093,6 +1142,11 @@ function updateLine(update: SlimUpdate): string { ); } +/** Real delay, for `runPlanUpdates`' default `io.wait`. */ +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + /** * `plan-updates`' records go only into `--output `, never stdout: a caller (the workflow) * reads warnings from stdout as `::warning ::…` lines, and would otherwise mistake one for a @@ -1100,32 +1154,60 @@ function updateLine(update: SlimUpdate): string { * * `io.listReleaseTags` defaults to the real network lister but is overridable — the minimal seam * a test uses to exercise this CLI mode's own file/stdout wiring (not just the pure planner) - * without a network call, the same pattern `RevisionIo` already uses. The `--service`/`--output` - * usage check runs before either the catalog read or the lister, so a test can also assert this - * mode fails fast on a missing flag without touching the network. + * without a network call, the same pattern `RevisionIo` already uses. `io.wait` likewise defaults + * to a real delay but is overridable, the seam `waitForExpectedRelease`'s own unit tests use. The + * `--service`/`--output` usage check runs before either the catalog read or the lister, so a test + * can also assert this mode fails fast on a missing flag without touching the network. + * + * With `--expect-release -r` given, `--r` must be among the listed tags + * before planning runs at all — otherwise this dispatch's own release could be missing from a + * releases API that hasn't caught up yet, and the run would plan and pass quietly without it + * (`waitForExpectedRelease` handles the resulting eventual-consistency wait). Still missing after + * the bounded retries: this mode throws instead of planning, so `main()`'s handler reports an + * actionable `::error ::` and exits non-zero, and no `--output` file is written. */ export async function runPlanUpdates( argv: ReadonlyArray, - io: { readonly listReleaseTags: () => Promise> } = { listReleaseTags }, + io: { + readonly listReleaseTags: () => Promise>; + readonly wait?: (ms: number) => Promise; + } = { listReleaseTags }, ): Promise { const flags = parseFlags(argv); const service = flags.get("service"); const output = flags.get("output"); + const expectRelease = flags.get("expect-release"); if (service === undefined || output === undefined) { throw new InvalidPayloadError( - "Usage: sync-artifacts-catalog.ts plan-updates --service --output [--format lines]", + "Usage: sync-artifacts-catalog.ts plan-updates --service --output " + + "[--format lines] [--expect-release -r]", ); } const format = flags.get("format") ?? "json"; if (format !== "json" && format !== "lines") { throw new InvalidPayloadError(`invalid --format '${format}' (expected 'json' or 'lines')`); } + if (expectRelease !== undefined && !RELEASE_VERSION_PATTERN.test(expectRelease)) { + throw new InvalidPayloadError(`invalid --expect-release ${JSON.stringify(expectRelease)}`); + } + + let listReleaseTags = io.listReleaseTags; + if (expectRelease !== undefined) { + const waited = await waitForExpectedRelease(service, expectRelease, { + listReleaseTags: io.listReleaseTags, + wait: io.wait ?? sleep, + }); + if (!waited.visible) { + throw new InvalidPayloadError( + `${service}-${expectRelease} is not visible yet from the slim-services releases API; re-run this workflow once it has propagated.`, + ); + } + // Reuses the listing the successful attempt already fetched, instead of listing again. + listReleaseTags = async () => waited.tags; + } + const catalog = await Bun.file(CATALOG_PATH).text(); - const { updates, warnings } = await planUpdatesForService({ - catalog, - service, - listReleaseTags: io.listReleaseTags, - }); + const { updates, warnings } = await planUpdatesForService({ catalog, service, listReleaseTags }); for (const warning of warnings) console.log(warning); const content = format === "lines" diff --git a/.github/workflows/slim-release-published.yml b/.github/workflows/slim-release-published.yml index 6c414216d2..2db3112a5e 100644 --- a/.github/workflows/slim-release-published.yml +++ b/.github/workflows/slim-release-published.yml @@ -60,7 +60,9 @@ jobs: # field — including `upstream_version` and `release_version`, not just `service` and # `revision` — is checked against an anchored charset before it is ever written to # `$GITHUB_OUTPUT`, a branch name, or a PR title. A value that fails validation makes the - # script exit non-zero without printing anything. + # script exit non-zero and print a single `::error ::…` line to stdout (the rejected value + # is JSON-escaped in that message, so it can never smuggle in a newline or workflow command), + # which this step re-echoes so it still surfaces as an annotation. - name: Validate payload id: validate env: @@ -83,15 +85,21 @@ jobs: # field-separated with `\x1f` (see `updateLine`), which a later step reads directly. Any # `::warning ::…` the planner emits (an ignored tag, say) prints to this step's own stdout # instead, so it can never be mistaken for a malformed record. + # + # `--expect-release` guards against the releases API lagging behind this very dispatch: it + # requires `-` to be a listed tag before planning runs at all, + # re-listing a bounded number of times first (`waitForExpectedRelease`). Without it, a slow + # API would make this run plan without the release that triggered it, and pass quietly. - name: Plan updates id: plan env: SERVICE: ${{ steps.validate.outputs.service }} + RELEASE_VERSION: ${{ steps.validate.outputs.release_version }} GITHUB_TOKEN: ${{ github.token }} run: | set -euo pipefail bun .github/scripts/sync-artifacts-catalog.ts plan-updates --service "$SERVICE" --format lines \ - --output "${RUNNER_TEMP}/slim-updates.tsv" + --expect-release "$RELEASE_VERSION" --output "${RUNNER_TEMP}/slim-updates.tsv" count="$(wc -l < "${RUNNER_TEMP}/slim-updates.tsv" | tr -d ' ')" echo "count=${count}" >>"$GITHUB_OUTPUT" From 48091197681bb6069eface5d7fb8e8d902a46c0b Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 18:51:34 +0200 Subject: [PATCH 36/45] test(cli): assert slim image tests against the real catalog pin Several slim-image tests computed their expected value with toSlimImage itself, so they passed whether or not the catalog lookup actually resolved. Under the single version table, a default Dockerfile tag always matches a catalog pin, so these now read the expected ghcr.io image straight from catalogPins() and assert its shape, catching a regression to docker.io. Also has services.shared.ts reuse slim-images.ts's imageTag/imageRepository instead of re-implementing the @-aware tag split. --- .../db-bootstrap/pinned-image.unit.test.ts | 56 +++++++++++++------ .../command-internal/db-image.unit.test.ts | 42 +++++++++++--- .../edge-runtime-image.unit.test.ts | 27 +++++++-- .../start/start.services.unit.test.ts | 46 ++++++++++++--- .../start/start.slim-images.e2e.test.ts | 56 ++++++++++++++----- .../functions/functions.shared.unit.test.ts | 33 +++++++++-- .../src/shared/services/services.shared.ts | 15 ++--- apps/cli/src/shared/services/slim-images.ts | 7 +++ 8 files changed, 217 insertions(+), 65 deletions(-) diff --git a/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts b/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts index 1ecbd27145..69a6302da2 100644 --- a/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts +++ b/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts @@ -1,7 +1,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { catalogPins } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImageRaw } from "../../shared/services/dockerfile-images.ts"; -import { toSlimImage } from "../../shared/services/slim-images.ts"; +import { slimCatalogPin } from "../../shared/services/slim-images.ts"; import { resolvePinnedImage } from "./pinned-image.ts"; const currentTag = (alias: string) => dockerfileServiceImageRaw(alias).split(":")[1] ?? ""; @@ -12,6 +13,30 @@ const currentPoolerTag = currentTag("supavisor"); const currentPostgres = dockerfileServiceImageRaw("pg"); const currentPostgresTag = currentTag("pg"); +/** + * The catalog's own pinned image for `alias`'s (docker.io) `image` — read straight from + * `catalogPins()`, independent of `toSlimImage`, so a test asserting against this actually + * exercises the catalog lookup instead of passing whether or not it resolves (design B: a + * default Dockerfile tag always matches a catalog pin). Only reuses `slimCatalogPin` for alias + * and tag normalization (`v`-prefixing), not the catalog image lookup itself. + */ +function expectedPinnedImage(alias: string, image: string): string { + const pin = slimCatalogPin(alias, image); + if (pin === undefined) { + throw new Error(`no slim catalog pin for ${alias} ${image}`); + } + const entry = catalogPins().find( + (candidate) => + candidate.sourceService === pin.service && candidate.pin.upstreamVersion === pin.version, + ); + if (entry === undefined) { + throw new Error(`no catalog pin for ${pin.service} ${pin.version}`); + } + return entry.pin.image; +} + +const GHCR_SLIM_IMAGE_PATTERN = /^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/; + afterEach(() => { vi.unstubAllEnvs(); }); @@ -30,12 +55,10 @@ describe("resolvePinnedImage", () => { it("resolves slim images when the flag is on and the pin is current", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); - expect(resolvePinnedImage("gotrue", "auth", {})).toBe( - toSlimImage("gotrue", currentAuth) ?? currentAuth, - ); - expect(resolvePinnedImage("gotrue", "auth", { auth: currentAuthTag })).toBe( - toSlimImage("gotrue", currentAuth) ?? currentAuth, - ); + const pinned = expectedPinnedImage("gotrue", currentAuth); + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); + expect(resolvePinnedImage("gotrue", "auth", {})).toBe(pinned); + expect(resolvePinnedImage("gotrue", "auth", { auth: currentAuthTag })).toBe(pinned); }); it("keeps a historical pin on docker.io", () => { @@ -53,19 +76,16 @@ describe("resolvePinnedImage", () => { it("normalizes a current pooler pin onto the slim tag scheme", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); - expect(resolvePinnedImage("supavisor", "pooler", { pooler: currentPoolerTag })).toBe( - toSlimImage("supavisor", currentPooler) ?? currentPooler, - ); - // The opposite `v`-prefix variant of the same pin still counts as current (`pinMatchesCurrentImage` - // normalizes both before comparing), so it slim-translates the same way, or — with no catalog - // match — keeps this variant's own tag rather than the default Dockerfile one. + const pinned = expectedPinnedImage("supavisor", currentPooler); + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); + expect(resolvePinnedImage("supavisor", "pooler", { pooler: currentPoolerTag })).toBe(pinned); + // The opposite `v`-prefix variant of the same pin still counts as current + // (`pinMatchesCurrentImage` normalizes both before comparing), so it resolves to the very same + // catalog-pinned slim image. const altPoolerTag = currentPoolerTag.startsWith("v") ? currentPoolerTag.slice(1) : `v${currentPoolerTag}`; - const altPooler = currentPooler.replace(/:[^:]+$/, `:${altPoolerTag}`); - expect(resolvePinnedImage("supavisor", "pooler", { pooler: altPoolerTag })).toBe( - toSlimImage("supavisor", altPooler) ?? altPooler, - ); + expect(resolvePinnedImage("supavisor", "pooler", { pooler: altPoolerTag })).toBe(pinned); }); it("keeps a historical postgres pin on docker.io", () => { @@ -78,7 +98,7 @@ describe("resolvePinnedImage", () => { "supabase/postgres:17.4.1.1", ); expect(resolvePinnedImage("pg", "postgres", { postgres: currentPostgresTag })).toBe( - toSlimImage("pg", currentPostgres) ?? currentPostgres, + expectedPinnedImage("pg", currentPostgres), ); }); }); diff --git a/apps/cli/src/command-internal/db-image.unit.test.ts b/apps/cli/src/command-internal/db-image.unit.test.ts index 22fd3a96d8..0c00d93652 100644 --- a/apps/cli/src/command-internal/db-image.unit.test.ts +++ b/apps/cli/src/command-internal/db-image.unit.test.ts @@ -5,12 +5,13 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Path } from "effect"; import { afterEach, beforeEach, vi } from "vitest"; +import { catalogPins } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImage, dockerfileServiceImageRaw, } from "../shared/services/dockerfile-images.ts"; -import { imageTag, toSlimImage } from "../shared/services/slim-images.ts"; +import { imageTag } from "../shared/services/slim-images.ts"; import { resolveDbImage } from "./db-image.ts"; const currentPostgres = dockerfileServiceImageRaw("pg"); @@ -21,6 +22,25 @@ const pg15Image = dockerfileServiceImageRaw("pg15"); const pg15Tag = imageTag(pg15Image) ?? ""; const pg14Image = dockerfileServiceImageRaw("pg14"); +/** + * The catalog's own pinned image for postgres at `tag` — read straight from `catalogPins()`, + * independent of `toSlimImage`, so a test asserting against this actually exercises the catalog + * lookup instead of passing whether or not it resolves (design B: a default Dockerfile tag always + * matches a catalog pin). + */ +function expectedPinnedPostgresImage(tag: string): string { + const entry = catalogPins().find( + (candidate) => candidate.sourceService === "postgres" && candidate.pin.upstreamVersion === tag, + ); + if (entry === undefined) { + throw new Error(`no catalog pin for postgres ${tag}`); + } + return entry.pin.image; +} + +/** A regression to the docker.io fallback must fail an assertion built from this. */ +const GHCR_SLIM_IMAGE_PATTERN = /^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/; + const withTemp = () => mkdtempSync(join(tmpdir(), "db-image-")); const writePin = (workdir: string, pinned: string) => { @@ -112,16 +132,18 @@ describe("resolveDbImage", () => { }); }); - it.effect("rewrites the current PG15 fallback to the slim registry", () => { + it.effect("rewrites the current PG15 default tag to its catalog-pinned slim image", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const dir = withTemp(); + const pinned = expectedPinnedPostgresImage(pg15Tag); + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); return Effect.gen(function* () { expect(yield* resolve(dir, 15)).toEqual({ - image: toSlimImage("pg", pg15Image) ?? pg15Image, + image: pinned, configImage: pg15Image, }); expect(yield* resolve(dir, 13)).toEqual({ - image: toSlimImage("pg", pg15Image) ?? pg15Image, + image: pinned, configImage: pg15Image, }); rmSync(dir, { recursive: true, force: true }); @@ -141,13 +163,15 @@ describe("resolveDbImage", () => { }); }); - it.effect("rewrites a current PG15 pin to the slim registry", () => { + it.effect("rewrites a current PG15 pin to its catalog-pinned slim image", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const dir = withTemp(); writePin(dir, pg15Tag); + const pinned = expectedPinnedPostgresImage(pg15Tag); + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); return Effect.gen(function* () { expect(yield* resolve(dir, 15)).toEqual({ - image: toSlimImage("pg", pg15Image) ?? pg15Image, + image: pinned, configImage: pg15Image, }); rmSync(dir, { recursive: true, force: true }); @@ -167,13 +191,15 @@ describe("resolveDbImage", () => { }); }); - it.effect("rewrites the current Dockerfile pin to the slim registry", () => { + it.effect("rewrites the current Dockerfile pin to its catalog-pinned slim image", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const dir = withTemp(); writePin(dir, currentPostgresTag); + const pinned = expectedPinnedPostgresImage(currentPostgresTag); + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); return Effect.gen(function* () { expect(yield* resolve(dir, 17)).toEqual({ - image: toSlimImage("pg", currentPostgres) ?? currentPostgres, + image: pinned, configImage: currentPostgres, }); rmSync(dir, { recursive: true, force: true }); diff --git a/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts b/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts index 1ebb3f3326..a45ef53be5 100644 --- a/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts +++ b/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts @@ -5,17 +5,34 @@ import { BunServices } from "@effect/platform-bun"; import { afterEach, describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Path } from "effect"; import { vi } from "vitest"; +import { catalogPins } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImage, dockerfileServiceImageRaw, } from "../shared/services/dockerfile-images.ts"; -import { toSlimImage } from "../shared/services/slim-images.ts"; import { resolveEdgeRuntimeImage } from "./edge-runtime-image.ts"; const currentEdgeRuntime = dockerfileServiceImageRaw("edgeruntime"); const currentEdgeRuntimeTag = currentEdgeRuntime.split(":")[1] ?? ""; +/** + * The catalog's own pinned image for edge-runtime at `tag` — read straight from `catalogPins()`, + * independent of `toSlimImage`, so this test actually exercises the catalog lookup instead of + * passing whether or not it resolves (design B: a default Dockerfile tag always matches a + * catalog pin). + */ +function expectedPinnedEdgeRuntimeImage(tag: string): string { + const entry = catalogPins().find( + (candidate) => + candidate.sourceService === "edge-runtime" && candidate.pin.upstreamVersion === tag, + ); + if (entry === undefined) { + throw new Error(`no catalog pin for edge-runtime ${tag}`); + } + return entry.pin.image; +} + const resolve = (workdir: string, denoVersion: number) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -82,7 +99,7 @@ describe("resolveEdgeRuntimeImage", () => { ); }); - it.effect("rewrites the current Dockerfile pin onto the slim base", () => { + it.effect("rewrites the current Dockerfile pin to its catalog-pinned slim image", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "1"); const dir = mkdtempSync(join(tmpdir(), "edge-img-")); mkdirSync(join(dir, "supabase", ".temp"), { recursive: true }); @@ -90,12 +107,12 @@ describe("resolveEdgeRuntimeImage", () => { join(dir, "supabase", ".temp", "edge-runtime-version"), `${currentEdgeRuntimeTag}\n`, ); + const pinned = expectedPinnedEdgeRuntimeImage(currentEdgeRuntimeTag); + expect(pinned).toMatch(/^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/); return resolve(dir, 2).pipe( Effect.tap((image) => Effect.sync(() => { - expect(image).toBe( - toSlimImage("edgeruntime", currentEdgeRuntime) ?? currentEdgeRuntime, - ); + expect(image).toBe(pinned); rmSync(dir, { recursive: true, force: true }); }), ), diff --git a/apps/cli/src/commands/start/start.services.unit.test.ts b/apps/cli/src/commands/start/start.services.unit.test.ts index ea2a7e0535..7315a29371 100644 --- a/apps/cli/src/commands/start/start.services.unit.test.ts +++ b/apps/cli/src/commands/start/start.services.unit.test.ts @@ -1,10 +1,11 @@ import { CliConfigSchema, type CliConfig } from "@supabase/config"; import { Schema } from "effect"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { catalogPins } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImageRaw } from "../../shared/services/dockerfile-images.ts"; import type { LocalServiceVersionOverrides } from "../../shared/services/services.shared.ts"; -import { toSlimImage } from "../../shared/services/slim-images.ts"; +import { slimCatalogPin } from "../../shared/services/slim-images.ts"; import { serviceContainerIds, localDbContainerId } from "../../command-internal/docker-ids.ts"; import { SERVICE_CATALOG } from "../../command-internal/service-catalog.ts"; import { resolveStartGates, resolveStartImagePlan, type StartGates } from "./start.gates.ts"; @@ -16,6 +17,30 @@ const currentVector = dockerfileServiceImageRaw("vector"); const currentPooler = dockerfileServiceImageRaw("supavisor"); const currentPoolerTag = currentPooler.split(":")[1] ?? ""; +/** + * The catalog's own pinned image for `alias`'s (docker.io) `image` — read straight from + * `catalogPins()`, independent of `toSlimImage`, so a test asserting against this actually + * exercises the catalog lookup instead of passing whether or not it resolves (design B: a + * default Dockerfile tag always matches a catalog pin). Only reuses `slimCatalogPin` for alias + * and tag normalization, not the catalog image lookup itself. + */ +function expectedPinnedImage(alias: string, image: string): string { + const pin = slimCatalogPin(alias, image); + if (pin === undefined) { + throw new Error(`no slim catalog pin for ${alias} ${image}`); + } + const entry = catalogPins().find( + (candidate) => + candidate.sourceService === pin.service && candidate.pin.upstreamVersion === pin.version, + ); + if (entry === undefined) { + throw new Error(`no catalog pin for ${pin.service} ${pin.version}`); + } + return entry.pin.image; +} + +const GHCR_SLIM_IMAGE_PATTERN = /^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/; + describe("START_SERVICES", () => { it("has one row per SERVICE_CATALOG entry, in the catalog's startOrder", () => { expect(START_SERVICES).toHaveLength(SERVICE_CATALOG.length); @@ -249,13 +274,20 @@ describe("resolveStartImagePlan under SUPABASE_USE_SLIM_IMAGES", () => { it("plans slim images when the flag is on, keeping unmapped services on docker.io", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); - expect(imageFor("gotrue")).toBe(toSlimImage("gotrue", currentGotrue) ?? currentGotrue); - expect(imageFor("logflare")).toBe(toSlimImage("logflare", currentLogflare) ?? currentLogflare); - expect(imageFor("vector")).toBe(toSlimImage("vector", currentVector) ?? currentVector); - expect(imageFor("supavisor", { pooler: currentPoolerTag })).toBe( - toSlimImage("supavisor", currentPooler) ?? currentPooler, - ); + const gotruePinned = expectedPinnedImage("gotrue", currentGotrue); + const logflarePinned = expectedPinnedImage("logflare", currentLogflare); + const vectorPinned = expectedPinnedImage("vector", currentVector); + const poolerPinned = expectedPinnedImage("supavisor", currentPooler); + for (const pinned of [gotruePinned, logflarePinned, vectorPinned, poolerPinned]) { + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); + } + expect(imageFor("gotrue")).toBe(gotruePinned); + expect(imageFor("logflare")).toBe(logflarePinned); + expect(imageFor("vector")).toBe(vectorPinned); + expect(imageFor("supavisor", { pooler: currentPoolerTag })).toBe(poolerPinned); + // Deliberate fallback: a historical pin the catalog doesn't carry stays on docker.io. expect(imageFor("supavisor", { pooler: "2.0.0" })).toBe("supabase/supavisor:2.0.0"); + // Deliberate fallback: kong has no slim build at all. expect(imageFor("kong")).toBe("library/kong:2.8.1"); }); }); diff --git a/apps/cli/src/commands/start/start.slim-images.e2e.test.ts b/apps/cli/src/commands/start/start.slim-images.e2e.test.ts index 897e37df08..bd80bcfb5e 100644 --- a/apps/cli/src/commands/start/start.slim-images.e2e.test.ts +++ b/apps/cli/src/commands/start/start.slim-images.e2e.test.ts @@ -5,7 +5,7 @@ import { beforeAll, describe, expect, it } from "@effect/vitest"; import { catalogPins, type ServiceKind } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImageRaw } from "../../shared/services/dockerfile-images.ts"; -import { isSlimImageRef, toSlimImage } from "../../shared/services/slim-images.ts"; +import { isSlimImageRef, slimCatalogPin, toSlimImage } from "../../shared/services/slim-images.ts"; import { buildHealthCmdArg } from "../../command-internal/db-bootstrap/docker-create-args.ts"; import { slimWgetHealthcheck, @@ -102,18 +102,40 @@ const containerImage = Effect.fnUntraced(function* (name: string) { return stdout.trim(); }); +/** `kong` has no slim build at all, so this stays fallback-tolerant for the pull-ahead list. */ function expectedSlimImage(alias: string): string { const raw = dockerfileServiceImageRaw(alias); return toSlimImage(alias, raw) ?? raw; } +/** + * The catalog's own pinned image for `alias`'s current Dockerfile tag — read straight from + * `catalogPins()`, independent of `toSlimImage`. Design B guarantees every slim-capable alias's + * default Dockerfile tag matches a catalog pin (the Dockerfile is generated from the catalog), so + * — unlike `expectedSlimImage` above, which still needs to fall back for `kong` — this never + * falls back, and a regression to docker.io fails the assertion instead of passing quietly. + */ +function expectedPinnedImage(alias: string): string { + const raw = dockerfileServiceImageRaw(alias); + const pin = slimCatalogPin(alias, raw); + if (pin === undefined) { + throw new Error(`no slim catalog pin for ${alias} ${raw}`); + } + const entry = catalogPins().find( + (candidate) => + candidate.sourceService === pin.service && candidate.pin.upstreamVersion === pin.version, + ); + if (entry === undefined) { + throw new Error(`no catalog pin for ${pin.service} ${pin.version}`); + } + return entry.pin.image; +} + /** * The Dockerfile aliases whose spec builder switches its healthcheck on `usesSlimImageRuntime` - * (`*.service.ts`). Whether each one is actually running its slim image today depends on whether - * the Dockerfile's tag currently matches that service's catalog pin (`expectedSlimImage`) — - * Dependabot moves that independently for every service, so the assertions below always derive - * the expected healthcheck from the image actually resolved, never from an assumption that it's - * one family or the other. + * (`*.service.ts`). Every one of them is slim today (`expectedPinnedImage`), but the assertions + * below still derive the expected healthcheck from the image actually resolved, not from that + * assumption directly. */ type HealthcheckedAlias = "gotrue" | "storage" | "realtime"; @@ -321,17 +343,21 @@ describe("supabase start slim images (e2e)", () => { }); expect(start.exitCode, `stdout:\n${start.stdout}\nstderr:\n${start.stderr}`).toBe(0); - // Each service's expected image follows the Dockerfile: slim when its tag matches the - // catalog pin, upstream otherwise (`expectedSlimImage`). Today only auth, pgmeta, - // edge-runtime and pooler match — Dependabot moves the Dockerfile independently of the - // catalog, so which of these is slim today is not assumed anywhere below. - const authImage = expectedSlimImage("gotrue"); - const realtimeImage = expectedSlimImage("realtime"); - const storageImage = expectedSlimImage("storage"); + // Every alias here is slim-capable, and design B guarantees its default Dockerfile tag + // matches a catalog pin — so each expected image is read straight from the catalog + // (`expectedPinnedImage`), independent of `toSlimImage`. + const authImage = expectedPinnedImage("gotrue"); + const realtimeImage = expectedPinnedImage("realtime"); + const storageImage = expectedPinnedImage("storage"); + for (const image of [authImage, realtimeImage, storageImage]) { + expect(image).toMatch(/^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/); + } - expect(yield* containerImage(dbContainer)).toBe(expectedSlimImage("pg")); + expect(yield* containerImage(dbContainer)).toBe(expectedPinnedImage("pg")); expect(yield* containerImage(storageContainer)).toBe(storageImage); - expect(yield* containerImage(edgeRuntimeContainer)).toBe(expectedSlimImage("edgeruntime")); + expect(yield* containerImage(edgeRuntimeContainer)).toBe( + expectedPinnedImage("edgeruntime"), + ); expect(yield* containerImage(authContainer)).toBe(authImage); expect(yield* containerImage(realtimeContainer)).toBe(realtimeImage); diff --git a/apps/cli/src/shared/functions/functions.shared.unit.test.ts b/apps/cli/src/shared/functions/functions.shared.unit.test.ts index 2836cd12b9..63c3dd591b 100644 --- a/apps/cli/src/shared/functions/functions.shared.unit.test.ts +++ b/apps/cli/src/shared/functions/functions.shared.unit.test.ts @@ -1,8 +1,9 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { Effect } from "effect"; +import { catalogPins } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImageRaw } from "../services/dockerfile-images.ts"; -import { toSlimImage } from "../services/slim-images.ts"; +import { slimCatalogPin } from "../services/slim-images.ts"; import { DENO1_EDGE_RUNTIME_VERSION, edgeRuntimeImage, @@ -12,6 +13,28 @@ import { const rawEdgeRuntimeImage = dockerfileServiceImageRaw("edgeruntime"); const currentEdgeRuntimeTag = rawEdgeRuntimeImage.slice(rawEdgeRuntimeImage.lastIndexOf(":") + 1); +/** + * The catalog's own pinned image for `alias`'s (docker.io) `image` — read straight from + * `catalogPins()`, independent of `toSlimImage`, so a test asserting against this actually + * exercises the catalog lookup instead of passing whether or not it resolves (design B: a + * default Dockerfile tag always matches a catalog pin). Only reuses `slimCatalogPin` for alias + * and tag normalization, not the catalog image lookup itself. + */ +function expectedPinnedImage(alias: string, image: string): string { + const pin = slimCatalogPin(alias, image); + if (pin === undefined) { + throw new Error(`no slim catalog pin for ${alias} ${image}`); + } + const entry = catalogPins().find( + (candidate) => + candidate.sourceService === pin.service && candidate.pin.upstreamVersion === pin.version, + ); + if (entry === undefined) { + throw new Error(`no catalog pin for ${pin.service} ${pin.version}`); + } + return entry.pin.image; +} + afterEach(() => { vi.unstubAllEnvs(); }); @@ -24,11 +47,11 @@ describe("edgeRuntimeImage", () => { ); }); - it("rewrites the current Dockerfile tag onto the slim ghcr.io image when the flag is on", () => { + it("rewrites the current Dockerfile tag onto the catalog-pinned slim ghcr.io image when the flag is on", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); - expect(edgeRuntimeImage(currentEdgeRuntimeTag)).toBe( - toSlimImage("edgeruntime", rawEdgeRuntimeImage) ?? rawEdgeRuntimeImage, - ); + const pinned = expectedPinnedImage("edgeruntime", rawEdgeRuntimeImage); + expect(pinned).toMatch(/^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/); + expect(edgeRuntimeImage(currentEdgeRuntimeTag)).toBe(pinned); }); it("keeps a historical pin on docker.io when the flag is on", () => { diff --git a/apps/cli/src/shared/services/services.shared.ts b/apps/cli/src/shared/services/services.shared.ts index 20680bdc43..ccaf34a1e0 100644 --- a/apps/cli/src/shared/services/services.shared.ts +++ b/apps/cli/src/shared/services/services.shared.ts @@ -16,6 +16,8 @@ import { type DockerfileImageSpec, } from "./dockerfile-images.ts"; import { + imageRepository, + imageTag, replaceImageTag, slimImageForAlias, slimImageForCurrentPin, @@ -224,15 +226,14 @@ function toServiceVersionRow( service: ServiceImageSpec, remote: Partial> = {}, ): ServiceVersionRow { - // `@`-aware: a slim catalog pin's image carries a `@sha256:…` digest after the tag. - const withoutDigest = service.image.split("@")[0] ?? service.image; - const tagSeparator = withoutDigest.lastIndexOf(":"); - if (tagSeparator === -1) { + // `@`-aware (via `imageTag`/`imageRepository`): a slim catalog pin's image carries a + // `@sha256:…` digest after the tag. + const name = imageRepository(service.image); + const tag = imageTag(service.image); + if (name === undefined || tag === undefined) { throw new Error(`Invalid service image entry: ${service.image}`); } - - const name = withoutDigest.slice(0, tagSeparator); - const local = upstreamVersionFromTag(withoutDigest.slice(tagSeparator + 1)); + const local = upstreamVersionFromTag(tag); return { name, diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index 401c130ddc..68821ebe27 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -145,6 +145,13 @@ export function imageTag(image: string): string | undefined { return tagSeparator === -1 ? undefined : withoutDigest.slice(tagSeparator + 1); } +/** The repository portion of `image` (before the tag), the other half of `imageTag`'s split. */ +export function imageRepository(image: string): string | undefined { + const withoutDigest = image.split("@")[0] ?? image; + const tagSeparator = withoutDigest.lastIndexOf(":"); + return tagSeparator === -1 ? undefined : withoutDigest.slice(0, tagSeparator); +} + /** The `@sha256:…` digest suffix of `image`, if it carries one. */ export function imageDigest(image: string): string | undefined { const at = image.indexOf("@"); From 4884fcecd4196a3aeea65d63ded4b38f0c601123 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 18:59:14 +0200 Subject: [PATCH 37/45] test(cli): share the expected pinned image helper Six test files each carried a near-identical "look up the catalog's pinned image for this alias" helper, plus three copies of the ghcr.io shape regex. Consolidates both into slim-images.test-support.ts, next to slim-images.ts. --- .../db-bootstrap/pinned-image.unit.test.ts | 30 ++---------- .../command-internal/db-image.unit.test.ts | 30 +++--------- .../edge-runtime-image.unit.test.ts | 26 +++-------- .../start/start.services.unit.test.ts | 30 ++---------- .../start/start.slim-images.e2e.test.ts | 46 +++++++------------ .../functions/functions.shared.unit.test.ts | 30 ++---------- .../services/slim-images.test-support.ts | 29 ++++++++++++ 7 files changed, 71 insertions(+), 150 deletions(-) create mode 100644 apps/cli/src/shared/services/slim-images.test-support.ts diff --git a/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts b/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts index 69a6302da2..b3192331ab 100644 --- a/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts +++ b/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts @@ -1,8 +1,10 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { catalogPins } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImageRaw } from "../../shared/services/dockerfile-images.ts"; -import { slimCatalogPin } from "../../shared/services/slim-images.ts"; +import { + expectedPinnedImage, + GHCR_SLIM_IMAGE_PATTERN, +} from "../../shared/services/slim-images.test-support.ts"; import { resolvePinnedImage } from "./pinned-image.ts"; const currentTag = (alias: string) => dockerfileServiceImageRaw(alias).split(":")[1] ?? ""; @@ -13,30 +15,6 @@ const currentPoolerTag = currentTag("supavisor"); const currentPostgres = dockerfileServiceImageRaw("pg"); const currentPostgresTag = currentTag("pg"); -/** - * The catalog's own pinned image for `alias`'s (docker.io) `image` — read straight from - * `catalogPins()`, independent of `toSlimImage`, so a test asserting against this actually - * exercises the catalog lookup instead of passing whether or not it resolves (design B: a - * default Dockerfile tag always matches a catalog pin). Only reuses `slimCatalogPin` for alias - * and tag normalization (`v`-prefixing), not the catalog image lookup itself. - */ -function expectedPinnedImage(alias: string, image: string): string { - const pin = slimCatalogPin(alias, image); - if (pin === undefined) { - throw new Error(`no slim catalog pin for ${alias} ${image}`); - } - const entry = catalogPins().find( - (candidate) => - candidate.sourceService === pin.service && candidate.pin.upstreamVersion === pin.version, - ); - if (entry === undefined) { - throw new Error(`no catalog pin for ${pin.service} ${pin.version}`); - } - return entry.pin.image; -} - -const GHCR_SLIM_IMAGE_PATTERN = /^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/; - afterEach(() => { vi.unstubAllEnvs(); }); diff --git a/apps/cli/src/command-internal/db-image.unit.test.ts b/apps/cli/src/command-internal/db-image.unit.test.ts index 0c00d93652..150d2bfea2 100644 --- a/apps/cli/src/command-internal/db-image.unit.test.ts +++ b/apps/cli/src/command-internal/db-image.unit.test.ts @@ -5,13 +5,16 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Path } from "effect"; import { afterEach, beforeEach, vi } from "vitest"; -import { catalogPins } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImage, dockerfileServiceImageRaw, } from "../shared/services/dockerfile-images.ts"; import { imageTag } from "../shared/services/slim-images.ts"; +import { + expectedPinnedImage, + GHCR_SLIM_IMAGE_PATTERN, +} from "../shared/services/slim-images.test-support.ts"; import { resolveDbImage } from "./db-image.ts"; const currentPostgres = dockerfileServiceImageRaw("pg"); @@ -22,25 +25,6 @@ const pg15Image = dockerfileServiceImageRaw("pg15"); const pg15Tag = imageTag(pg15Image) ?? ""; const pg14Image = dockerfileServiceImageRaw("pg14"); -/** - * The catalog's own pinned image for postgres at `tag` — read straight from `catalogPins()`, - * independent of `toSlimImage`, so a test asserting against this actually exercises the catalog - * lookup instead of passing whether or not it resolves (design B: a default Dockerfile tag always - * matches a catalog pin). - */ -function expectedPinnedPostgresImage(tag: string): string { - const entry = catalogPins().find( - (candidate) => candidate.sourceService === "postgres" && candidate.pin.upstreamVersion === tag, - ); - if (entry === undefined) { - throw new Error(`no catalog pin for postgres ${tag}`); - } - return entry.pin.image; -} - -/** A regression to the docker.io fallback must fail an assertion built from this. */ -const GHCR_SLIM_IMAGE_PATTERN = /^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/; - const withTemp = () => mkdtempSync(join(tmpdir(), "db-image-")); const writePin = (workdir: string, pinned: string) => { @@ -135,7 +119,7 @@ describe("resolveDbImage", () => { it.effect("rewrites the current PG15 default tag to its catalog-pinned slim image", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const dir = withTemp(); - const pinned = expectedPinnedPostgresImage(pg15Tag); + const pinned = expectedPinnedImage("pg", pg15Image); expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); return Effect.gen(function* () { expect(yield* resolve(dir, 15)).toEqual({ @@ -167,7 +151,7 @@ describe("resolveDbImage", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const dir = withTemp(); writePin(dir, pg15Tag); - const pinned = expectedPinnedPostgresImage(pg15Tag); + const pinned = expectedPinnedImage("pg", pg15Image); expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); return Effect.gen(function* () { expect(yield* resolve(dir, 15)).toEqual({ @@ -195,7 +179,7 @@ describe("resolveDbImage", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const dir = withTemp(); writePin(dir, currentPostgresTag); - const pinned = expectedPinnedPostgresImage(currentPostgresTag); + const pinned = expectedPinnedImage("pg", currentPostgres); expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); return Effect.gen(function* () { expect(yield* resolve(dir, 17)).toEqual({ diff --git a/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts b/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts index a45ef53be5..ad53727c84 100644 --- a/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts +++ b/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts @@ -5,34 +5,20 @@ import { BunServices } from "@effect/platform-bun"; import { afterEach, describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Path } from "effect"; import { vi } from "vitest"; -import { catalogPins } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImage, dockerfileServiceImageRaw, } from "../shared/services/dockerfile-images.ts"; +import { + expectedPinnedImage, + GHCR_SLIM_IMAGE_PATTERN, +} from "../shared/services/slim-images.test-support.ts"; import { resolveEdgeRuntimeImage } from "./edge-runtime-image.ts"; const currentEdgeRuntime = dockerfileServiceImageRaw("edgeruntime"); const currentEdgeRuntimeTag = currentEdgeRuntime.split(":")[1] ?? ""; -/** - * The catalog's own pinned image for edge-runtime at `tag` — read straight from `catalogPins()`, - * independent of `toSlimImage`, so this test actually exercises the catalog lookup instead of - * passing whether or not it resolves (design B: a default Dockerfile tag always matches a - * catalog pin). - */ -function expectedPinnedEdgeRuntimeImage(tag: string): string { - const entry = catalogPins().find( - (candidate) => - candidate.sourceService === "edge-runtime" && candidate.pin.upstreamVersion === tag, - ); - if (entry === undefined) { - throw new Error(`no catalog pin for edge-runtime ${tag}`); - } - return entry.pin.image; -} - const resolve = (workdir: string, denoVersion: number) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -107,8 +93,8 @@ describe("resolveEdgeRuntimeImage", () => { join(dir, "supabase", ".temp", "edge-runtime-version"), `${currentEdgeRuntimeTag}\n`, ); - const pinned = expectedPinnedEdgeRuntimeImage(currentEdgeRuntimeTag); - expect(pinned).toMatch(/^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/); + const pinned = expectedPinnedImage("edgeruntime", currentEdgeRuntime); + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); return resolve(dir, 2).pipe( Effect.tap((image) => Effect.sync(() => { diff --git a/apps/cli/src/commands/start/start.services.unit.test.ts b/apps/cli/src/commands/start/start.services.unit.test.ts index 7315a29371..0a54d4777b 100644 --- a/apps/cli/src/commands/start/start.services.unit.test.ts +++ b/apps/cli/src/commands/start/start.services.unit.test.ts @@ -1,11 +1,13 @@ import { CliConfigSchema, type CliConfig } from "@supabase/config"; import { Schema } from "effect"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { catalogPins } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImageRaw } from "../../shared/services/dockerfile-images.ts"; import type { LocalServiceVersionOverrides } from "../../shared/services/services.shared.ts"; -import { slimCatalogPin } from "../../shared/services/slim-images.ts"; +import { + expectedPinnedImage, + GHCR_SLIM_IMAGE_PATTERN, +} from "../../shared/services/slim-images.test-support.ts"; import { serviceContainerIds, localDbContainerId } from "../../command-internal/docker-ids.ts"; import { SERVICE_CATALOG } from "../../command-internal/service-catalog.ts"; import { resolveStartGates, resolveStartImagePlan, type StartGates } from "./start.gates.ts"; @@ -17,30 +19,6 @@ const currentVector = dockerfileServiceImageRaw("vector"); const currentPooler = dockerfileServiceImageRaw("supavisor"); const currentPoolerTag = currentPooler.split(":")[1] ?? ""; -/** - * The catalog's own pinned image for `alias`'s (docker.io) `image` — read straight from - * `catalogPins()`, independent of `toSlimImage`, so a test asserting against this actually - * exercises the catalog lookup instead of passing whether or not it resolves (design B: a - * default Dockerfile tag always matches a catalog pin). Only reuses `slimCatalogPin` for alias - * and tag normalization, not the catalog image lookup itself. - */ -function expectedPinnedImage(alias: string, image: string): string { - const pin = slimCatalogPin(alias, image); - if (pin === undefined) { - throw new Error(`no slim catalog pin for ${alias} ${image}`); - } - const entry = catalogPins().find( - (candidate) => - candidate.sourceService === pin.service && candidate.pin.upstreamVersion === pin.version, - ); - if (entry === undefined) { - throw new Error(`no catalog pin for ${pin.service} ${pin.version}`); - } - return entry.pin.image; -} - -const GHCR_SLIM_IMAGE_PATTERN = /^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/; - describe("START_SERVICES", () => { it("has one row per SERVICE_CATALOG entry, in the catalog's startOrder", () => { expect(START_SERVICES).toHaveLength(SERVICE_CATALOG.length); diff --git a/apps/cli/src/commands/start/start.slim-images.e2e.test.ts b/apps/cli/src/commands/start/start.slim-images.e2e.test.ts index bd80bcfb5e..34a244a08d 100644 --- a/apps/cli/src/commands/start/start.slim-images.e2e.test.ts +++ b/apps/cli/src/commands/start/start.slim-images.e2e.test.ts @@ -5,7 +5,11 @@ import { beforeAll, describe, expect, it } from "@effect/vitest"; import { catalogPins, type ServiceKind } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImageRaw } from "../../shared/services/dockerfile-images.ts"; -import { isSlimImageRef, slimCatalogPin, toSlimImage } from "../../shared/services/slim-images.ts"; +import { isSlimImageRef, toSlimImage } from "../../shared/services/slim-images.ts"; +import { + expectedPinnedImage, + GHCR_SLIM_IMAGE_PATTERN, +} from "../../shared/services/slim-images.test-support.ts"; import { buildHealthCmdArg } from "../../command-internal/db-bootstrap/docker-create-args.ts"; import { slimWgetHealthcheck, @@ -108,29 +112,6 @@ function expectedSlimImage(alias: string): string { return toSlimImage(alias, raw) ?? raw; } -/** - * The catalog's own pinned image for `alias`'s current Dockerfile tag — read straight from - * `catalogPins()`, independent of `toSlimImage`. Design B guarantees every slim-capable alias's - * default Dockerfile tag matches a catalog pin (the Dockerfile is generated from the catalog), so - * — unlike `expectedSlimImage` above, which still needs to fall back for `kong` — this never - * falls back, and a regression to docker.io fails the assertion instead of passing quietly. - */ -function expectedPinnedImage(alias: string): string { - const raw = dockerfileServiceImageRaw(alias); - const pin = slimCatalogPin(alias, raw); - if (pin === undefined) { - throw new Error(`no slim catalog pin for ${alias} ${raw}`); - } - const entry = catalogPins().find( - (candidate) => - candidate.sourceService === pin.service && candidate.pin.upstreamVersion === pin.version, - ); - if (entry === undefined) { - throw new Error(`no catalog pin for ${pin.service} ${pin.version}`); - } - return entry.pin.image; -} - /** * The Dockerfile aliases whose spec builder switches its healthcheck on `usesSlimImageRuntime` * (`*.service.ts`). Every one of them is slim today (`expectedPinnedImage`), but the assertions @@ -346,17 +327,22 @@ describe("supabase start slim images (e2e)", () => { // Every alias here is slim-capable, and design B guarantees its default Dockerfile tag // matches a catalog pin — so each expected image is read straight from the catalog // (`expectedPinnedImage`), independent of `toSlimImage`. - const authImage = expectedPinnedImage("gotrue"); - const realtimeImage = expectedPinnedImage("realtime"); - const storageImage = expectedPinnedImage("storage"); + const authImage = expectedPinnedImage("gotrue", dockerfileServiceImageRaw("gotrue")); + const realtimeImage = expectedPinnedImage( + "realtime", + dockerfileServiceImageRaw("realtime"), + ); + const storageImage = expectedPinnedImage("storage", dockerfileServiceImageRaw("storage")); for (const image of [authImage, realtimeImage, storageImage]) { - expect(image).toMatch(/^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/); + expect(image).toMatch(GHCR_SLIM_IMAGE_PATTERN); } - expect(yield* containerImage(dbContainer)).toBe(expectedPinnedImage("pg")); + expect(yield* containerImage(dbContainer)).toBe( + expectedPinnedImage("pg", dockerfileServiceImageRaw("pg")), + ); expect(yield* containerImage(storageContainer)).toBe(storageImage); expect(yield* containerImage(edgeRuntimeContainer)).toBe( - expectedPinnedImage("edgeruntime"), + expectedPinnedImage("edgeruntime", dockerfileServiceImageRaw("edgeruntime")), ); expect(yield* containerImage(authContainer)).toBe(authImage); expect(yield* containerImage(realtimeContainer)).toBe(realtimeImage); diff --git a/apps/cli/src/shared/functions/functions.shared.unit.test.ts b/apps/cli/src/shared/functions/functions.shared.unit.test.ts index 63c3dd591b..17da904fea 100644 --- a/apps/cli/src/shared/functions/functions.shared.unit.test.ts +++ b/apps/cli/src/shared/functions/functions.shared.unit.test.ts @@ -1,9 +1,11 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { Effect } from "effect"; -import { catalogPins } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImageRaw } from "../services/dockerfile-images.ts"; -import { slimCatalogPin } from "../services/slim-images.ts"; +import { + expectedPinnedImage, + GHCR_SLIM_IMAGE_PATTERN, +} from "../services/slim-images.test-support.ts"; import { DENO1_EDGE_RUNTIME_VERSION, edgeRuntimeImage, @@ -13,28 +15,6 @@ import { const rawEdgeRuntimeImage = dockerfileServiceImageRaw("edgeruntime"); const currentEdgeRuntimeTag = rawEdgeRuntimeImage.slice(rawEdgeRuntimeImage.lastIndexOf(":") + 1); -/** - * The catalog's own pinned image for `alias`'s (docker.io) `image` — read straight from - * `catalogPins()`, independent of `toSlimImage`, so a test asserting against this actually - * exercises the catalog lookup instead of passing whether or not it resolves (design B: a - * default Dockerfile tag always matches a catalog pin). Only reuses `slimCatalogPin` for alias - * and tag normalization, not the catalog image lookup itself. - */ -function expectedPinnedImage(alias: string, image: string): string { - const pin = slimCatalogPin(alias, image); - if (pin === undefined) { - throw new Error(`no slim catalog pin for ${alias} ${image}`); - } - const entry = catalogPins().find( - (candidate) => - candidate.sourceService === pin.service && candidate.pin.upstreamVersion === pin.version, - ); - if (entry === undefined) { - throw new Error(`no catalog pin for ${pin.service} ${pin.version}`); - } - return entry.pin.image; -} - afterEach(() => { vi.unstubAllEnvs(); }); @@ -50,7 +30,7 @@ describe("edgeRuntimeImage", () => { it("rewrites the current Dockerfile tag onto the catalog-pinned slim ghcr.io image when the flag is on", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const pinned = expectedPinnedImage("edgeruntime", rawEdgeRuntimeImage); - expect(pinned).toMatch(/^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/); + expect(pinned).toMatch(GHCR_SLIM_IMAGE_PATTERN); expect(edgeRuntimeImage(currentEdgeRuntimeTag)).toBe(pinned); }); diff --git a/apps/cli/src/shared/services/slim-images.test-support.ts b/apps/cli/src/shared/services/slim-images.test-support.ts new file mode 100644 index 0000000000..ee17311f5f --- /dev/null +++ b/apps/cli/src/shared/services/slim-images.test-support.ts @@ -0,0 +1,29 @@ +import { catalogPins } from "@supabase/stack/internal/artifacts"; + +import { slimCatalogPin } from "./slim-images.ts"; + +/** A regression to the docker.io fallback must fail an assertion built from this. */ +export const GHCR_SLIM_IMAGE_PATTERN = /^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/; + +/** + * The catalog's own pinned image for `alias`'s (docker.io) `image` — read straight from + * `catalogPins()`, independent of `toSlimImage`, so a test asserting against this actually + * exercises the catalog lookup instead of passing whether or not it resolves (design B: a + * default Dockerfile tag always matches a catalog pin). Only reuses `slimCatalogPin` for alias + * and tag normalization (`v`-prefixing), not the catalog image lookup itself. Throws when + * nothing is pinned, so a caller never silently falls back to a weaker assertion. + */ +export function expectedPinnedImage(alias: string, image: string): string { + const pin = slimCatalogPin(alias, image); + if (pin === undefined) { + throw new Error(`no slim catalog pin for ${alias} ${image}`); + } + const entry = catalogPins().find( + (candidate) => + candidate.sourceService === pin.service && candidate.pin.upstreamVersion === pin.version, + ); + if (entry === undefined) { + throw new Error(`no catalog pin for ${pin.service} ${pin.version}`); + } + return entry.pin.image; +} From d3088ecf4c0428b111aab0b3b29d56f59df4bdaa Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 19:01:13 +0200 Subject: [PATCH 38/45] test(cli): keep the pinned image helper with the other test helpers --- .github/scripts/sync-artifacts-catalog.test.ts | 7 +++---- .github/scripts/sync-artifacts-catalog.ts | 2 +- apps/cli/scripts/render-service-dockerfile.ts | 2 +- .../db-bootstrap/pinned-image.unit.test.ts | 2 +- apps/cli/src/command-internal/db-image.unit.test.ts | 5 +---- .../src/command-internal/edge-runtime-image.unit.test.ts | 5 +---- apps/cli/src/commands/start/start.services.unit.test.ts | 2 +- apps/cli/src/commands/start/start.slim-images.e2e.test.ts | 6 +++--- .../cli/src/shared/functions/functions.shared.unit.test.ts | 2 +- .../services/slim-images.catalog-alignment.unit.test.ts | 2 +- .../helpers/slim-images.ts} | 6 +++--- 11 files changed, 17 insertions(+), 24 deletions(-) rename apps/cli/{src/shared/services/slim-images.test-support.ts => tests/helpers/slim-images.ts} (84%) diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index b621fe5b66..5e52ba73be 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -74,8 +74,7 @@ async function formatWithOxfmt(source: string): Promise { /** * A resolved `ArtifactPin` literal, for fixtures that exercise refreshing an already-resolved - * catalog entry to a new revision (design B: a committed catalog is never an unresolved - * placeholder). `seed` keys a real, distinct digest via the module's `digest` helper. + * catalog entry to a new revision (the committed catalog only carries resolved pins). `seed` keys a real, distinct digest via the module's `digest` helper. */ function resolvedPinLiteral( service: string, @@ -495,8 +494,8 @@ describe("planSlimUpdates", () => { // The catalog's own pinned upstream carries a space here — never written by `refreshCatalogPin` // (which validates every field it resolves), but this simulates a corrupted catalog, or a // release tag whose upstream portion isn't newline-only-unsafe (`.` already excludes - // newlines) yet still fails `UPSTREAM_VERSION_PATTERN`. Ruling 6 requires every value the - // planner emits to be checked, regardless of source. + // newlines) yet still fails `UPSTREAM_VERSION_PATTERN`. Every value the planner emits is + // checked, regardless of source. const adversarial = `const workloadCatalog = { rest: definition( "postgrest", diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index aad6b9e514..dd002e47c2 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -731,7 +731,7 @@ const SINGLE_LINE_KEY = "*"; * into a file the workflow parses as records; a `console.log`'d warning on the same stdout the * workflow captures would corrupt that file instead of just being informational. * - * Per release line (ruling: the default pin's line, plus one per additional pin — only postgres + * Per release line (the default pin's line, plus one per additional pin — only postgres * has more than one) a **hotfix** fires when the pinned upstream has a committed revision higher * than the pinned one; an **upgrade** fires when the newest committed upstream on the line is * newer than the pinned one (ties, e.g. two Studio builds dated the same day, are not newer). diff --git a/apps/cli/scripts/render-service-dockerfile.ts b/apps/cli/scripts/render-service-dockerfile.ts index da5d146ffc..6c6c114eb6 100644 --- a/apps/cli/scripts/render-service-dockerfile.ts +++ b/apps/cli/scripts/render-service-dockerfile.ts @@ -31,7 +31,7 @@ type CatalogEntry = ReturnType[number]; const REPOSITORY_OVERRIDES: Readonly> = { // slim-services mirrors ghcr.io/imgproxy/imgproxy (its `upstreamImage`), but the Dockerfile — // and `mirror-template-images.yml` and the docker.io registry rewrite — keep darthsim/imgproxy, - // which carries the same tags on Docker Hub. See the design doc's "Repository rule". + // which carries the same tags on Docker Hub. imgproxy: "darthsim/imgproxy", }; diff --git a/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts b/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts index b3192331ab..dd92af2556 100644 --- a/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts +++ b/apps/cli/src/command-internal/db-bootstrap/pinned-image.unit.test.ts @@ -4,7 +4,7 @@ import { dockerfileServiceImageRaw } from "../../shared/services/dockerfile-imag import { expectedPinnedImage, GHCR_SLIM_IMAGE_PATTERN, -} from "../../shared/services/slim-images.test-support.ts"; +} from "../../../tests/helpers/slim-images.ts"; import { resolvePinnedImage } from "./pinned-image.ts"; const currentTag = (alias: string) => dockerfileServiceImageRaw(alias).split(":")[1] ?? ""; diff --git a/apps/cli/src/command-internal/db-image.unit.test.ts b/apps/cli/src/command-internal/db-image.unit.test.ts index 150d2bfea2..bf06d83719 100644 --- a/apps/cli/src/command-internal/db-image.unit.test.ts +++ b/apps/cli/src/command-internal/db-image.unit.test.ts @@ -11,10 +11,7 @@ import { dockerfileServiceImageRaw, } from "../shared/services/dockerfile-images.ts"; import { imageTag } from "../shared/services/slim-images.ts"; -import { - expectedPinnedImage, - GHCR_SLIM_IMAGE_PATTERN, -} from "../shared/services/slim-images.test-support.ts"; +import { expectedPinnedImage, GHCR_SLIM_IMAGE_PATTERN } from "../../tests/helpers/slim-images.ts"; import { resolveDbImage } from "./db-image.ts"; const currentPostgres = dockerfileServiceImageRaw("pg"); diff --git a/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts b/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts index ad53727c84..934500bb6a 100644 --- a/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts +++ b/apps/cli/src/command-internal/edge-runtime-image.unit.test.ts @@ -10,10 +10,7 @@ import { dockerfileServiceImage, dockerfileServiceImageRaw, } from "../shared/services/dockerfile-images.ts"; -import { - expectedPinnedImage, - GHCR_SLIM_IMAGE_PATTERN, -} from "../shared/services/slim-images.test-support.ts"; +import { expectedPinnedImage, GHCR_SLIM_IMAGE_PATTERN } from "../../tests/helpers/slim-images.ts"; import { resolveEdgeRuntimeImage } from "./edge-runtime-image.ts"; const currentEdgeRuntime = dockerfileServiceImageRaw("edgeruntime"); diff --git a/apps/cli/src/commands/start/start.services.unit.test.ts b/apps/cli/src/commands/start/start.services.unit.test.ts index 0a54d4777b..d7fc46fd66 100644 --- a/apps/cli/src/commands/start/start.services.unit.test.ts +++ b/apps/cli/src/commands/start/start.services.unit.test.ts @@ -7,7 +7,7 @@ import type { LocalServiceVersionOverrides } from "../../shared/services/service import { expectedPinnedImage, GHCR_SLIM_IMAGE_PATTERN, -} from "../../shared/services/slim-images.test-support.ts"; +} from "../../../tests/helpers/slim-images.ts"; import { serviceContainerIds, localDbContainerId } from "../../command-internal/docker-ids.ts"; import { SERVICE_CATALOG } from "../../command-internal/service-catalog.ts"; import { resolveStartGates, resolveStartImagePlan, type StartGates } from "./start.gates.ts"; diff --git a/apps/cli/src/commands/start/start.slim-images.e2e.test.ts b/apps/cli/src/commands/start/start.slim-images.e2e.test.ts index 34a244a08d..2a44aacfb3 100644 --- a/apps/cli/src/commands/start/start.slim-images.e2e.test.ts +++ b/apps/cli/src/commands/start/start.slim-images.e2e.test.ts @@ -9,7 +9,7 @@ import { isSlimImageRef, toSlimImage } from "../../shared/services/slim-images.t import { expectedPinnedImage, GHCR_SLIM_IMAGE_PATTERN, -} from "../../shared/services/slim-images.test-support.ts"; +} from "../../../tests/helpers/slim-images.ts"; import { buildHealthCmdArg } from "../../command-internal/db-bootstrap/docker-create-args.ts"; import { slimWgetHealthcheck, @@ -324,8 +324,8 @@ describe("supabase start slim images (e2e)", () => { }); expect(start.exitCode, `stdout:\n${start.stdout}\nstderr:\n${start.stderr}`).toBe(0); - // Every alias here is slim-capable, and design B guarantees its default Dockerfile tag - // matches a catalog pin — so each expected image is read straight from the catalog + // Every alias here is slim-capable, and its default Dockerfile tag is generated from the + // catalog, so it matches a catalog pin — so each expected image is read straight from the catalog // (`expectedPinnedImage`), independent of `toSlimImage`. const authImage = expectedPinnedImage("gotrue", dockerfileServiceImageRaw("gotrue")); const realtimeImage = expectedPinnedImage( diff --git a/apps/cli/src/shared/functions/functions.shared.unit.test.ts b/apps/cli/src/shared/functions/functions.shared.unit.test.ts index 17da904fea..2ced067395 100644 --- a/apps/cli/src/shared/functions/functions.shared.unit.test.ts +++ b/apps/cli/src/shared/functions/functions.shared.unit.test.ts @@ -5,7 +5,7 @@ import { dockerfileServiceImageRaw } from "../services/dockerfile-images.ts"; import { expectedPinnedImage, GHCR_SLIM_IMAGE_PATTERN, -} from "../services/slim-images.test-support.ts"; +} from "../../../tests/helpers/slim-images.ts"; import { DENO1_EDGE_RUNTIME_VERSION, edgeRuntimeImage, diff --git a/apps/cli/src/shared/services/slim-images.catalog-alignment.unit.test.ts b/apps/cli/src/shared/services/slim-images.catalog-alignment.unit.test.ts index 12baff5a9b..ea5fd0dae4 100644 --- a/apps/cli/src/shared/services/slim-images.catalog-alignment.unit.test.ts +++ b/apps/cli/src/shared/services/slim-images.catalog-alignment.unit.test.ts @@ -7,7 +7,7 @@ import { slimCatalogPin, toSlimImage } from "./slim-images.ts"; * Every slim-capable Dockerfile alias, against the real (unmocked) catalog. The Dockerfile is * generated from that same catalog (`render-service-dockerfile.ts`), so each alias's raw tag is * always one of the catalog's own upstream versions by construction — no fixture, no - * `vi.mock`, exercising the true "one-time alignment" invariant end to end. + * `vi.mock`, exercising the Dockerfile/catalog alignment end to end. */ const SLIM_CAPABLE_ALIASES = [ "pg", diff --git a/apps/cli/src/shared/services/slim-images.test-support.ts b/apps/cli/tests/helpers/slim-images.ts similarity index 84% rename from apps/cli/src/shared/services/slim-images.test-support.ts rename to apps/cli/tests/helpers/slim-images.ts index ee17311f5f..0bab6745fc 100644 --- a/apps/cli/src/shared/services/slim-images.test-support.ts +++ b/apps/cli/tests/helpers/slim-images.ts @@ -1,6 +1,6 @@ import { catalogPins } from "@supabase/stack/internal/artifacts"; -import { slimCatalogPin } from "./slim-images.ts"; +import { slimCatalogPin } from "../../src/shared/services/slim-images.ts"; /** A regression to the docker.io fallback must fail an assertion built from this. */ export const GHCR_SLIM_IMAGE_PATTERN = /^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/; @@ -8,8 +8,8 @@ export const GHCR_SLIM_IMAGE_PATTERN = /^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9 /** * The catalog's own pinned image for `alias`'s (docker.io) `image` — read straight from * `catalogPins()`, independent of `toSlimImage`, so a test asserting against this actually - * exercises the catalog lookup instead of passing whether or not it resolves (design B: a - * default Dockerfile tag always matches a catalog pin). Only reuses `slimCatalogPin` for alias + * exercises the catalog lookup instead of passing whether or not it resolves (a default + * Dockerfile tag is generated from the catalog, so it always matches a catalog pin). Only reuses `slimCatalogPin` for alias * and tag normalization (`v`-prefixing), not the catalog image lookup itself. Throws when * nothing is pinned, so a caller never silently falls back to a weaker assertion. */ From 24136023656321dfba9e15dead398f765bd10239 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 19:15:45 +0200 Subject: [PATCH 39/45] fix(ci): encode workflow command data at the output boundary Routes every ::error :: and ::warning :: line this script emits through one encoder (workflow-command data encoding: %, then \r/\n), so an external value reaching the top-level catch or a planner warning can never inject an extra workflow command via an embedded newline. Also validates --service in plan-updates before any listing or wait, and makes the expect-release wait interval reject a malformed SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS instead of silently coercing it to NaN, 0, or a negative delay. --- .../scripts/sync-artifacts-catalog.test.ts | 97 +++++++++++++++++++ .github/scripts/sync-artifacts-catalog.ts | 67 ++++++++++--- 2 files changed, 153 insertions(+), 11 deletions(-) diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index 5e52ba73be..1f134236ba 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -448,6 +448,20 @@ describe("planSlimUpdates", () => { ]); }); + test("a `%` in a non-comparable version is encoded, staying a single workflow-command line", () => { + // `.` in the tag pattern allows `%` (only a real newline can't reach this far — the pattern + // can't match across one), so a real, legitimately-listed tag can still carry a `%` here. + const { updates, warnings } = planSlimUpdates(plannerFixture, "postgrest", [ + "postgrest-v16.2%off-r0", + ]); + + expect(updates).toEqual([]); + expect(warnings).toEqual([ + "::warning ::postgrest v16.2%25off is not a comparable version; ignoring.", + ]); + expect(warnings[0]?.split("\n")).toHaveLength(1); + }); + test("a legacy tag with no -rN is ignored", () => { const { updates, warnings } = planSlimUpdates(plannerFixture, "postgrest", ["postgrest-v16.4"]); @@ -752,6 +766,89 @@ describe("runPlanUpdates (the actual plan-updates CLI mode, not just the pure pl "Usage: sync-artifacts-catalog.ts plan-updates --service --output ", ); }); + + test("a newline-bearing --service is rejected before any listing or wait", async () => { + const dir = await mkdtemp(join(tmpdir(), "plan-updates-bad-service-")); + const outputPath = join(dir, "slim-updates.tsv"); + try { + const proc = Bun.spawn( + [ + "bun", + ".github/scripts/sync-artifacts-catalog.ts", + "plan-updates", + "--service", + "auth\n::error ::injected", + "--output", + outputPath, + ], + { stdout: "pipe", stderr: "pipe" }, + ); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + + expect(exitCode).toBe(1); + expect(stdout).toContain("invalid --service"); + await expect(readFile(outputPath, "utf8")).rejects.toThrow(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("a newline-bearing --format is rejected, producing exactly one ::error :: line (not JSON.stringify'd; the boundary encoder is what protects it)", async () => { + const dir = await mkdtemp(join(tmpdir(), "plan-updates-bad-format-")); + const outputPath = join(dir, "slim-updates.tsv"); + try { + const proc = Bun.spawn( + [ + "bun", + ".github/scripts/sync-artifacts-catalog.ts", + "plan-updates", + "--service", + "auth", + "--output", + outputPath, + "--format", + "lines\n::error ::injected", + ], + { stdout: "pipe", stderr: "pipe" }, + ); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + + expect(exitCode).toBe(1); + const lines = stdout.trimEnd().split("\n"); + expect(lines).toHaveLength(1); + expect(lines[0]).toMatch(/^::error ::/); + expect(lines[0]).toContain("%0A"); + await expect(readFile(outputPath, "utf8")).rejects.toThrow(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("an invalid SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS exits non-zero with a configuration error", async () => { + const proc = Bun.spawn( + [ + "bun", + ".github/scripts/sync-artifacts-catalog.ts", + "plan-updates", + "--service", + "postgrest", + "--expect-release", + "v999.0-r0", + "--output", + join(await mkdtemp(join(tmpdir(), "plan-updates-bad-wait-")), "slim-updates.tsv"), + ], + { + stdout: "pipe", + stderr: "pipe", + env: { ...globalThis.process.env, SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS: "5ms" }, + }, + ); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + + expect(exitCode).toBe(1); + expect(stdout).toContain("SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS"); + expect(stdout).toContain("non-negative integer"); + }); }); describe("refreshCatalogPin", () => { diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index dd002e47c2..8655fe397e 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -66,10 +66,38 @@ const RELEASE_DOWNLOAD_BASE = "https://github.com/supabase/slim-services/release /** Bounded retry for `waitForExpectedRelease`: 6 attempts, 10s apart, by default. */ const EXPECT_RELEASE_ATTEMPTS = 6; -/** Overridable so an integration test can shrink the real delay between attempts. */ -const EXPECT_RELEASE_INTERVAL_MS = Number( - process.env.SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS ?? 10_000, -); +const DEFAULT_EXPECT_RELEASE_INTERVAL_MS = 10_000; + +/** + * Parses `SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS` fresh on every call (not once at module load), so + * an integration test can shrink the real delay between `waitForExpectedRelease` attempts by + * setting the env var before spawning, not before this module is first imported. Unset keeps the + * default; anything else must be a non-negative integer (no `"5ms"`, no `""`, no negative value, + * no decimal) or this throws a clear configuration error instead of silently coercing it to NaN, + * 0, or a meaningless delay. + */ +function expectReleaseIntervalMs(): number { + const raw = process.env.SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS; + if (raw === undefined) return DEFAULT_EXPECT_RELEASE_INTERVAL_MS; + if (!/^(0|[1-9][0-9]*)$/.test(raw)) { + throw new InvalidPayloadError( + `invalid SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS ${JSON.stringify(raw)}: expected a non-negative integer`, + ); + } + return Number(raw); +} + +/** + * A GitHub Actions workflow-command line (`::error ::…`/`::warning ::…`), with `message` run + * through the workflow-command data encoding (`%` first, so encoding `\r`/`\n` into `%0D`/`%0A` + * never gets re-escaped) — the single boundary every `::error ::`/`::warning ::` this script + * emits goes through, so an external value embedding a newline can never start a second line the + * runner's log would parse as its own workflow command. + */ +function workflowCommand(kind: "error" | "warning", message: string): string { + const encoded = message.replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A"); + return `::${kind} ::${encoded}`; +} /** Leading numeric component, `v` stripped. Only postgres carries more than one line. */ function releaseLine(version: string): string { @@ -750,7 +778,12 @@ export function planSlimUpdates( const warnings: string[] = []; const pins = collectServicePins(catalog, service); if (pins === undefined) { - warnings.push(`::warning ::${CATALOG_PATH} has no slim entry for ${service}; nothing to plan.`); + warnings.push( + workflowCommand( + "warning", + `${CATALOG_PATH} has no slim entry for ${service}; nothing to plan.`, + ), + ); return { updates: [], warnings }; } @@ -782,7 +815,10 @@ export function planSlimUpdates( line = releaseLine(upstream); if (!pinnedByLine.has(line)) { warnings.push( - `::warning ::${service} ${upstream} is not on a release line ${CATALOG_PATH} carries for it; ignoring ${tag}.`, + workflowCommand( + "warning", + `${service} ${upstream} is not on a release line ${CATALOG_PATH} carries for it; ignoring ${tag}.`, + ), ); continue; } @@ -822,7 +858,12 @@ export function planSlimUpdates( for (const upstream of highestRevisionByUpstream.keys()) { const comparedToPinned = compareVersions(upstream, pinned.upstream); if (comparedToPinned === undefined) { - warnings.push(`::warning ::${service} ${upstream} is not a comparable version; ignoring.`); + warnings.push( + workflowCommand( + "warning", + `${service} ${upstream} is not a comparable version; ignoring.`, + ), + ); continue; } if (bestUpstream === undefined || (compareVersions(upstream, bestUpstream) ?? 0) > 0) { @@ -864,8 +905,9 @@ export interface WaitForExpectedReleaseResult { * dispatch that triggered this run: re-lists up to `attempts` times, `io.wait`-ing between * attempts, until `-` appears. Not a test retry — a real, bounded wait * for an external API to catch up. `io.wait` is the seam a unit test overrides to skip the real - * delay; an integration test instead shrinks `EXPECT_RELEASE_INTERVAL_MS` via - * `SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS`, since a subprocess can't be handed a function. + * delay; an integration test instead shrinks the real delay via `SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS` + * (parsed fresh by `expectReleaseIntervalMs` on every call this default reaches), since a + * subprocess can't be handed a function. */ export async function waitForExpectedRelease( service: string, @@ -875,7 +917,7 @@ export async function waitForExpectedRelease( readonly wait: (ms: number) => Promise; }, attempts: number = EXPECT_RELEASE_ATTEMPTS, - intervalMs: number = EXPECT_RELEASE_INTERVAL_MS, + intervalMs: number = expectReleaseIntervalMs(), ): Promise { const expectedTag = `${service}-${releaseVersion}`; let tags: ReadonlyArray = []; @@ -1183,6 +1225,9 @@ export async function runPlanUpdates( "[--format lines] [--expect-release -r]", ); } + if (!SERVICE_NAME_PATTERN.test(service)) { + throw new InvalidPayloadError(`invalid --service ${JSON.stringify(service)}`); + } const format = flags.get("format") ?? "json"; if (format !== "json" && format !== "lines") { throw new InvalidPayloadError(`invalid --format '${format}' (expected 'json' or 'lines')`); @@ -1266,7 +1311,7 @@ async function main(argv: ReadonlyArray): Promise { if (import.meta.main) { main(process.argv.slice(2)).catch((error: unknown) => { - console.log(`::error ::${error instanceof Error ? error.message : String(error)}`); + console.log(workflowCommand("error", error instanceof Error ? error.message : String(error))); process.exit(1); }); } From 89fb625fb075d2bc7b65ea043cd5c9ff646cf684 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 19:21:52 +0200 Subject: [PATCH 40/45] fix(ci): reject wait overrides a timer cannot honor --- .../scripts/sync-artifacts-catalog.test.ts | 51 ++++++++++--------- .github/scripts/sync-artifacts-catalog.ts | 10 ++-- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index 1f134236ba..46af42abe2 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -824,31 +824,34 @@ describe("runPlanUpdates (the actual plan-updates CLI mode, not just the pure pl } }); - test("an invalid SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS exits non-zero with a configuration error", async () => { - const proc = Bun.spawn( - [ - "bun", - ".github/scripts/sync-artifacts-catalog.ts", - "plan-updates", - "--service", - "postgrest", - "--expect-release", - "v999.0-r0", - "--output", - join(await mkdtemp(join(tmpdir(), "plan-updates-bad-wait-")), "slim-updates.tsv"), - ], - { - stdout: "pipe", - stderr: "pipe", - env: { ...globalThis.process.env, SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS: "5ms" }, - }, - ); - const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); + test.each(["5ms", "9".repeat(309)])( + "an invalid SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS (%s) exits non-zero with a configuration error", + async (waitMs) => { + const proc = Bun.spawn( + [ + "bun", + ".github/scripts/sync-artifacts-catalog.ts", + "plan-updates", + "--service", + "postgrest", + "--expect-release", + "v999.0-r0", + "--output", + join(await mkdtemp(join(tmpdir(), "plan-updates-bad-wait-")), "slim-updates.tsv"), + ], + { + stdout: "pipe", + stderr: "pipe", + env: { ...globalThis.process.env, SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS: waitMs }, + }, + ); + const [stdout, exitCode] = await Promise.all([new Response(proc.stdout).text(), proc.exited]); - expect(exitCode).toBe(1); - expect(stdout).toContain("SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS"); - expect(stdout).toContain("non-negative integer"); - }); + expect(exitCode).toBe(1); + expect(stdout).toContain("SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS"); + expect(stdout).toContain("non-negative integer"); + }, + ); }); describe("refreshCatalogPin", () => { diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index 8655fe397e..431a81c771 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -76,15 +76,19 @@ const DEFAULT_EXPECT_RELEASE_INTERVAL_MS = 10_000; * no decimal) or this throws a clear configuration error instead of silently coercing it to NaN, * 0, or a meaningless delay. */ +const MAX_TIMER_DELAY_MS = 2 ** 31 - 1; + function expectReleaseIntervalMs(): number { const raw = process.env.SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS; if (raw === undefined) return DEFAULT_EXPECT_RELEASE_INTERVAL_MS; - if (!/^(0|[1-9][0-9]*)$/.test(raw)) { + const value = Number(raw); + // `setTimeout` clamps anything above 2^31 - 1 ms, so a larger value can't be honored either. + if (!/^(0|[1-9][0-9]*)$/.test(raw) || value > MAX_TIMER_DELAY_MS) { throw new InvalidPayloadError( - `invalid SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS ${JSON.stringify(raw)}: expected a non-negative integer`, + `invalid SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS ${JSON.stringify(raw)}: expected a non-negative integer of at most ${MAX_TIMER_DELAY_MS}`, ); } - return Number(raw); + return value; } /** From 3eefe2e2ca133caf087c032f238ff0b8c4690a00 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 20:22:20 +0200 Subject: [PATCH 41/45] fix(ci): wait for the S3 mirror before pinning a new release The S3 native-archive mirror uploads in parallel with the dispatch, best- effort, so a freshly committed release's S3 copy can lag GitHub by the time the sync script pins it. Waits, bounded, while an object is missing (reusing the same retry helper --expect-release's visibility wait uses), but fails immediately on a digest mismatch, since that's corruption rather than lag. --- .../scripts/sync-artifacts-catalog.test.ts | 82 +++++++++++ .github/scripts/sync-artifacts-catalog.ts | 129 +++++++++++++----- docs/adr/0026-slim-artifact-mirrors.md | 13 +- 3 files changed, 186 insertions(+), 38 deletions(-) diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index 46af42abe2..1a266d12be 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -9,6 +9,7 @@ import { planSlimUpdates, planUpdatesForService, refreshCatalogPin, + resolveRevisionPin, validateSlimReleasePublishedPayload, waitForExpectedRelease, type RevisionIo, @@ -1030,6 +1031,87 @@ describe("refreshCatalogPin", () => { }); }); +describe("resolveRevisionPin waits for the S3 mirror", () => { + test("waits while one S3 object is missing, then resolves once it appears", async () => { + const digests = nativeDigests("s3-wait"); + const releaseVersion = "v16.2-r0"; + const missingUrl = nativeObjectUrl( + "postgrest", + releaseVersion, + nativeFileNames("postgrest", releaseVersion, "darwin-arm64").archive, + ); + const present = matchingS3("postgrest", releaseVersion, digests); + let missingCalls = 0; + const waits: number[] = []; + const io: RevisionIo = { + listReleaseTags: async () => [`postgrest-${releaseVersion}`], + fetchChecksums: async () => checksumsFor("postgrest", releaseVersion, digests), + imageDigest: async () => digest("s3-wait-digest"), + s3Sha256: async (url) => { + if (url === missingUrl) { + missingCalls += 1; + if (missingCalls < 2) return undefined; + } + return present(url); + }, + wait: async (ms) => { + waits.push(ms); + }, + }; + + const resolution = await resolveRevisionPin("postgrest", "v16.2", io); + + expect(resolution.status).toBe("resolved"); + expect(missingCalls).toBe(2); + expect(waits).toHaveLength(1); + }); + + test("fails immediately on a digest mismatch, without waiting", async () => { + const digests = nativeDigests("s3-mismatch"); + const releaseVersion = "v16.2-r0"; + const waits: number[] = []; + const io: RevisionIo = { + listReleaseTags: async () => [`postgrest-${releaseVersion}`], + fetchChecksums: async () => checksumsFor("postgrest", releaseVersion, digests), + imageDigest: async () => digest("s3-mismatch-digest"), + s3Sha256: async () => hex("wrong-bytes"), + wait: async (ms) => { + waits.push(ms); + }, + }; + + const resolution = await resolveRevisionPin("postgrest", "v16.2", io); + + expect(resolution.status).toBe("stale"); + if (resolution.status !== "stale") throw new Error("expected status 'stale'"); + expect(resolution.message).toContain("digest mismatch"); + expect(waits).toEqual([]); + }); + + test("fails after the bounded attempts when an S3 object never appears, with an actionable message", async () => { + const digests = nativeDigests("s3-never"); + const releaseVersion = "v16.2-r0"; + const waits: number[] = []; + const io: RevisionIo = { + listReleaseTags: async () => [`postgrest-${releaseVersion}`], + fetchChecksums: async () => checksumsFor("postgrest", releaseVersion, digests), + imageDigest: async () => digest("s3-never-digest"), + s3Sha256: async () => undefined, + wait: async (ms) => { + waits.push(ms); + }, + }; + + const resolution = await resolveRevisionPin("postgrest", "v16.2", io); + + expect(resolution.status).toBe("stale"); + if (resolution.status !== "stale") throw new Error("expected status 'stale'"); + expect(resolution.message).toContain("mirror-slim-image.yml"); + expect(resolution.message).toContain("hasn't finished"); + expect(waits.length).toBeGreaterThan(0); + }); +}); + describe("refreshCatalogPin --release", () => { test("pins exactly the requested committed release, not the highest one", async () => { const digests = nativeDigests("release-0"); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index 431a81c771..0ec85a6b02 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -67,36 +67,73 @@ const RELEASE_DOWNLOAD_BASE = "https://github.com/supabase/slim-services/release /** Bounded retry for `waitForExpectedRelease`: 6 attempts, 10s apart, by default. */ const EXPECT_RELEASE_ATTEMPTS = 6; const DEFAULT_EXPECT_RELEASE_INTERVAL_MS = 10_000; +/** Bounded retry for the S3-mirror wait in `resolveRevisionPin`: ~10 min, covering a native upload of all three targets (`mirror-slim-image.yml`'s `upload-natives-s3`). */ +const S3_WAIT_ATTEMPTS = 20; +const DEFAULT_S3_WAIT_INTERVAL_MS = 30_000; +const MAX_TIMER_DELAY_MS = 2 ** 31 - 1; // setTimeout's own ceiling. /** - * Parses `SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS` fresh on every call (not once at module load), so - * an integration test can shrink the real delay between `waitForExpectedRelease` attempts by - * setting the env var before spawning, not before this module is first imported. Unset keeps the - * default; anything else must be a non-negative integer (no `"5ms"`, no `""`, no negative value, - * no decimal) or this throws a clear configuration error instead of silently coercing it to NaN, - * 0, or a meaningless delay. + * Parses a millisecond wait-interval override from `envVar` fresh on every call, not once at + * module load, so a test can set it right before spawning. Unset keeps `fallback`; anything else + * must be a non-negative integer of at most `MAX_TIMER_DELAY_MS`, or this throws. */ -const MAX_TIMER_DELAY_MS = 2 ** 31 - 1; - -function expectReleaseIntervalMs(): number { - const raw = process.env.SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS; - if (raw === undefined) return DEFAULT_EXPECT_RELEASE_INTERVAL_MS; +function waitIntervalMs(envVar: string, fallback: number): number { + const raw = process.env[envVar]; + if (raw === undefined) return fallback; const value = Number(raw); - // `setTimeout` clamps anything above 2^31 - 1 ms, so a larger value can't be honored either. if (!/^(0|[1-9][0-9]*)$/.test(raw) || value > MAX_TIMER_DELAY_MS) { throw new InvalidPayloadError( - `invalid SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS ${JSON.stringify(raw)}: expected a non-negative integer of at most ${MAX_TIMER_DELAY_MS}`, + `invalid ${envVar} ${JSON.stringify(raw)}: expected a non-negative integer of at most ${MAX_TIMER_DELAY_MS}`, ); } return value; } +const expectReleaseIntervalMs = (): number => + waitIntervalMs("SLIM_UPDATES_EXPECT_RELEASE_WAIT_MS", DEFAULT_EXPECT_RELEASE_INTERVAL_MS); +const s3WaitIntervalMs = (): number => + waitIntervalMs("SLIM_UPDATES_S3_WAIT_MS", DEFAULT_S3_WAIT_INTERVAL_MS); + +/** Real delay, for a caller that didn't override `wait`. */ +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +type BoundedCheck = + | { readonly kind: "done"; readonly value: T } + | { readonly kind: "retry" } + | { readonly kind: "failed"; readonly message: string }; + +type BoundedWaitOutcome = + | { readonly status: "done"; readonly value: T } + | { readonly status: "timed-out" } + | { readonly status: "failed"; readonly message: string }; + +/** + * Generic bounded retry-with-wait: `waitForExpectedRelease`'s release-visibility wait and + * `resolveRevisionPin`'s S3-mirror wait both build on this. Calls `check` up to `attempts` + * times, `wait`-ing `intervalMs` between, until it reports `"done"` or an unrecoverable + * `"failed"` (never waited out). Exhausting every attempt on `"retry"` yields `"timed-out"`. + */ +async function boundedWait( + check: () => Promise>, + wait: (ms: number) => Promise, + attempts: number, + intervalMs: number, +): Promise> { + for (let attempt = 0; attempt < attempts; attempt++) { + const result = await check(); + if (result.kind === "done") return { status: "done", value: result.value }; + if (result.kind === "failed") return { status: "failed", message: result.message }; + if (attempt < attempts - 1) await wait(intervalMs); + } + return { status: "timed-out" }; +} + /** - * A GitHub Actions workflow-command line (`::error ::…`/`::warning ::…`), with `message` run - * through the workflow-command data encoding (`%` first, so encoding `\r`/`\n` into `%0D`/`%0A` - * never gets re-escaped) — the single boundary every `::error ::`/`::warning ::` this script - * emits goes through, so an external value embedding a newline can never start a second line the - * runner's log would parse as its own workflow command. + * A GitHub Actions workflow-command line, `message` run through the workflow-command data + * encoding (`%` first, so encoding `\r`/`\n` never gets re-escaped) — the boundary every + * `::error ::`/`::warning ::` this script emits goes through. */ function workflowCommand(kind: "error" | "warning", message: string): string { const encoded = message.replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A"); @@ -192,6 +229,8 @@ export interface RevisionIo { readonly fetchChecksums: (service: string, releaseVersion: string) => Promise; readonly imageDigest: (service: string, releaseVersion: string) => Promise; readonly s3Sha256: (url: string) => Promise; + /** Overridable real delay between the S3-mirror wait's bounded attempts. */ + readonly wait?: (ms: number) => Promise; /** * Raw contents of a native target's `.manifest.json` release asset, for a derived service's * `upstream_image` (or `source_image` on an image-derived build, e.g. postgrest's Linux @@ -300,15 +339,42 @@ export async function resolveRevisionPin( }; } + // `mirror-slim-image.yml`'s S3 upload runs in parallel, best-effort, so a freshly committed + // revision's S3 copy can briefly lag GitHub: a missing object waits, bounded; a present object + // with the wrong bytes fails immediately instead of waiting. for (const target of NATIVE_TARGETS) { const files = nativeFileNames(service, releaseVersion, target); for (const part of ["archive", "manifest"] as const) { const url = nativeObjectUrl(service, releaseVersion, files[part]); - const actual = await io.s3Sha256(url); - if (actual === undefined || actual !== natives[target]?.[part]) { + const expected = natives[target]?.[part] as string; + const outcome = await boundedWait( + async () => { + const actual = await io.s3Sha256(url); + if (actual === undefined) { + console.log( + `Waiting for the S3 mirror of ${service}-${releaseVersion} ${target} (${part})...`, + ); + return { kind: "retry" }; + } + if (actual !== expected) { + return { + kind: "failed", + message: `S3 copy of ${releaseVersion} ${target} (${part}) is stale (digest mismatch); run the slim-services mirror backfill.`, + }; + } + return { kind: "done", value: true }; + }, + io.wait ?? sleep, + S3_WAIT_ATTEMPTS, + s3WaitIntervalMs(), + ); + if (outcome.status === "failed") { + return { status: "stale", message: outcome.message }; + } + if (outcome.status === "timed-out") { return { status: "stale", - message: `S3 copy of ${releaseVersion} ${target} is stale; run the slim-services mirror backfill`, + message: `S3 copy of ${releaseVersion} ${target} (${part}) never appeared; the S3 mirror hasn't finished — check mirror-slim-image.yml for this release, backfill if needed, and re-run.`, }; } } @@ -925,12 +991,16 @@ export async function waitForExpectedRelease( ): Promise { const expectedTag = `${service}-${releaseVersion}`; let tags: ReadonlyArray = []; - for (let attempt = 0; attempt < attempts; attempt++) { - tags = await io.listReleaseTags(); - if (tags.includes(expectedTag)) return { visible: true, tags }; - if (attempt < attempts - 1) await io.wait(intervalMs); - } - return { visible: false, tags }; + const outcome = await boundedWait( + async () => { + tags = await io.listReleaseTags(); + return tags.includes(expectedTag) ? { kind: "done", value: true } : { kind: "retry" }; + }, + io.wait, + attempts, + intervalMs, + ); + return { visible: outcome.status === "done", tags }; } export interface CatalogRefreshResult { @@ -1188,11 +1258,6 @@ function updateLine(update: SlimUpdate): string { ); } -/** Real delay, for `runPlanUpdates`' default `io.wait`. */ -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - /** * `plan-updates`' records go only into `--output `, never stdout: a caller (the workflow) * reads warnings from stdout as `::warning ::…` lines, and would otherwise mistake one for a diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index b56be1a060..cba056ffe3 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -77,12 +77,13 @@ and either `--upstream` (the highest committed revision of that upstream version (exactly that committed `-r`, never "highest at apply time"), it reads the target release's `SHA256SUMS` for the archive and manifest sha256 per target, and resolves the image digest with `regctl manifest head`. Before writing the pin, it downloads each target's S3 archive -and manifest and hashes them against those same release sums; a mismatch fails the sync instead -of pinning bytes that don't match GitHub. This exists because `publish-release` does not wait for -the ECR/S3 mirror to finish, so a freshly committed revision's S3 copy can briefly lag or hold -bytes from an earlier failed attempt. Hosts that reach GitHub are unaffected — GitHub is the -primary mirror — but a host that can only reach S3 would otherwise fail verification with no -fallback. A mismatch here means "run the mirror backfill", not "the CLI is broken". +and manifest and hashes them against those same release sums. This exists because +`publish-release` does not wait for the ECR/S3 mirror to finish, so a freshly committed +revision's S3 copy can briefly lag. A missing object waits, bounded (`waitForExpectedRelease`'s +retry helper, generalized); an object that exists with the wrong bytes fails the sync +immediately — that's corruption, not lag, and means "run the mirror backfill", not "the CLI is +broken". Hosts that reach GitHub are unaffected — GitHub is the primary mirror — but a host that +can only reach S3 would otherwise fail verification with no fallback. ### Hotfix and upgrade pickup From 6ef26260b040bfd67d492761c78bdab44b50de46 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 20:22:32 +0200 Subject: [PATCH 42/45] fix(ci): keep the app token away from third-party code The apply loop ran bun x oxfmt (fetching ~46 packages from the public registry) and every sync/generator call in the same step that held the app token in its env, and persisted the token into .git/config via git remote set-url. Formats with the locked pnpm install instead, pushes with the token only in an explicit URL computed once (never written to disk), and strips it from every bun/pnpm command with env -u. gh now also gets the token inline per call, and filters gh pr list to same-repository PRs (isCrossRepository) so a fork PR with a matching branch name can't be selected and edited with a trusted-looking title and body. --- .github/workflows/slim-release-published.yml | 40 ++++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/slim-release-published.yml b/.github/workflows/slim-release-published.yml index 2db3112a5e..1c4f317634 100644 --- a/.github/workflows/slim-release-published.yml +++ b/.github/workflows/slim-release-published.yml @@ -132,27 +132,22 @@ jobs: permission-contents: write permission-pull-requests: write - # Kept as its own step, before the loop, so the loop body below can be exercised against a - # local bare remote in a dry run (point `origin` there instead of running this step). - - name: Point origin at the authenticated remote - if: steps.plan.outputs.count != '0' - env: - APP_TOKEN: ${{ steps.app-token.outputs.token }} - run: | - set -euo pipefail - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git remote set-url origin "https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - # One branch per planned item, built fresh from the base commit recorded above (the default # branch's head at the start of this run) — never from wherever a previous loop iteration # left HEAD. Pins exactly the planned release (`--release`, never "highest at apply time"), # so plan and pin agree by construction: a revision minted a second later arrives with its # own dispatch, not by racing this one. + # + # The app token (contents + pull-requests write) never reaches third-party code or disk: + # `git push` takes it only as part of an explicit URL, computed once as `push_url` + # (overridable via `PUSH_REMOTE_URL`, the seam a dry run uses to target a local bare repo + # instead), and `gh` gets it inline per call. Every `bun`/`pnpm` command below — the sync + # script, the Dockerfile generator, the formatter — runs under `env -u APP_TOKEN` so a + # compromised transitive dependency of any of them (they import `effect`) can't read it. - name: Apply planned updates if: steps.plan.outputs.count != '0' env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} + APP_TOKEN: ${{ steps.app-token.outputs.token }} GITHUB_TOKEN: ${{ github.token }} SERVICE: ${{ steps.validate.outputs.service }} RELEASE_VERSION: ${{ steps.validate.outputs.release_version }} @@ -160,6 +155,9 @@ jobs: DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + push_url="${PUSH_REMOTE_URL:-https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}.git}" manual_prefix="bun .github/scripts/sync-artifacts-catalog.ts --service ${SERVICE} --release" # Read the records on fd 3, not stdin: every command the loop body runs (`bun`, `gh`, @@ -173,9 +171,9 @@ jobs: git checkout -B "$branch" "$BASE_SHA" - bun .github/scripts/sync-artifacts-catalog.ts --service "$SERVICE" --release "$release" - bun x oxfmt@0.65.0 --config .oxfmtrc.json packages/stack/src/Artifacts.ts - bun apps/cli/scripts/render-service-dockerfile.ts + env -u APP_TOKEN bun .github/scripts/sync-artifacts-catalog.ts --service "$SERVICE" --release "$release" + env -u APP_TOKEN -u GITHUB_TOKEN pnpm exec oxfmt --config .oxfmtrc.json packages/stack/src/Artifacts.ts + env -u APP_TOKEN -u GITHUB_TOKEN bun apps/cli/scripts/render-service-dockerfile.ts generated_files=( packages/stack/src/Artifacts.ts @@ -190,7 +188,7 @@ jobs: git add -- "${generated_files[@]}" git commit -m "$title" - if ! git push --force origin "HEAD:refs/heads/${branch}"; then + if ! git push --force "$push_url" "HEAD:refs/heads/${branch}"; then echo "::error ::Failed to push ${branch}. Run manually: ${manual_prefix} ${release}, regenerate the Dockerfiles with \`bun apps/cli/scripts/render-service-dockerfile.ts\`, then open a pull request by hand." exit 1 fi @@ -207,13 +205,15 @@ jobs: body="$(printf 'This %s %s from %s to %s.\n\nhttps://github.com/supabase/slim-services/releases/tag/%s-%s\n\nPlanned after supabase/slim-services published %s. This branch is rewritten from %s whenever a newer relevant release arrives.\n' \ "$action" "$SERVICE" "$from" "$release" "$SERVICE" "$release" "$RELEASE_VERSION" "$DEFAULT_BRANCH")" - existing="$(gh pr list --head "$branch" --base "$DEFAULT_BRANCH" --state open --json number --jq '.[0].number // empty')" + # `--head` matches by branch name only and ignores the owner, so a fork PR with the + # same head branch name would otherwise match too; `isCrossRepository` excludes it. + existing="$(GH_TOKEN="$APP_TOKEN" gh pr list --head "$branch" --base "$DEFAULT_BRANCH" --state open --json number,isCrossRepository --jq 'map(select(.isCrossRepository | not)) | .[0].number // empty')" if [ -n "$existing" ]; then - if ! gh pr edit "$existing" --title "$title" --body "$body"; then + if ! GH_TOKEN="$APP_TOKEN" gh pr edit "$existing" --title "$title" --body "$body"; then echo "::error ::Pushed ${branch} but failed to edit pull request #${existing}. Run manually: ${manual_prefix} ${release}, then edit the pull request from ${branch} by hand." exit 1 fi - elif ! gh pr create \ + elif ! GH_TOKEN="$APP_TOKEN" gh pr create \ --title "$title" \ --body "$body" \ --assignee jgoux \ From 9962b0ae37588cdee46e96bde0858b8eb7f48133 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 20:23:52 +0200 Subject: [PATCH 43/45] fix(ci): give the slim release pickup time for the S3 mirror wait --- .github/workflows/slim-release-published.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/slim-release-published.yml b/.github/workflows/slim-release-published.yml index 1c4f317634..9e90d9a362 100644 --- a/.github/workflows/slim-release-published.yml +++ b/.github/workflows/slim-release-published.yml @@ -34,7 +34,9 @@ concurrency: jobs: pickup: runs-on: ubuntu-latest - timeout-minutes: 20 + # Setup (~10 min) + release visibility (<1 min) + the S3-mirror wait (up to ~10 min, normally only + # for the first planned item) + pin, render and push per item. + timeout-minutes: 45 permissions: contents: read steps: From 0b6b1119b73a2d0b33c5810fb2059ad983e51c89 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 20:37:19 +0200 Subject: [PATCH 44/45] fix(cli): write the Vector config where Vector 0.58 images can hold it Vector 0.58's images (upstream and slim alike) ship no /etc/vector, so the legacy start entrypoint's heredoc write failed and crash-looped the container in both flag-off and slim supabase start. Create the directory before writing the config. --- .../commands/start/services/vector.service.ts | 15 +++--- .../services/vector.service.unit.test.ts | 2 +- .../start/start.lifecycle.e2e.test.ts | 54 +++++++++++++++++++ 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/apps/cli/src/commands/start/services/vector.service.ts b/apps/cli/src/commands/start/services/vector.service.ts index fac0f51757..ec49dab78b 100644 --- a/apps/cli/src/commands/start/services/vector.service.ts +++ b/apps/cli/src/commands/start/services/vector.service.ts @@ -222,11 +222,14 @@ const VECTOR_HEALTHCHECK = { } as const; /** - * Writes the rendered `vector.yaml` via a `cat <<'EOF'` heredoc, waits on Logflare's `/health` - * (sinks would otherwise start too early), then `exec`s Vector so it stays PID 1. A TERM trap - * covers the wait so `docker stop` does not burn 10s if Logflare is still down; `-T 2` bounds each - * probe so a hung health endpoint can't defer the trap. Slim Vector ships BusyBox wget, so the - * wait uses `-q --spider` instead of GNU's `--no-verbose --tries`. + * Creates `/etc/vector` (absent from Vector 0.58's images, both slim and upstream), writes the + * rendered `vector.yaml` via a `cat <<'EOF'` heredoc, waits on Logflare's `/health` (sinks would + * otherwise start too early), then `exec`s Vector so it stays PID 1. A TERM trap covers the wait + * so `docker stop` does not burn 10s if Logflare is still down; `-T 2` bounds each probe so a hung + * health endpoint can't defer the trap. Slim Vector ships BusyBox wget, so the wait uses + * `-q --spider` instead of GNU's `--no-verbose --tries`. Both images run as root, so `mkdir -p` + * needs no separate ownership handling, and `/var/lib/vector` (the `docker_logs` source's + * checkpoint `data_dir`) already exists in both. */ export function buildVectorEntrypointScript( vectorYaml: string, @@ -237,7 +240,7 @@ export function buildVectorEntrypointScript( ? slimWgetWaitCommand(`http://${logflareId}:4000/health`) : `wget --no-verbose --tries=1 -T 2 --spider http://${logflareId}:4000/health`; return ( - "cat <<'EOF' > /etc/vector/vector.yaml\n" + + "mkdir -p /etc/vector\ncat <<'EOF' > /etc/vector/vector.yaml\n" + vectorYaml + "\nEOF\ntrap 'exit 143' TERM\nuntil " + wget + diff --git a/apps/cli/src/commands/start/services/vector.service.unit.test.ts b/apps/cli/src/commands/start/services/vector.service.unit.test.ts index c82f2bae8b..10c149fc10 100644 --- a/apps/cli/src/commands/start/services/vector.service.unit.test.ts +++ b/apps/cli/src/commands/start/services/vector.service.unit.test.ts @@ -208,7 +208,7 @@ describe("resolveVectorDockerSocketPlan", () => { describe("buildVectorEntrypointScript", () => { test("writes vector.yaml then waits on Logflare's health endpoint before exec'ing vector (start.go:449-454)", () => { expect(buildVectorEntrypointScript("VECTOR_YAML", "supabase_analytics_proj")).toBe( - "cat <<'EOF' > /etc/vector/vector.yaml\n" + + "mkdir -p /etc/vector\ncat <<'EOF' > /etc/vector/vector.yaml\n" + "VECTOR_YAML" + "\nEOF\ntrap 'exit 143' TERM\nuntil wget --no-verbose --tries=1 -T 2 --spider http://" + "supabase_analytics_proj" + diff --git a/apps/cli/src/commands/start/start.lifecycle.e2e.test.ts b/apps/cli/src/commands/start/start.lifecycle.e2e.test.ts index c7429d744b..3d3a69a733 100644 --- a/apps/cli/src/commands/start/start.lifecycle.e2e.test.ts +++ b/apps/cli/src/commands/start/start.lifecycle.e2e.test.ts @@ -338,4 +338,58 @@ describe("supabase start (e2e)", () => { }).pipe(Effect.provide(BunServices.layer)), START_TIMEOUT_MS + LIFECYCLE_OVERHEAD_MS + CLEANUP_TIMEOUT_MS, ); + + // Regression coverage for the legacy (non-slim) `timberio/vector:0.58.0-alpine` image, which + // ships no `/etc/vector` — `buildVectorEntrypointScript` used to write its config straight into + // that nonexistent directory and crash-loop the container (start.go's own health wait then + // fails `start` itself). Only Postgres, Logflare, and Vector run; both are excludable in every + // other scenario here, so this is the only place that exercises them together. + it.live( + "starts Vector against a real Logflare and reaches healthy (CLI-2512: Vector 0.58 ships no /etc/vector)", + () => + Effect.gen(function* () { + const projectDir = yield* makeProject("sb-start-e2e-vector-"); + const path = yield* Path.Path; + const projectId = sanitizeProjectId(path.basename(projectDir)); + const vectorContainer = serviceContainerName("vector", projectId); + + const init = yield* runSupabaseEffect(["init"], { + cwd: projectDir, + exitTimeoutMs: SHORT_E2E_TIMEOUT_MS, + }); + requireCliSuccess(init, "init setup"); + yield* overridePorts(projectDir); + + const excludeArgs = SERVICE_CATALOG.flatMap((entry) => + entry.excludeKey === undefined || + entry.excludeKey === "logflare" || + entry.excludeKey === "vector" + ? [] + : ["--exclude", entry.excludeKey], + ); + const start = yield* runSupabaseEffect(["start", ...excludeArgs], { + cwd: projectDir, + exitTimeoutMs: START_TIMEOUT_MS, + }); + + if (start.exitCode !== 0) { + const logs = yield* runDockerEffect(["logs", vectorContainer]).pipe( + Effect.map(({ stdout, stderr }) => `${stdout}${stderr}`.trim() || ""), + Effect.catch((error) => Effect.succeed(``)), + ); + throw new Error( + `start failed (exit ${start.exitCode})\nstdout:\n${start.stdout}\nstderr:\n${start.stderr}\n${vectorContainer} logs:\n${logs}`, + ); + } + + const health = yield* runDockerEffect([ + "inspect", + vectorContainer, + "--format", + "{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}", + ]); + expect(health.stdout.trim()).toBe("healthy"); + }).pipe(Effect.provide(BunServices.layer)), + START_TIMEOUT_MS + LIFECYCLE_OVERHEAD_MS + CLEANUP_TIMEOUT_MS, + ); }); From 4446e85fadf24a06ee07307e12408977bab23bc0 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 20:40:36 +0200 Subject: [PATCH 45/45] test(cli): describe the Vector start scenario by what it covers --- apps/cli/src/commands/start/start.lifecycle.e2e.test.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/apps/cli/src/commands/start/start.lifecycle.e2e.test.ts b/apps/cli/src/commands/start/start.lifecycle.e2e.test.ts index 3d3a69a733..59b3468509 100644 --- a/apps/cli/src/commands/start/start.lifecycle.e2e.test.ts +++ b/apps/cli/src/commands/start/start.lifecycle.e2e.test.ts @@ -339,13 +339,10 @@ describe("supabase start (e2e)", () => { START_TIMEOUT_MS + LIFECYCLE_OVERHEAD_MS + CLEANUP_TIMEOUT_MS, ); - // Regression coverage for the legacy (non-slim) `timberio/vector:0.58.0-alpine` image, which - // ships no `/etc/vector` — `buildVectorEntrypointScript` used to write its config straight into - // that nonexistent directory and crash-loop the container (start.go's own health wait then - // fails `start` itself). Only Postgres, Logflare, and Vector run; both are excludable in every - // other scenario here, so this is the only place that exercises them together. + // Only Postgres, Logflare, and Vector run. Every other scenario here excludes Logflare and Vector, + // so this is the one that exercises the Vector image's entrypoint end to end. it.live( - "starts Vector against a real Logflare and reaches healthy (CLI-2512: Vector 0.58 ships no /etc/vector)", + "starts Vector against a real Logflare and reaches healthy", () => Effect.gen(function* () { const projectDir = yield* makeProject("sb-start-e2e-vector-");