feat(webapp,cli): let self-hosted instances require deploy base images (#5005)

Closes #5004

Lets a self-hosted operator require custom base images for every deploy
to their instance, such as FIPS-validated or hardened Node images. Cloud
never sets the variables, so nothing changes there.

```bash
DEPLOY_BASE_IMAGES="node-26=registry.example.com/node-fips:26@sha256:<digest>"
DEPLOY_BUILD_BASE_IMAGES="node-26=registry.example.com/node:26-dev@sha256:<digest>"  # optional
```

- Webapp validates both variables at startup (known runtime,
digest-pinned, no duplicates) and fails loud on bad values.
- Deployments return the images for their runtime as `baseImages`; the
CLI rewrites the Containerfile with them. A configured build image is
also used when a project has `image.instructions`.
- Deploys from CLIs that can't apply them, and `--native-build`,
`--local-bundle` and `--from-bundle` deploys, are rejected with a clear
error.
- Docs: env rows plus a "Custom base images" section in the self-hosting
overview.

## Testing

`apps/webapp/test/deployBaseImages.test.ts` and
`packages/cli-v3/src/deploy/buildImage.test.ts` cover the parsing,
rejection and Containerfile cases. Typecheck, format and lint are clean.

---------

Co-authored-by: nicktrn <55853254+nicktrn@users.noreply.github.com>

Author: brentshulman-silkline

.changeset/instance-deploy-base-images.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/core": patch
+"trigger.dev": patch
+---
+
+Self-hosted instances can require custom deploy base images per runtime via the new `DEPLOY_BASE_IMAGES` and `DEPLOY_BUILD_BASE_IMAGES` webapp settings. The CLI builds on the images the instance specifies, and older CLIs are rejected with an upgrade message.

apps/webapp/app/env.server.ts

@@ -5,6 +5,7 @@ import {
 } from "@trigger.dev/core/v3/isomorphic";
 import { BoolEnv } from "./utils/boolEnv";
 import { isValidDatabaseUrl } from "./utils/db";
+import { parseDeployBaseImages } from "~/v3/deployBaseImages.server";
 import { parseRunOpsShards, validateShardListAgainstNewUrl } from "~/v3/runOpsShards.server";
 import { isValidRegex } from "./utils/regex";
 import { isValidDuration } from "./services/realtime/duration.server";
@@ -16,6 +17,18 @@ function durationString() {
   return z.string().refine(isValidDuration, "must be a duration like 7d, 30d, 365d, 1h, 1y");
 }
 
+const parseDeployBaseImagesEnv = (
+  value: string | undefined,
+  envVarName: string,
+  ctx: z.RefinementCtx
+) => {
+  const { images, errors } = parseDeployBaseImages(value, envVarName);
+  for (const message of errors) {
+    ctx.addIssue({ code: z.ZodIssueCode.custom, message });
+  }
+  return errors.length > 0 ? z.NEVER : images;
+};
+
 const GithubAppEnvSchema = z.preprocess(
   (val) => {
     const obj = val as any;
@@ -856,6 +869,14 @@ const EnvironmentSchema = z
       .transform((v) => v ?? process.env.DEPLOY_REGISTRY_ECR_DEFAULT_REPOSITORY_POLICY),
 
     DEPLOY_IMAGE_PLATFORM: z.string().default("linux/amd64"),
+    DEPLOY_BASE_IMAGES: z
+      .string()
+      .optional()
+      .transform((v, ctx) => parseDeployBaseImagesEnv(v, "DEPLOY_BASE_IMAGES", ctx)),
+    DEPLOY_BUILD_BASE_IMAGES: z
+      .string()
+      .optional()
+      .transform((v, ctx) => parseDeployBaseImagesEnv(v, "DEPLOY_BUILD_BASE_IMAGES", ctx)),
     DEPLOY_TIMEOUT_MS: z.coerce
       .number()
       .int()

apps/webapp/app/routes/api.v1.deployments.$deploymentId.ts

@@ -2,6 +2,8 @@ import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime";
 import { type GetDeploymentResponseBody } from "@trigger.dev/core/v3";
 import { z } from "zod";
 import { prisma } from "~/db.server";
+import { env } from "~/env.server";
+import { resolveDeployBaseImages } from "~/v3/deployBaseImages.server";
 import { authenticateApiKeyWithScope } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 
@@ -65,6 +67,10 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
       externalId: deployment.externalId ?? undefined,
       externalBuildData:
         deployment.externalBuildData as GetDeploymentResponseBody["externalBuildData"],
+      baseImages: resolveDeployBaseImages(deployment.runtime, {
+        base: env.DEPLOY_BASE_IMAGES,
+        buildBase: env.DEPLOY_BUILD_BASE_IMAGES,
+      }),
       errorData: deployment.errorData as GetDeploymentResponseBody["errorData"],
       canceledReason: deployment.canceledReason,
       worker: deployment.worker

apps/webapp/app/routes/api.v1.deployments.ts

@@ -9,6 +9,8 @@ import { authenticateApiKeyWithScope } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server";
 import { ServiceValidationError } from "~/v3/services/baseService.server";
+import { env } from "~/env.server";
+import { resolveDeployBaseImages } from "~/v3/deployBaseImages.server";
 import { InitializeDeploymentService } from "~/v3/services/initializeDeployment.server";
 
 export async function action({ request, params }: ActionFunctionArgs) {
@@ -60,6 +62,10 @@ export async function action({ request, params }: ActionFunctionArgs) {
         ? {
             externalBuildData: result.deployment
               .externalBuildData as InitializeDeploymentResponseBody["externalBuildData"],
+            baseImages: resolveDeployBaseImages(result.deployment.runtime, {
+              base: env.DEPLOY_BASE_IMAGES,
+              buildBase: env.DEPLOY_BUILD_BASE_IMAGES,
+            }),
             eventStream: result.eventStream,
             canceledDeployments: result.canceledDeployments,
           }

apps/webapp/app/v3/deployBaseImages.server.ts

@@ -0,0 +1,77 @@
+import { BuildRuntime, DeployBaseImageRef } from "@trigger.dev/core/v3";
+
+type BaseImageMap = Partial<Record<BuildRuntime, string>>;
+
+export function parseDeployBaseImages(
+  value: string | undefined,
+  envVarName: string
+): { images: BaseImageMap; errors: string[] } {
+  const images: BaseImageMap = {};
+  const errors: string[] = [];
+
+  if (!value) {
+    return { images, errors };
+  }
+
+  for (const segment of value.split(",").map((s) => s.trim())) {
+    if (!segment) {
+      continue;
+    }
+
+    const fail = (reason: string) => errors.push(`${envVarName}: ${reason} in "${segment}"`);
+
+    const separator = segment.indexOf("=");
+    if (separator === -1) {
+      fail("expected runtime=image");
+      continue;
+    }
+
+    const runtimeName = segment.slice(0, separator).trim();
+    const image = segment.slice(separator + 1).trim();
+
+    if (runtimeName === "node") {
+      fail('runtime "node" is an alias; use the concrete runtime (node-22, node-24, node-26)');
+      continue;
+    }
+
+    const runtime = BuildRuntime.safeParse(runtimeName);
+    if (!runtime.success) {
+      fail(`unknown runtime "${runtimeName}" (expected one of ${BuildRuntime.options.join(", ")})`);
+      continue;
+    }
+
+    if (!image) {
+      fail("missing image");
+      continue;
+    }
+
+    if (!DeployBaseImageRef.safeParse(image).success) {
+      fail("image must be image@sha256:<64 hex chars> with no whitespace before the digest");
+      continue;
+    }
+
+    if (runtime.data in images) {
+      fail(`duplicate runtime "${runtimeName}"`);
+      continue;
+    }
+
+    images[runtime.data] = image;
+  }
+
+  return { images, errors };
+}
+
+export function resolveDeployBaseImages(
+  runtime: string | null | undefined,
+  config: { base: BaseImageMap; buildBase: BaseImageMap }
+): { base?: string; buildBase?: string } | undefined {
+  const parsedRuntime = BuildRuntime.safeParse(runtime);
+  if (!parsedRuntime.success) {
+    return undefined;
+  }
+
+  const base = config.base[parsedRuntime.data];
+  const buildBase = config.buildBase[parsedRuntime.data];
+
+  return base || buildBase ? { base, buildBase } : undefined;
+}

apps/webapp/app/v3/services/initializeDeployment.server.ts

@@ -14,6 +14,7 @@ import { generateFriendlyId } from "../friendlyIdentifiers";
 import { createRemoteImageBuild, remoteBuildsEnabled } from "../remoteImageBuilder.server";
 import { BaseService, ServiceValidationError } from "./baseService.server";
 import { TimeoutDeploymentService } from "./timeoutDeployment.server";
+import { resolveDeployBaseImages } from "../deployBaseImages.server";
 import { getDeploymentImageRef } from "../getDeploymentImageRef.server";
 import { tryCatch } from "@trigger.dev/core";
 import { getRegistryConfig } from "../registryConfig.server";
@@ -147,6 +148,25 @@ export class InitializeDeploymentService extends BaseService {
         throw new ServiceValidationError("UNMANAGED deployments are not supported");
       }
 
+      const requiredBaseImages = resolveDeployBaseImages(runtime, {
+        base: env.DEPLOY_BASE_IMAGES,
+        buildBase: env.DEPLOY_BUILD_BASE_IMAGES,
+      });
+
+      if (requiredBaseImages && payload.isNativeBuild) {
+        throw new ServiceValidationError(
+          "This instance requires custom deploy base images, which native builds cannot apply. Deploy without --native-build or --local-bundle.",
+          400
+        );
+      }
+
+      if (requiredBaseImages && payload.supportsInstanceBaseImages !== true) {
+        throw new ServiceValidationError(
+          "This instance requires custom deploy base images, which this version of the CLI cannot apply. Upgrade the trigger.dev CLI and deploy again.",
+          400
+        );
+      }
+
       // Upgrade the project to engine "V2" if it's not already. This should cover cases where people deploy to V2 without running dev first.
       if (payload.type === "MANAGED" && environment.project.engine === "V1") {
         await this._prisma.project.update({

apps/webapp/test/deployBaseImages.test.ts

@@ -0,0 +1,114 @@
+import { describe, expect, it } from "vitest";
+import { parseDeployBaseImages, resolveDeployBaseImages } from "~/v3/deployBaseImages.server";
+
+const digestA = `sha256:${"a".repeat(64)}`;
+const digestB = `sha256:${"b".repeat(64)}`;
+const digestC = `sha256:${"c".repeat(64)}`;
+
+describe("parseDeployBaseImages", () => {
+  it("returns an empty map for undefined and empty values", () => {
+    const empty = { images: {}, errors: [] };
+    expect(parseDeployBaseImages(undefined, "DEPLOY_BASE_IMAGES")).toEqual(empty);
+    expect(parseDeployBaseImages("", "DEPLOY_BASE_IMAGES")).toEqual(empty);
+    expect(parseDeployBaseImages(" , ,", "DEPLOY_BASE_IMAGES")).toEqual(empty);
+  });
+
+  it("parses multiple entries and trims whitespace", () => {
+    expect(
+      parseDeployBaseImages(
+        ` node-24 = acme/node-fips:24@${digestA} , bun=acme/bun:1@${digestB},`,
+        "DEPLOY_BASE_IMAGES"
+      )
+    ).toEqual({
+      images: {
+        "node-24": `acme/node-fips:24@${digestA}`,
+        bun: `acme/bun:1@${digestB}`,
+      },
+      errors: [],
+    });
+  });
+
+  it("accepts a registry with a port and a tag before the digest", () => {
+    const image = `registry.example.com:5000/ns/img:tag@${digestA}`;
+    expect(parseDeployBaseImages(`node-24=${image}`, "DEPLOY_BASE_IMAGES")).toEqual({
+      images: { "node-24": image },
+      errors: [],
+    });
+  });
+
+  it.each([
+    ["missing =", "garbage"],
+    ["unknown runtime", `node-23=acme/node:23@${digestA}`],
+    ["node alias", `node=acme/node:24@${digestA}`],
+    ["empty image", "node-24="],
+    ["missing digest", "node-24=acme/node:24"],
+    ["flag before image", `node-24=--platform=linux/arm64 acme/node@${digestA}`],
+    ["bare digest", `node-24=@${digestA}`],
+    ["newline in image", `node-24=acme/node\nx@${digestA}`],
+    ["duplicate runtime", `node-24=acme/a@${digestA},node-24=acme/b@${digestB}`],
+  ])("reports an error naming the env var and segment: %s", (_name, value) => {
+    const segments = value.split(",");
+    const offending = segments[segments.length - 1]!.trim();
+
+    const { images, errors } = parseDeployBaseImages(
+      `node-22=acme/ok@${digestC},${value}`,
+      "DEPLOY_BUILD_BASE_IMAGES"
+    );
+
+    expect(images["node-22"]).toBe(`acme/ok@${digestC}`);
+    expect(errors).toHaveLength(1);
+    expect(errors[0]).toContain("DEPLOY_BUILD_BASE_IMAGES");
+    expect(errors[0]).toContain(offending);
+  });
+
+  it("explains that node is an alias", () => {
+    const { errors } = parseDeployBaseImages(`node=acme/node@${digestA}`, "DEPLOY_BASE_IMAGES");
+    expect(errors[0]).toContain('runtime "node" is an alias; use the concrete runtime');
+  });
+
+  it("reports every bad segment", () => {
+    const { errors } = parseDeployBaseImages(
+      `garbage,node-23=acme/node@${digestA},bun=acme/bun`,
+      "DEPLOY_BASE_IMAGES"
+    );
+    expect(errors).toHaveLength(3);
+    expect(errors[0]).toContain("garbage");
+    expect(errors[1]).toContain("node-23");
+    expect(errors[2]).toContain("bun=acme/bun");
+  });
+});
+
+describe("resolveDeployBaseImages", () => {
+  const base = { "node-26": `acme/node-fips:26@${digestA}` } as const;
+  const buildBase = { "node-26": `acme/node:26-dev@${digestB}` } as const;
+
+  it("returns undefined for a missing or unknown runtime", () => {
+    expect(resolveDeployBaseImages("node-23", { base, buildBase })).toBeUndefined();
+    expect(resolveDeployBaseImages(null, { base, buildBase })).toBeUndefined();
+    expect(resolveDeployBaseImages(undefined, { base, buildBase })).toBeUndefined();
+  });
+
+  it("returns undefined when the runtime has no entries", () => {
+    expect(resolveDeployBaseImages("bun", { base, buildBase })).toBeUndefined();
+    expect(resolveDeployBaseImages("node-26", { base: {}, buildBase: {} })).toBeUndefined();
+  });
+
+  it("returns both images", () => {
+    expect(resolveDeployBaseImages("node-26", { base, buildBase })).toEqual({
+      base: base["node-26"],
+      buildBase: buildBase["node-26"],
+    });
+  });
+
+  it("returns only the base image", () => {
+    expect(resolveDeployBaseImages("node-26", { base, buildBase: {} })).toEqual({
+      base: base["node-26"],
+    });
+  });
+
+  it("returns only the build base image", () => {
+    expect(resolveDeployBaseImages("node-26", { base: {}, buildBase })).toEqual({
+      buildBase: buildBase["node-26"],
+    });
+  });
+});

docs/self-hosting/env/webapp.mdx

@@ -101,6 +101,8 @@ mode: "wide"
 | `DEPLOY_REGISTRY_NAMESPACE`                      | No       | trigger               | Deploy registry namespace.                                                                                         |
 | `DEPLOY_REGISTRY_ECR_DEFAULT_REPOSITORY_POLICY`  | No       | —                     | Raw IAM policy JSON applied via SetRepositoryPolicy to every ECR repo created by the webapp. Use to grant cross-account pull access to EKS workers when the ECR account is separate from the cluster account. |
 | `DEPLOY_IMAGE_PLATFORM`                          | No       | linux/amd64           | Deploy image platform, same values as docker `--platform` flag.                                                    |
+| `DEPLOY_BASE_IMAGES`                             | No       | —                     | Base images every deploy must build on, as comma-separated `runtime=image@sha256:<digest>`. See [custom base images](/self-hosting/overview#custom-base-images). |
+| `DEPLOY_BUILD_BASE_IMAGES`                       | No       | —                     | Build-stage images, in the same format as `DEPLOY_BASE_IMAGES`. See [custom base images](/self-hosting/overview#custom-base-images). |
 | `DEPLOY_TIMEOUT_MS`                              | No       | 480000 (8m)           | Deploy timeout (ms).                                                                                               |
 | `DEPLOY_QUEUE_TIMEOUT_MS`                        | No       | 900000 (15m)          | Deploy queue timeout (ms).                                                                                         |
 | **Object store (S3)**                            |          |                       |                                                                                                                    |

docs/self-hosting/overview.mdx

@@ -100,6 +100,32 @@ All fields are optional. Partial overrides are supported:
 }
 ```
 
+## Custom base images
+
+Deploys build on the published `triggerdotdev/node` and `triggerdotdev/bun` Debian images. To require a different base for every deploy to your instance, such as a FIPS-validated or hardened Node image, set `DEPLOY_BASE_IMAGES` on the webapp (and optionally `DEPLOY_BUILD_BASE_IMAGES` for the build stage):
+
+```bash
+DEPLOY_BASE_IMAGES="node-26=registry.example.com/node-fips:26@sha256:<64-character-digest>"
+```
+
+Entries are comma-separated `runtime=image@sha256:<digest>`. The runtimes are `node-22`, `node-24`, `node-26` and `bun`, and every image must be pinned by digest. An invalid value prevents the webapp from starting. Projects with `runtime: "node"` in their config resolve to the current default Node runtime (`node-24` today), so set that key for them. With the Helm chart, set the variables through `webapp.extraEnvVars`.
+
+Runtimes with an entry build on that image; the others keep the published images. Deploys from CLI versions that cannot apply the images are rejected with an error asking to upgrade, and so are `--native-build`, `--local-bundle` and `--from-bundle` deploys. This is a self-hosting setting and does not apply to Trigger.dev Cloud.
+
+You own a custom base image. It must provide:
+
+- `node` on `PATH` at the runtime's major version (for `bun`, both `bun` and `node`, because the final stage starts the app with `dumb-init node`)
+- `busybox`, `ca-certificates`, `dumb-init`, `git` and `openssl`
+- a `node` user (a `bun` user for the Bun runtime)
+- glibc, so native modules built in the build stage load at runtime
+
+A `DEPLOY_BUILD_BASE_IMAGES` image needs everything the base image provides, plus `python3`, `make` and `g++`. Build extensions that add image instructions are replayed on it. Without an entry, the build stage uses the published build image, except for projects whose build extensions add image instructions: those build from your base image and install the toolchain with `apt-get`, so that base must be Debian-based. To avoid the published images entirely, set both variables.
+
+<Warning>
+  `image.pkgs` and build extensions that run `apt-get` (such as `aptGet` and `playwright`) assume a
+  Debian base. On other distributions, install those packages in your base image instead.
+</Warning>
+
 ## Community support
 
 It's dangerous to go alone! Join the self-hosting channel on our [Discord server](https://discord.gg/NQTxt5NA7s).

packages/cli-v3/src/build/buildWorker.ts

@@ -276,7 +276,7 @@ async function readProjectPackageJson(packageJsonPath: string) {
   return packageJson;
 }
 
-async function writeContainerfile(outputPath: string, buildManifest: BuildManifest) {
+export async function writeContainerfile(outputPath: string, buildManifest: BuildManifest) {
   if (!buildManifest.runControllerEntryPoint || !buildManifest.indexControllerEntryPoint) {
     throw new Error("Something went wrong with the build. Aborting deployment. [code 7789]");
   }