feat(cli-v3): support offline build via TRIGGER_INDEX_OFFLINE + containerfile-module

packages/cli-v3/docs/build-offline.md

@@ -0,0 +1,65 @@
+# Offline-build flow (`TRIGGER_INDEX_OFFLINE=1`)
+
+This document covers the build pipeline for operators who need to build a
+tasks image **without calling out to a live trigger.dev API at build time**
+— e.g. air-gapped or regulated build environments where the build host
+cannot reach the webapp the image will eventually be deployed against.
+
+## How it works
+
+Two pieces compose:
+
+1. `TRIGGER_INDEX_OFFLINE=1` switches the in-image indexer to a stub
+   `CliApiClient` that writes deployment metadata to disk instead of
+   POST-ing it to the API. See
+   [`packages/cli-v3/src/entryPoints/managed-index-controller.ts`](../src/entryPoints/managed-index-controller.ts)
+   (`createOfflineCliApiClient`). The shim implements only the three
+   methods that `indexDeployment` actually calls:
+   - `getEnvironmentVariables()` → returns `{}` (no project env vars
+     reachable at build time)
+   - `createDeploymentBackgroundWorker(_id, body)` → writes
+     `index-metadata.json` to `process.cwd()`
+   - `failDeployment(_id, body)` → writes `index-error.json`
+2. The build script (a small `build.mjs` driven by `@trigger.dev/cli-v3`'s
+   `internal` subpath — see [`packages/cli-v3/src/internal.ts`](../src/internal.ts))
+   passes `offlineIndex: true` to `buildImage` and `--containerfile-module=<path>`
+   to use a custom Containerfile generator. See
+   [`packages/cli-v3/src/deploy/buildImage.ts`](../src/deploy/buildImage.ts) —
+   when `offlineIndex: true`, the generated Containerfile sets
+   `ARG TRIGGER_INDEX_OFFLINE` and forwards it as a build arg into the
+   indexer stage; the final stage copies `/app/index-metadata.json` into
+   the runtime image at `/app/`.
+
+The metadata gets baked into the runtime image, then re-played against
+the real webapp API later by a separate **register** process running
+inside the cluster (where it _can_ reach the webapp). The register
+process uses the same image, reads `/app/index-metadata.json`, hits the
+real webapp API via `getProjectClient`, and finalizes the deployment.
+
+## Operator responsibilities
+
+- Provide a `containerfileModule` if the default Containerfile doesn't fit
+  (e.g. base on UBI / chainguard / a FIPS-validated runtime). The module's
+  job is to emit the multi-stage Containerfile text that invokes the
+  indexer stage with `TRIGGER_INDEX_OFFLINE=1` and copies
+  `/app/index-metadata.json` into the final image.
+- Run your `build.mjs` equivalent on the build host (no API access
+  required). The output is just the tasks image pushed to your registry.
+- Run your `register.mjs` equivalent on the deploy host (Kubernetes Job /
+  systemd unit / etc.) using **the same image**. It reads
+  `/app/index-metadata.json` and re-issues `createDeploymentBackgroundWorker`
+  against the live webapp.
+
+## Caveats
+
+- Project env vars are **not** available at build time in offline mode.
+  Any task whose indexing step depends on a project env var will index
+  with `{}`. If your tasks need env vars at index time, use the online
+  flow.
+- The offline shim returns synthetic `id: "offline"` / `version: "offline"`
+  from `createDeploymentBackgroundWorker`. Anything in your build pipeline
+  reading those fields must be aware that the real id/version is assigned
+  later, by the register step.
+- `TRIGGER_INDEX_OFFLINE=1` only affects the indexer stage. The rest of
+  the runtime (the tasks themselves) still talks to the live API at
+  runtime as normal.

packages/cli-v3/src/build/buildWorker.ts

@@ -39,6 +39,7 @@ export type BuildWorkerOptions = {
   rewritePaths?: boolean;
   forcedExternals?: string[];
   plain?: boolean;
+  containerfileModule?: string;
 };
 
 export async function buildWorker(options: BuildWorkerOptions) {
@@ -137,6 +138,7 @@ export async function buildWorker(options: BuildWorkerOptions) {
       resolvedConfig,
       outputPath: options.destination,
       bundleResult,
+      containerfileModule: options.containerfileModule,
     });
   }
 
@@ -196,11 +198,13 @@ async function writeDeployFiles({
   resolvedConfig,
   outputPath,
   bundleResult,
+  containerfileModule,
 }: {
   buildManifest: BuildManifest;
   resolvedConfig: ResolvedConfig;
   outputPath: string;
   bundleResult: BundleResult;
+  containerfileModule?: string;
 }) {
   // Step 1. Read the package.json file
   const packageJson = await readProjectPackageJson(resolvedConfig.packageJsonPath);
@@ -237,7 +241,7 @@ async function writeDeployFiles({
   );
 
   await writeJSONFile(join(outputPath, "build.json"), buildManifestToJSON(buildManifest));
-  await writeContainerfile(outputPath, buildManifest);
+  await writeContainerfile(outputPath, buildManifest, containerfileModule);
 }
 
 async function readProjectPackageJson(packageJsonPath: string) {
@@ -246,7 +250,11 @@ async function readProjectPackageJson(packageJsonPath: string) {
   return packageJson;
 }
 
-async function writeContainerfile(outputPath: string, buildManifest: BuildManifest) {
+async function writeContainerfile(
+  outputPath: string,
+  buildManifest: BuildManifest,
+  containerfileModule?: string
+) {
   if (!buildManifest.runControllerEntryPoint || !buildManifest.indexControllerEntryPoint) {
     throw new Error("Something went wrong with the build. Aborting deployment. [code 7789]");
   }
@@ -257,6 +265,7 @@ async function writeContainerfile(outputPath: string, buildManifest: BuildManife
     build: buildManifest.build,
     image: buildManifest.image,
     indexScript: buildManifest.indexControllerEntryPoint,
+    containerfileModule,
   });
 
   const containerfilePath = join(outputPath, "Containerfile");

packages/cli-v3/src/deploy/buildImage.ts

@@ -3,7 +3,7 @@ import { depot } from "@depot/cli";
 import { x } from "tinyexec";
 import { BuildManifest, BuildRuntime } from "@trigger.dev/core/v3/schemas";
 import { networkInterfaces } from "os";
-import { join } from "path";
+import { join, resolve } from "path";
 import { safeReadJSONFile } from "../utilities/fileSystem.js";
 import { readFileSync } from "fs";
 
@@ -12,6 +12,8 @@ import { z } from "zod";
 import { assertExhaustive } from "../utilities/assertExhaustive.js";
 import { tryCatch } from "@trigger.dev/core";
 import { CliApiClient } from "../apiClient.js";
+import { pathToFileURL } from "url";
+import type { ContainerfileTemplate } from "./containerfile-template.js";
 
 export interface BuildImageOptions {
   // Common options
@@ -46,9 +48,10 @@ export interface BuildImageOptions {
   extraCACerts?: string;
   apiUrl: string;
   apiKey: string;
-  apiClient: CliApiClient;
+  apiClient?: CliApiClient;
   branchName?: string;
   buildEnvVars?: Record<string, string | undefined>;
+  offlineIndex?: boolean; // When true, skip API-based indexing in the container
   onLog?: (log: string) => void;
 
   // Optional deployment spinner
@@ -81,6 +84,7 @@ export async function buildImage(options: BuildImageOptions): Promise<BuildImage
     apiClient,
     branchName,
     buildEnvVars,
+    offlineIndex,
     network,
     builder,
     compression,
@@ -111,6 +115,7 @@ export async function buildImage(options: BuildImageOptions): Promise<BuildImage
       apiClient,
       branchName,
       buildEnvVars,
+      offlineIndex,
       network,
       builder,
       compression,
@@ -336,12 +341,13 @@ interface SelfHostedBuildImageOptions {
   authenticateToRegistry?: boolean;
   apiUrl: string;
   apiKey: string;
-  apiClient: CliApiClient;
+  apiClient?: CliApiClient;
   branchName?: string;
   noCache?: boolean;
   useRegistryCache?: boolean;
   extraCACerts?: string;
   buildEnvVars?: Record<string, string | undefined>;
+  offlineIndex?: boolean;
   network?: string;
   builder: string;
   load?: boolean;
@@ -487,6 +493,14 @@ async function localBuildImage(options: SelfHostedBuildImageOptions): Promise<Bu
       };
     }
 
+    if (!apiClient) {
+      return {
+        ok: false as const,
+        error: "API client is required for registry authentication",
+        logs: "",
+      };
+    }
+
     const [credentialsError, credentials] = await tryCatch(
       getDockerUsernameAndPassword(apiClient, deploymentId)
     );
@@ -586,6 +600,7 @@ async function localBuildImage(options: SelfHostedBuildImageOptions): Promise<Bu
     `TRIGGER_PREVIEW_BRANCH=${options.branchName ?? ""}`,
     "--build-arg",
     `TRIGGER_SECRET_KEY=${options.apiKey}`,
+    ...(options.offlineIndex ? ["--build-arg", "TRIGGER_INDEX_OFFLINE=1"] : []),
     ...(buildArgs || []),
     ...(options.extraCACerts ? ["--build-arg", `NODE_EXTRA_CA_CERTS=${options.extraCACerts}`] : []),
     "--progress",
@@ -685,6 +700,7 @@ export type GenerateContainerfileOptions = {
   image: BuildManifest["image"];
   indexScript: string;
   entrypoint: string;
+  containerfileModule?: string;
 };
 
 const BASE_IMAGE: Record<BuildRuntime, string> = {
@@ -696,7 +712,44 @@ const BASE_IMAGE: Record<BuildRuntime, string> = {
 
 const DEFAULT_PACKAGES = ["busybox", "ca-certificates", "dumb-init", "git", "openssl"];
 
+async function loadContainerfileModule(modulePath: string): Promise<ContainerfileTemplate> {
+  const absolutePath = resolve(modulePath);
+
+  try {
+    // Convert to file URL for proper ESM import
+    const moduleUrl = pathToFileURL(absolutePath).href;
+
+    // Dynamic import of the module
+    const module = await import(moduleUrl);
+
+    // Return the default export
+    if (!module.default) {
+      throw new Error(`Module ${modulePath} does not have a default export`);
+    }
+
+    return module.default;
+  } catch (error) {
+    logger.error(`Failed to load containerfile module from ${modulePath}`, error);
+    throw new Error(`Failed to load containerfile module: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+
 export async function generateContainerfile(options: GenerateContainerfileOptions) {
+  // If a custom module is specified, use it
+  if (options.containerfileModule) {
+    try {
+      const template = await loadContainerfileModule(options.containerfileModule);
+
+      // Pass the full options directly to the template for complete control
+      const containerfile = await template.generate(options);
+      return containerfile;
+    } catch (error) {
+      logger.error("Failed to generate containerfile from module", error);
+      throw error;
+    }
+  }
+
+  // Fall back to built-in templates
   switch (options.runtime) {
     case "node":
     case "node-22": {
@@ -788,6 +841,7 @@ ARG NODE_EXTRA_CA_CERTS
 ARG TRIGGER_SECRET_KEY
 ARG TRIGGER_API_URL
 ARG TRIGGER_PREVIEW_BRANCH
+ARG TRIGGER_INDEX_OFFLINE
 
 ENV TRIGGER_PROJECT_ID=\${TRIGGER_PROJECT_ID} \
     TRIGGER_DEPLOYMENT_ID=\${TRIGGER_DEPLOYMENT_ID} \
@@ -798,6 +852,7 @@ ENV TRIGGER_PROJECT_ID=\${TRIGGER_PROJECT_ID} \
     TRIGGER_API_URL=\${TRIGGER_API_URL} \
     TRIGGER_PREVIEW_BRANCH=\${TRIGGER_PREVIEW_BRANCH} \
     NODE_EXTRA_CA_CERTS=\${NODE_EXTRA_CA_CERTS} \
+    TRIGGER_INDEX_OFFLINE=\${TRIGGER_INDEX_OFFLINE} \
     NODE_ENV=production
 
 ARG TARGETPLATFORM
@@ -896,6 +951,7 @@ ARG NODE_EXTRA_CA_CERTS
 ARG TRIGGER_SECRET_KEY
 ARG TRIGGER_API_URL
 ARG TRIGGER_PREVIEW_BRANCH
+ARG TRIGGER_INDEX_OFFLINE
 
 ENV TRIGGER_PROJECT_ID=\${TRIGGER_PROJECT_ID} \
     TRIGGER_DEPLOYMENT_ID=\${TRIGGER_DEPLOYMENT_ID} \
@@ -907,6 +963,7 @@ ENV TRIGGER_PROJECT_ID=\${TRIGGER_PROJECT_ID} \
     TRIGGER_PREVIEW_BRANCH=\${TRIGGER_PREVIEW_BRANCH} \
     TRIGGER_LOG_LEVEL=debug \
     NODE_EXTRA_CA_CERTS=\${NODE_EXTRA_CA_CERTS} \
+    TRIGGER_INDEX_OFFLINE=\${TRIGGER_INDEX_OFFLINE} \
     NODE_ENV=production \
     NODE_OPTIONS="--max_old_space_size=8192"
 
@@ -938,8 +995,11 @@ ENV TRIGGER_PROJECT_ID=\${TRIGGER_PROJECT_ID} \
 # Copy the files from the install stage
 COPY --from=build --chown=node:node /app ./
 
-# Copy the index.json file from the indexer stage
+# Copy the index.json file from the indexer stage if it exists
 COPY --from=indexer --chown=node:node /app/index.json ./
+COPY --from=indexer --chown=node:node /app/index-metadata.json ./
+# index-error.json is optional - it only exists if indexing failed
+COPY --from=indexer --chown=node:node /app/index-error.json* ./
 
 ENTRYPOINT [ "dumb-init", "node", "${options.entrypoint}" ]
 CMD []

packages/cli-v3/src/deploy/containerfile-template.ts

@@ -0,0 +1,5 @@
+import type { GenerateContainerfileOptions } from "./buildImage.js";
+
+export interface ContainerfileTemplate {
+  generate(options: GenerateContainerfileOptions): Promise<string> | string;
+}