diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index f2d54741eee..a8007bf835a 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -1,7 +1,7 @@ import { randomUUID } from "crypto"; import { env as stdEnv } from "std-env"; import { z } from "zod"; -import { AdditionalEnvVars, BoolEnv } from "./envUtil.js"; +import { AdditionalEnvVars, BoolEnv, JsonAny, JsonObjectEnv } from "./envUtil.js"; const Env = z .object({ @@ -92,6 +92,37 @@ const Env = z KUBERNETES_FORCE_ENABLED: BoolEnv.default(false), KUBERNETES_NAMESPACE: z.string().default("default"), KUBERNETES_WORKER_NODETYPE_LABEL: z.string().default("v4-worker"), + KUBERNETES_WORKER_SERVICE_ACCOUNT: z.string().optional(), // Service account for worker pods + KUBERNETES_WORKER_AUTOMOUNT_SERVICE_ACCOUNT_TOKEN: BoolEnv.default(false), // Whether to mount SA token + // Extra annotations to apply to every worker pod (e.g. for service mesh + // sidecar injection, certificate injection, scheduling hints). + KUBERNETES_WORKER_POD_ANNOTATIONS: JsonObjectEnv("KUBERNETES_WORKER_POD_ANNOTATIONS"), + // Pod-level securityContext applied to every worker pod (V1PodSecurityContext shape). + // Default is empty `{}`, preserving the upstream behavior of not setting + // a pod-level securityContext. Provide a JSON object to enforce e.g. + // `{"runAsNonRoot": true, "runAsUser": 1000, "fsGroup": 1000}`. + // OpenShift and other clusters with arbitrary-UID SCCs typically want + // to leave this empty and let the SCC inject values. + KUBERNETES_WORKER_POD_SECURITY_CONTEXT: JsonObjectEnv("KUBERNETES_WORKER_POD_SECURITY_CONTEXT", { + valueValidator: JsonAny, + }), + // Container-level securityContext applied to the worker container of every + // worker pod (V1SecurityContext shape). Default is empty `{}` (matches + // upstream's previous behavior of not setting a container securityContext). + // Provide a JSON object to enforce e.g. + // `{"runAsNonRoot": true, "runAsUser": 1000, "allowPrivilegeEscalation": false, + // "capabilities": {"drop": ["ALL"]}, "seccompProfile": {"type": "RuntimeDefault"}}`. + KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT: JsonObjectEnv( + "KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT", + { valueValidator: JsonAny } + ), + // Name of a Kubernetes Secret to envFrom-mount into every worker pod's + // container. Pulls every key/value pair in the secret as env vars on + // the worker. Resolved by the kubelet at pod creation time; the + // supervisor never reads the secret values, so this needs no extra + // RBAC. Use case: keep task-time secrets (DB URLs, API keys) in + // Kubernetes rather than syncing them through the trigger.dev webapp. + KUBERNETES_WORKER_ENV_FROM_SECRET: z.string().optional(), KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"), KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"), diff --git a/apps/supervisor/src/envUtil.test.ts b/apps/supervisor/src/envUtil.test.ts index c3d35758f16..7708deadc49 100644 --- a/apps/supervisor/src/envUtil.test.ts +++ b/apps/supervisor/src/envUtil.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect } from "vitest"; -import { BoolEnv, AdditionalEnvVars } from "./envUtil.js"; +import { z } from "zod"; +import { BoolEnv, AdditionalEnvVars, JsonObjectEnv, JsonAny } from "./envUtil.js"; describe("BoolEnv", () => { it("should parse string 'true' as true", () => { @@ -78,3 +79,78 @@ describe("AdditionalEnvVars", () => { }); }); }); + +describe("JsonObjectEnv (string-valued)", () => { + const schema = JsonObjectEnv("TEST_ENV"); + + it("returns empty object for default (no value)", () => { + expect(schema.parse(undefined)).toEqual({}); + }); + + it("parses a simple string-valued JSON object", () => { + expect(schema.parse('{"a":"1","b":"2"}')).toEqual({ a: "1", b: "2" }); + }); + + it("parses an empty JSON object", () => { + expect(schema.parse("{}")).toEqual({}); + }); + + it("rejects non-JSON input", () => { + expect(() => schema.parse("not json")).toThrowError(/not valid JSON/); + }); + + it("rejects JSON arrays", () => { + expect(() => schema.parse("[]")).toThrowError(/must be a JSON object \(got array\)/); + }); + + it("rejects JSON primitives", () => { + expect(() => schema.parse('"foo"')).toThrowError(/must be a JSON object \(got string\)/); + expect(() => schema.parse("42")).toThrowError(/must be a JSON object \(got number\)/); + expect(() => schema.parse("null")).toThrowError(/must be a JSON object \(got object\)/); + }); + + it("rejects values that are not strings (with default validator)", () => { + expect(() => schema.parse('{"a": 1}')).toThrowError(/has invalid value/); + expect(() => schema.parse('{"a": true}')).toThrowError(/has invalid value/); + }); +}); + +describe("JsonObjectEnv (arbitrary-value)", () => { + const schema = JsonObjectEnv("TEST_ANY", { valueValidator: JsonAny }); + + it("accepts nested objects", () => { + expect( + schema.parse( + JSON.stringify({ + runAsNonRoot: true, + runAsUser: 1000, + capabilities: { drop: ["ALL"] }, + }) + ) + ).toEqual({ + runAsNonRoot: true, + runAsUser: 1000, + capabilities: { drop: ["ALL"] }, + }); + }); + + it("accepts mixed value types", () => { + expect(schema.parse('{"s":"x","n":1,"b":true,"a":[1,2],"o":{"k":"v"}}')).toEqual({ + s: "x", + n: 1, + b: true, + a: [1, 2], + o: { k: "v" }, + }); + }); + + it("still rejects non-object roots", () => { + expect(() => schema.parse('"x"')).toThrowError(/must be a JSON object/); + expect(() => schema.parse("[1,2,3]")).toThrowError(/must be a JSON object/); + }); + + it("includes the env var name in error messages", () => { + const named = JsonObjectEnv("KUBERNETES_WORKER_POD_SECURITY_CONTEXT"); + expect(() => named.parse("{notjson")).toThrowError(/KUBERNETES_WORKER_POD_SECURITY_CONTEXT/); + }); +}); diff --git a/apps/supervisor/src/envUtil.ts b/apps/supervisor/src/envUtil.ts index 917f984cc37..2e023102c5e 100644 --- a/apps/supervisor/src/envUtil.ts +++ b/apps/supervisor/src/envUtil.ts @@ -45,3 +45,83 @@ export const AdditionalEnvVars = z.preprocess((val) => { return undefined; } }, z.record(z.string(), z.string()).optional()); + +/** + * Factory for env vars that hold a JSON object. The default is the empty object, + * so callers can spread the parsed result into Kubernetes manifests without + * branching on undefined. + * + * `valueValidator` constrains the shape of the parsed values: + * - `JsonStringMap` for `Record` (e.g. annotations, labels) + * - `JsonAny` for arbitrary nested objects (e.g. `securityContext`) + * + * @example + * KUBERNETES_WORKER_POD_ANNOTATIONS: JsonObjectEnv("KUBERNETES_WORKER_POD_ANNOTATIONS", { + * valueValidator: JsonStringMap, + * }), + */ +export const JsonStringMap = z.record(z.string(), z.string()); +export const JsonAny: z.ZodTypeAny = z.lazy(() => + z.union([ + z.string(), + z.number(), + z.boolean(), + z.null(), + z.array(JsonAny), + z.record(z.string(), JsonAny), + ]) +); + +type JsonObjectEnvOpts = { + /** + * Schema applied to each *value* in the parsed object. Defaults to + * `JsonStringMap` (string values). + */ + valueValidator?: TSchema; +}; + +export const JsonObjectEnv = ( + envName: string, + opts: JsonObjectEnvOpts = {} +) => { + const valueValidator = (opts.valueValidator ?? JsonStringMap) as TSchema; + + return z + .string() + .default("{}") + .transform((raw, ctx) => { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (e) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `${envName} is not valid JSON: ${e instanceof Error ? e.message : String(e)}`, + }); + return z.NEVER; + } + + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `${envName} must be a JSON object (got ${ + Array.isArray(parsed) ? "array" : typeof parsed + })`, + }); + return z.NEVER; + } + + const validated = z.record(z.string(), valueValidator).safeParse(parsed); + if (!validated.success) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `${envName} has invalid value(s): ${validated.error.message}`, + }); + return z.NEVER; + } + + return validated.data as z.infer extends z.ZodTypeAny + ? Record> + : Record; + }); +}; diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts index b2ed05c9f11..4198c600414 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.ts @@ -117,6 +117,13 @@ export class KubernetesWorkloadManager implements WorkloadManager { "app.kubernetes.io/part-of": "trigger-worker", "app.kubernetes.io/component": "create", }, + ...(Object.keys(env.KUBERNETES_WORKER_POD_ANNOTATIONS).length > 0 + ? { + annotations: { + ...env.KUBERNETES_WORKER_POD_ANNOTATIONS, + } as Record, + } + : {}), }, spec: { ...this.addPlacementTags(this.#defaultPodSpec, opts.placementTags), @@ -133,6 +140,16 @@ export class KubernetesWorkloadManager implements WorkloadManager { }, ], resources: this.#getResourcesForMachine(opts.machine), + ...(Object.keys(env.KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT).length > 0 + ? { securityContext: env.KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT } + : {}), + ...(env.KUBERNETES_WORKER_ENV_FROM_SECRET + ? { + envFrom: [ + { secretRef: { name: env.KUBERNETES_WORKER_ENV_FROM_SECRET } }, + ], + } + : {}), env: [ { name: "TRIGGER_DEQUEUED_AT_MS", @@ -307,13 +324,21 @@ export class KubernetesWorkloadManager implements WorkloadManager { get #defaultPodSpec(): Omit { return { restartPolicy: "Never", - automountServiceAccountToken: false, + // Explicit control over service account token mounting (defaults to false for security) + automountServiceAccountToken: env.KUBERNETES_WORKER_AUTOMOUNT_SERVICE_ACCOUNT_TOKEN, imagePullSecrets: this.getImagePullSecrets(), ...(env.KUBERNETES_SCHEDULER_NAME ? { schedulerName: env.KUBERNETES_SCHEDULER_NAME, } : {}), + // Optionally specify a service account for the worker pods + ...(env.KUBERNETES_WORKER_SERVICE_ACCOUNT + ? { serviceAccountName: env.KUBERNETES_WORKER_SERVICE_ACCOUNT } + : {}), + ...(Object.keys(env.KUBERNETES_WORKER_POD_SECURITY_CONTEXT).length > 0 + ? { securityContext: env.KUBERNETES_WORKER_POD_SECURITY_CONTEXT } + : {}), ...(env.KUBERNETES_WORKER_NODETYPE_LABEL ? { nodeSelector: { diff --git a/hosting/k8s/helm/templates/supervisor.yaml b/hosting/k8s/helm/templates/supervisor.yaml index 11fd7a7f6d9..46b0d6ec60b 100644 --- a/hosting/k8s/helm/templates/supervisor.yaml +++ b/hosting/k8s/helm/templates/supervisor.yaml @@ -170,6 +170,28 @@ spec: value: {{ .Values.supervisor.config.kubernetes.forceEnabled | quote }} - name: KUBERNETES_WORKER_NODETYPE_LABEL value: {{ .Values.supervisor.config.kubernetes.workerNodetypeLabel | quote }} + {{- if .Values.supervisor.config.kubernetes.workerServiceAccount }} + - name: KUBERNETES_WORKER_SERVICE_ACCOUNT + value: {{ .Values.supervisor.config.kubernetes.workerServiceAccount | quote }} + {{- end }} + - name: KUBERNETES_WORKER_AUTOMOUNT_SERVICE_ACCOUNT_TOKEN + value: {{ .Values.supervisor.config.kubernetes.workerAutomountServiceAccountToken | quote }} + {{- if .Values.supervisor.config.kubernetes.workerPodAnnotations }} + - name: KUBERNETES_WORKER_POD_ANNOTATIONS + value: {{ .Values.supervisor.config.kubernetes.workerPodAnnotations | toJson | quote }} + {{- end }} + {{- if .Values.supervisor.config.kubernetes.workerPodSecurityContext }} + - name: KUBERNETES_WORKER_POD_SECURITY_CONTEXT + value: {{ .Values.supervisor.config.kubernetes.workerPodSecurityContext | toJson | quote }} + {{- end }} + {{- if .Values.supervisor.config.kubernetes.workerContainerSecurityContext }} + - name: KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT + value: {{ .Values.supervisor.config.kubernetes.workerContainerSecurityContext | toJson | quote }} + {{- end }} + {{- if .Values.supervisor.config.kubernetes.workerEnvFromSecret }} + - name: KUBERNETES_WORKER_ENV_FROM_SECRET + value: {{ .Values.supervisor.config.kubernetes.workerEnvFromSecret | quote }} + {{- end }} {{- $registryAuthEnabled := false }} {{- if .Values.registry.deploy }} {{- $registryAuthEnabled = .Values.registry.auth.enabled }} @@ -292,4 +314,4 @@ spec: protocol: TCP name: metrics selector: - {{- include "trigger-v4.componentSelectorLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }} \ No newline at end of file + {{- include "trigger-v4.componentSelectorLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }} diff --git a/hosting/k8s/helm/values.yaml b/hosting/k8s/helm/values.yaml index 062bebf9c7f..aca72080ff6 100644 --- a/hosting/k8s/helm/values.yaml +++ b/hosting/k8s/helm/values.yaml @@ -291,6 +291,45 @@ supervisor: forceEnabled: true namespace: "" # Default: uses release namespace workerNodetypeLabel: "" # When set, runs will only be scheduled on nodes with "nodetype=