Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 32 additions & 1 deletion apps/supervisor/src/env.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { randomUUID } from "crypto";
import { env as stdEnv } from "std-env";
import { z } from "zod";
import { AdditionalEnvVars, BoolEnv } from "./envUtil.js";
import { AdditionalEnvVars, BoolEnv, JsonAny, JsonObjectEnv } from "./envUtil.js";

const Env = z
.object({
Expand Down Expand Up @@ -92,6 +92,37 @@ const Env = z
KUBERNETES_FORCE_ENABLED: BoolEnv.default(false),
KUBERNETES_NAMESPACE: z.string().default("default"),
KUBERNETES_WORKER_NODETYPE_LABEL: z.string().default("v4-worker"),
KUBERNETES_WORKER_SERVICE_ACCOUNT: z.string().optional(), // Service account for worker pods
KUBERNETES_WORKER_AUTOMOUNT_SERVICE_ACCOUNT_TOKEN: BoolEnv.default(false), // Whether to mount SA token
// Extra annotations to apply to every worker pod (e.g. for service mesh
// sidecar injection, certificate injection, scheduling hints).
KUBERNETES_WORKER_POD_ANNOTATIONS: JsonObjectEnv("KUBERNETES_WORKER_POD_ANNOTATIONS"),
// Pod-level securityContext applied to every worker pod (V1PodSecurityContext shape).
// Default is empty `{}`, preserving the upstream behavior of not setting
// a pod-level securityContext. Provide a JSON object to enforce e.g.
// `{"runAsNonRoot": true, "runAsUser": 1000, "fsGroup": 1000}`.
// OpenShift and other clusters with arbitrary-UID SCCs typically want
// to leave this empty and let the SCC inject values.
KUBERNETES_WORKER_POD_SECURITY_CONTEXT: JsonObjectEnv("KUBERNETES_WORKER_POD_SECURITY_CONTEXT", {
valueValidator: JsonAny,
}),
// Container-level securityContext applied to the worker container of every
// worker pod (V1SecurityContext shape). Default is empty `{}` (matches
// upstream's previous behavior of not setting a container securityContext).
// Provide a JSON object to enforce e.g.
// `{"runAsNonRoot": true, "runAsUser": 1000, "allowPrivilegeEscalation": false,
// "capabilities": {"drop": ["ALL"]}, "seccompProfile": {"type": "RuntimeDefault"}}`.
KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT: JsonObjectEnv(
"KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT",
{ valueValidator: JsonAny }
),
// Name of a Kubernetes Secret to envFrom-mount into every worker pod's
// container. Pulls every key/value pair in the secret as env vars on
// the worker. Resolved by the kubelet at pod creation time; the
// supervisor never reads the secret values, so this needs no extra
// RBAC. Use case: keep task-time secrets (DB URLs, API keys) in
// Kubernetes rather than syncing them through the trigger.dev webapp.
KUBERNETES_WORKER_ENV_FROM_SECRET: z.string().optional(),
KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv
KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"),
KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"),
Expand Down
78 changes: 77 additions & 1 deletion apps/supervisor/src/envUtil.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { describe, it, expect } from "vitest";
import { BoolEnv, AdditionalEnvVars } from "./envUtil.js";
import { z } from "zod";
import { BoolEnv, AdditionalEnvVars, JsonObjectEnv, JsonAny } from "./envUtil.js";

describe("BoolEnv", () => {
it("should parse string 'true' as true", () => {
Expand Down Expand Up @@ -78,3 +79,78 @@ describe("AdditionalEnvVars", () => {
});
});
});

describe("JsonObjectEnv (string-valued)", () => {
const schema = JsonObjectEnv("TEST_ENV");

it("returns empty object for default (no value)", () => {
expect(schema.parse(undefined)).toEqual({});
});

it("parses a simple string-valued JSON object", () => {
expect(schema.parse('{"a":"1","b":"2"}')).toEqual({ a: "1", b: "2" });
});

it("parses an empty JSON object", () => {
expect(schema.parse("{}")).toEqual({});
});

it("rejects non-JSON input", () => {
expect(() => schema.parse("not json")).toThrowError(/not valid JSON/);
});

it("rejects JSON arrays", () => {
expect(() => schema.parse("[]")).toThrowError(/must be a JSON object \(got array\)/);
});

it("rejects JSON primitives", () => {
expect(() => schema.parse('"foo"')).toThrowError(/must be a JSON object \(got string\)/);
expect(() => schema.parse("42")).toThrowError(/must be a JSON object \(got number\)/);
expect(() => schema.parse("null")).toThrowError(/must be a JSON object \(got object\)/);
});

it("rejects values that are not strings (with default validator)", () => {
expect(() => schema.parse('{"a": 1}')).toThrowError(/has invalid value/);
expect(() => schema.parse('{"a": true}')).toThrowError(/has invalid value/);
});
});

describe("JsonObjectEnv (arbitrary-value)", () => {
const schema = JsonObjectEnv("TEST_ANY", { valueValidator: JsonAny });

it("accepts nested objects", () => {
expect(
schema.parse(
JSON.stringify({
runAsNonRoot: true,
runAsUser: 1000,
capabilities: { drop: ["ALL"] },
})
)
).toEqual({
runAsNonRoot: true,
runAsUser: 1000,
capabilities: { drop: ["ALL"] },
});
});

it("accepts mixed value types", () => {
expect(schema.parse('{"s":"x","n":1,"b":true,"a":[1,2],"o":{"k":"v"}}')).toEqual({
s: "x",
n: 1,
b: true,
a: [1, 2],
o: { k: "v" },
});
});

it("still rejects non-object roots", () => {
expect(() => schema.parse('"x"')).toThrowError(/must be a JSON object/);
expect(() => schema.parse("[1,2,3]")).toThrowError(/must be a JSON object/);
});

it("includes the env var name in error messages", () => {
const named = JsonObjectEnv("KUBERNETES_WORKER_POD_SECURITY_CONTEXT");
expect(() => named.parse("{notjson")).toThrowError(/KUBERNETES_WORKER_POD_SECURITY_CONTEXT/);
});
});
80 changes: 80 additions & 0 deletions apps/supervisor/src/envUtil.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,83 @@ export const AdditionalEnvVars = z.preprocess((val) => {
return undefined;
}
}, z.record(z.string(), z.string()).optional());

/**
* Factory for env vars that hold a JSON object. The default is the empty object,
* so callers can spread the parsed result into Kubernetes manifests without
* branching on undefined.
*
* `valueValidator` constrains the shape of the parsed values:
* - `JsonStringMap` for `Record<string, string>` (e.g. annotations, labels)
* - `JsonAny` for arbitrary nested objects (e.g. `securityContext`)
*
* @example
* KUBERNETES_WORKER_POD_ANNOTATIONS: JsonObjectEnv("KUBERNETES_WORKER_POD_ANNOTATIONS", {
* valueValidator: JsonStringMap,
* }),
*/
export const JsonStringMap = z.record(z.string(), z.string());
export const JsonAny: z.ZodTypeAny = z.lazy(() =>
z.union([
z.string(),
z.number(),
z.boolean(),
z.null(),
z.array(JsonAny),
z.record(z.string(), JsonAny),
])
);

type JsonObjectEnvOpts<TSchema extends z.ZodTypeAny> = {
/**
* Schema applied to each *value* in the parsed object. Defaults to
* `JsonStringMap` (string values).
*/
valueValidator?: TSchema;
};

export const JsonObjectEnv = <TSchema extends z.ZodTypeAny = typeof JsonStringMap>(
envName: string,
opts: JsonObjectEnvOpts<TSchema> = {}
) => {
const valueValidator = (opts.valueValidator ?? JsonStringMap) as TSchema;

return z
.string()
.default("{}")
.transform((raw, ctx) => {
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch (e) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: `${envName} is not valid JSON: ${e instanceof Error ? e.message : String(e)}`,
});
return z.NEVER;
}

if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: `${envName} must be a JSON object (got ${
Array.isArray(parsed) ? "array" : typeof parsed
})`,
});
return z.NEVER;
}

const validated = z.record(z.string(), valueValidator).safeParse(parsed);
if (!validated.success) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message: `${envName} has invalid value(s): ${validated.error.message}`,
});
return z.NEVER;
}

return validated.data as z.infer<TSchema> extends z.ZodTypeAny
? Record<string, z.infer<TSchema>>
: Record<string, unknown>;
});
};
27 changes: 26 additions & 1 deletion apps/supervisor/src/workloadManager/kubernetes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,13 @@ export class KubernetesWorkloadManager implements WorkloadManager {
"app.kubernetes.io/part-of": "trigger-worker",
"app.kubernetes.io/component": "create",
},
...(Object.keys(env.KUBERNETES_WORKER_POD_ANNOTATIONS).length > 0
? {
annotations: {
...env.KUBERNETES_WORKER_POD_ANNOTATIONS,
} as Record<string, string>,
}
: {}),
},
spec: {
...this.addPlacementTags(this.#defaultPodSpec, opts.placementTags),
Expand All @@ -133,6 +140,16 @@ export class KubernetesWorkloadManager implements WorkloadManager {
},
],
resources: this.#getResourcesForMachine(opts.machine),
...(Object.keys(env.KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT).length > 0
? { securityContext: env.KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT }
: {}),
...(env.KUBERNETES_WORKER_ENV_FROM_SECRET
? {
envFrom: [
{ secretRef: { name: env.KUBERNETES_WORKER_ENV_FROM_SECRET } },
],
}
: {}),
env: [
{
name: "TRIGGER_DEQUEUED_AT_MS",
Expand Down Expand Up @@ -307,13 +324,21 @@ export class KubernetesWorkloadManager implements WorkloadManager {
get #defaultPodSpec(): Omit<k8s.V1PodSpec, "containers"> {
return {
restartPolicy: "Never",
automountServiceAccountToken: false,
// Explicit control over service account token mounting (defaults to false for security)
automountServiceAccountToken: env.KUBERNETES_WORKER_AUTOMOUNT_SERVICE_ACCOUNT_TOKEN,
imagePullSecrets: this.getImagePullSecrets(),
...(env.KUBERNETES_SCHEDULER_NAME
? {
schedulerName: env.KUBERNETES_SCHEDULER_NAME,
}
: {}),
// Optionally specify a service account for the worker pods
...(env.KUBERNETES_WORKER_SERVICE_ACCOUNT
? { serviceAccountName: env.KUBERNETES_WORKER_SERVICE_ACCOUNT }
: {}),
...(Object.keys(env.KUBERNETES_WORKER_POD_SECURITY_CONTEXT).length > 0
? { securityContext: env.KUBERNETES_WORKER_POD_SECURITY_CONTEXT }
: {}),
...(env.KUBERNETES_WORKER_NODETYPE_LABEL
? {
nodeSelector: {
Expand Down
24 changes: 23 additions & 1 deletion hosting/k8s/helm/templates/supervisor.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,28 @@ spec:
value: {{ .Values.supervisor.config.kubernetes.forceEnabled | quote }}
- name: KUBERNETES_WORKER_NODETYPE_LABEL
value: {{ .Values.supervisor.config.kubernetes.workerNodetypeLabel | quote }}
{{- if .Values.supervisor.config.kubernetes.workerServiceAccount }}
- name: KUBERNETES_WORKER_SERVICE_ACCOUNT
value: {{ .Values.supervisor.config.kubernetes.workerServiceAccount | quote }}
{{- end }}
- name: KUBERNETES_WORKER_AUTOMOUNT_SERVICE_ACCOUNT_TOKEN
value: {{ .Values.supervisor.config.kubernetes.workerAutomountServiceAccountToken | quote }}
{{- if .Values.supervisor.config.kubernetes.workerPodAnnotations }}
- name: KUBERNETES_WORKER_POD_ANNOTATIONS
value: {{ .Values.supervisor.config.kubernetes.workerPodAnnotations | toJson | quote }}
{{- end }}
{{- if .Values.supervisor.config.kubernetes.workerPodSecurityContext }}
- name: KUBERNETES_WORKER_POD_SECURITY_CONTEXT
value: {{ .Values.supervisor.config.kubernetes.workerPodSecurityContext | toJson | quote }}
{{- end }}
{{- if .Values.supervisor.config.kubernetes.workerContainerSecurityContext }}
- name: KUBERNETES_WORKER_CONTAINER_SECURITY_CONTEXT
value: {{ .Values.supervisor.config.kubernetes.workerContainerSecurityContext | toJson | quote }}
{{- end }}
{{- if .Values.supervisor.config.kubernetes.workerEnvFromSecret }}
- name: KUBERNETES_WORKER_ENV_FROM_SECRET
value: {{ .Values.supervisor.config.kubernetes.workerEnvFromSecret | quote }}
{{- end }}
{{- $registryAuthEnabled := false }}
{{- if .Values.registry.deploy }}
{{- $registryAuthEnabled = .Values.registry.auth.enabled }}
Expand Down Expand Up @@ -292,4 +314,4 @@ spec:
protocol: TCP
name: metrics
selector:
{{- include "trigger-v4.componentSelectorLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }}
{{- include "trigger-v4.componentSelectorLabels" (dict "Chart" .Chart "Release" .Release "Values" .Values "component" $component) | nindent 4 }}
39 changes: 39 additions & 0 deletions hosting/k8s/helm/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,45 @@ supervisor:
forceEnabled: true
namespace: "" # Default: uses release namespace
workerNodetypeLabel: "" # When set, runs will only be scheduled on nodes with "nodetype=<label>"
# Service account name for worker pods. Empty = use the namespace's
# "default" SA. Set to e.g. "trigger-worker" to pin a dedicated SA
# (useful for IRSA / Workload Identity, image-pull secrets, etc.).
# Operators are expected to create the SA themselves; this chart
# does not manage worker SAs.
workerServiceAccount: ""
# Whether to mount the SA token inside worker pods. Defaults to
# false; only enable if your worker tasks need to call the
# Kubernetes API.
workerAutomountServiceAccountToken: false
# Annotations applied to every worker pod (e.g. service-mesh
# sidecar opt-out, audit tags). Empty by default.
workerPodAnnotations: {}
# Pod-level securityContext applied to every worker pod
# (V1PodSecurityContext shape). Empty by default. Set this on
# clusters that enforce pod-security admission "restricted",
# OpenShift restricted SCCs, FedRAMP/IL5, etc. Example:
# workerPodSecurityContext:
# runAsNonRoot: true
# runAsUser: 1000
# fsGroup: 1000
workerPodSecurityContext: {}
# Container-level securityContext applied to the worker container
# of every worker pod (V1SecurityContext shape). Example:
# workerContainerSecurityContext:
# runAsNonRoot: true
# runAsUser: 1000
# allowPrivilegeEscalation: false
# capabilities:
# drop: [ALL]
# seccompProfile:
# type: RuntimeDefault
workerContainerSecurityContext: {}
# Name of a Kubernetes Secret to envFrom-mount into every worker
# pod's container. Pulls every key/value pair in the secret as env
# vars on the worker. Empty disables. Resolved by the kubelet at
# pod creation; the supervisor never reads the secret values, so
# no extra RBAC is required.
workerEnvFromSecret: ""
ephemeralStorageSizeLimit: "" # Default: 10Gi
ephemeralStorageSizeRequest: "" # Default: 2Gi´
podCleaner:
Expand Down
Loading