Skip to content

Commit e657927

Browse files
committed
fix(cert-manager): add bootstrap_mode_enabled to fix SLA-3 spinup
During SLA-3 bootstrap, cert-manager webhook requires instance-type anti-affinity (2 pods on different instance types), but only the EKS controller node group exists until Karpenter deploys at step 10. This leaves webhook pods Pending indefinitely, blocking bootstrap. Add bootstrap_mode_enabled to kube_certificates and kube_cert_manager, following the same two-phase pattern used for Vault (commit 4b2379c): - Phase 1 (setupCertificates): deploy with bootstrap_mode_enabled=true to disable webhook anti-affinity so pods schedule on controller nodes - Phase 2 (setupClusterExtensions): flip to false after Karpenter and node pools are available to restore proper pod spread No behavior change for SLA-1 or SLA-2 clusters.
1 parent 94bec53 commit e657927

14 files changed

Lines changed: 1266 additions & 1424 deletions

File tree

bun.lock

Lines changed: 5 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"check-dependency-version-consistency": "4.1.0"
1010
},
1111
"dependencies": {
12+
"typescript": "5.6.2",
1213
"yaml": "2.7.1"
1314
}
14-
}
15+
}

packages/cli/bun.lock

Lines changed: 80 additions & 76 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

packages/cli/bun.nix

Lines changed: 171 additions & 151 deletions
Large diffs are not rendered by default.

packages/cli/src/commands/cluster/add/setupCertificates.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,10 @@ export async function setupCertificates(
135135
module: MODULES.KUBE_CERTIFICATES,
136136
hclIfMissing: await Bun.file(kubeCertificatesTemplate).text(),
137137
inputUpdates: {
138+
bootstrap_mode_enabled: defineInputUpdate({
139+
schema: z.boolean(),
140+
update: () => true,
141+
}),
138142
self_generated_certs_enabled: defineInputUpdate({
139143
schema: z.boolean(),
140144
update: () => true,

packages/cli/src/commands/cluster/add/setupClusterExtensions.ts

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,30 @@ export async function setupClusterExtensions(
8888
);
8989
}
9090

91+
const shouldSkipCertManagerAntiAffinityAdjustment = async () => {
92+
const certManagerModuleInfo = await readYAMLFile({
93+
filePath: join(clusterPath, MODULES.KUBE_CERTIFICATES, "module.yaml"),
94+
context,
95+
validationSchema: z.object({
96+
extra_inputs: z.object({
97+
bootstrap_mode_enabled: z.boolean(),
98+
}),
99+
}),
100+
});
101+
102+
const certManagerPfData = await getModuleStatus({
103+
environment,
104+
region,
105+
module: MODULES.KUBE_CERTIFICATES,
106+
context,
107+
});
108+
109+
return (
110+
certManagerPfData.deploy_status === "success" &&
111+
certManagerModuleInfo?.extra_inputs?.bootstrap_mode_enabled === false
112+
);
113+
};
114+
91115
interface IContext {
92116
vaultProxyPid?: number;
93117
vaultProxyPort?: number;
@@ -280,6 +304,23 @@ export async function setupClusterExtensions(
280304
}),
281305
},
282306
}),
307+
await buildDeployModuleTask({
308+
taskTitle: "Cert-Manager Anti-Affinity Adjustment",
309+
context,
310+
env: {
311+
...process.env,
312+
},
313+
environment,
314+
region,
315+
skipIfAlreadyApplied: await shouldSkipCertManagerAntiAffinityAdjustment(),
316+
module: MODULES.KUBE_CERTIFICATES,
317+
inputUpdates: {
318+
bootstrap_mode_enabled: defineInputUpdate({
319+
schema: z.boolean(),
320+
update: () => false,
321+
}),
322+
},
323+
}),
283324
],
284325
{ ctx, concurrent: true }
285326
);

packages/cli/tsconfig.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,4 @@
5050
},
5151
"include": ["src/**/*"],
5252
"exclude": ["node_modules", "dist", "bin", "build.config.ts"]
53-
}
53+
}

packages/infrastructure/kube_cert_manager/main.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ module "util_webhook" {
5353
source = "../kube_workload_utility"
5454

5555
workload_name = "cert-manager-webhook"
56-
instance_type_anti_affinity_required = var.sla_target == 3
56+
instance_type_anti_affinity_required = var.bootstrap_mode_enabled ? false : var.sla_target == 3
5757
az_spread_preferred = var.sla_target >= 2
5858
host_anti_affinity_required = var.sla_target >= 2
5959
panfactum_scheduler_enabled = var.panfactum_scheduler_enabled

packages/infrastructure/kube_cert_manager/vars.tf

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,4 +79,10 @@ variable "controller_nodes_enabled" {
7979
description = "Whether to allow pods to schedule on EKS Node Group nodes (controller nodes)"
8080
type = bool
8181
default = true
82+
}
83+
84+
variable "bootstrap_mode_enabled" {
85+
description = "Whether the cluster is being bootstrapped and does not yet have the autoscaler enabled"
86+
type = bool
87+
default = false
8288
}

packages/infrastructure/kube_certificates/main.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ module "util_webhook" {
113113
source = "../kube_workload_utility"
114114

115115
workload_name = "cert-manager-webhook"
116-
instance_type_anti_affinity_required = var.sla_target == 3
116+
instance_type_anti_affinity_required = var.bootstrap_mode_enabled ? false : var.sla_target == 3
117117
az_spread_preferred = var.sla_target >= 2
118118
host_anti_affinity_required = var.sla_target >= 2
119119
panfactum_scheduler_enabled = var.panfactum_scheduler_enabled

0 commit comments

Comments
 (0)