fix: pod deletion race condition

Makefile

@@ -67,6 +67,50 @@ push-images: build-images ## Push container images.
 push-charts: build-chart ## Push OCI packages.
 	$(foreach chart, $(wildcard ./*.tgz), helm push ${chart} oci://$(REGISTRY)/charts ;)
 
+##@ Demo
+
+# Use a fixed cluster name for the demo; do not run on an existing cluster we did not create.
+KIND_CLUSTER_NAME ?= slurm-bridge-demo
+
+.PHONY: demo-cluster-create
+demo-cluster-create: ## Spin up a kind cluster (slurm-bridge-demo) and install slurm-bridge using hack/kind.sh.
+	@if ! kind get clusters 2>/dev/null | grep -q "^$(KIND_CLUSTER_NAME)$$"; then ./hack/kind.sh $(KIND_CLUSTER_NAME); fi
+	./hack/kind.sh --bridge $(KIND_CLUSTER_NAME)
+
+.PHONY: demo-cluster-delete
+demo-cluster-delete: ## Delete the kind cluster.
+	./hack/kind.sh --delete $(KIND_CLUSTER_NAME)
+
+.PHONY: install-dra
+install-dra: ## Add all DRA configs from hack/kind.sh (dra-driver-cpu and dra-example-driver).
+	./hack/kind.sh --dra-driver-cpu --dra-example-driver $(KIND_CLUSTER_NAME)
+
+.PHONY: setup-sysctl
+setup-sysctl: ## Set kernel/sysctl values recommended for kind/demo (requires sudo).
+	./hack/sysctl.sh
+
+# Exclude LWS (long-running) and DRA examples from main demo; DRA has its own demo-dra target.
+HACK_EXAMPLES ?= $(sort $(filter-out hack/examples/lws/lws.yaml $(wildcard hack/examples/dra/*.yaml),$(wildcard hack/examples/*/*.yaml)))
+HACK_EXAMPLES_DRA ?= $(sort $(wildcard hack/examples/dra/*.yaml))
+
+.PHONY: install-examples
+install-examples: ## run examples only-no cluster setup
+	for f in $(HACK_EXAMPLES); do $(KUBECTL) delete -f "$$f" || true; done; \
+    for f in $(HACK_EXAMPLES); do $(KUBECTL) apply -f "$$f"; done;
+
+.PHONY: demo-examples
+demo-examples: demo-cluster-create install-examples ## Run hack/examples YAMLs (except lws and dra) and watch (Ctrl+C to stop watch).
+	if [ "$$(uname -s)" != "Darwin" ]; then ./hack/demo_watch.sh || true; fi
+
+.PHONY: install-examples-dra
+install-examples-dra: ## install dra examples only-no cluster setup
+	for f in $(HACK_EXAMPLES_DRA); do $(KUBECTL) delete -f "$$f" || true; done; \
+	for f in $(HACK_EXAMPLES_DRA); do $(KUBECTL) apply -f "$$f"; done;
+
+.PHONY: demo-examples-dra
+demo-examples-dra: demo-cluster-create install-dra install-examples-dra ## Install DRA drivers and run DRA example pods and watch (Ctrl+C to stop).
+	if [ "$$(uname -s)" != "Darwin" ]; then ./hack/demo_watch.sh || true; fi
+
 ##@ Deployment
 
 # Get the OS to set platform specific commands
@@ -83,10 +127,6 @@ endif
 values-dev: ## Safely initialize values-dev.yaml files for Helm charts.
 	find "helm/" -type f -name "values.yaml" | $(SED) 'p;s/\.yaml/-dev\.yaml/' | xargs -n2 cp $(CP_FLAGS)
 
-ifndef ignore-not-found
-  ignore-not-found = false
-endif
-
 ##@ Build Dependencies
 
 ## Location to install dependencies to

README.md

@@ -71,12 +71,15 @@ Slurm is a full featured HPC workload manager. To highlight a few features:
 
 | Software   |                             Minimum Version                              |
 | :--------- | :----------------------------------------------------------------------: |
-| Kubernetes | [v1.34](https://kubernetes.io/blog/2025/08/27/kubernetes-v1-34-release/) |
+| Kubernetes | [v1.35](https://kubernetes.io/blog/2025/12/17/kubernetes-v1-35-release/) |
 | Slurm      | [25.05](https://www.schedmd.com/slurm-version-25-05-0-is-now-available/) |
 
 ## Limitations
 
 - Exclusive, whole node allocations are made for each pod.
+- Only supports the following DRA drivers:
+  - [DRA Driver CPU][dra-driver-cpu] for CPUs.
+  - [DRA Example Driver][dra-example-driver] for GPUs.
 
 ## Installation
 
@@ -132,6 +135,8 @@ specific language governing permissions and limitations under the License.
 [architecture]: ./docs/architecture.md
 [contact-schedmd]: https://www.schedmd.com/slurm-resources/contact-schedmd/
 [docs]: ./docs/
+[dra-driver-cpu]: https://github.com/kubernetes-sigs/dra-driver-cpu
+[dra-example-driver]: https://github.com/kubernetes-sigs/dra-example-driver
 [kubernetes]: https://kubernetes.io/
 [quickstart]: ./docs/quickstart.md
 [slinky]: https://slinky.ai/

docs/usermanagement.md

@@ -0,0 +1,195 @@
+# User Management in Slurm-bridge
+
+## Table of Contents
+
+<!-- mdformat-toc start --slug=github --no-anchors --maxlevel=6 --minlevel=1 -->
+
+- [User Management in Slurm-bridge](#user-management-in-slurm-bridge)
+  - [Table of Contents](#table-of-contents)
+  - [Overview](#overview)
+  - [Using Kyverno Policies with Slurm-bridge](#using-kyverno-policies-with-slurm-bridge)
+    - [Overview of Kyverno](#overview-of-kyverno)
+    - [Configuring Kyverno for use with Slurm-Bridge](#configuring-kyverno-for-use-with-slurm-bridge)
+    - [Example Kyverno Policies for Slurm-bridge](#example-kyverno-policies-for-slurm-bridge)
+    - [Enforce Specification of `slurmjob.slinky.slurm.net/account` and `slurmjob.slinky.slurm.net/user-id`](#enforce-specification-of-slurmjobslinkyslurmnetaccount-and-slurmjobslinkyslurmnetuser-id)
+    - [Prevent modification of Slinky annotations after workload submission](#prevent-modification-of-slinky-annotations-after-workload-submission)
+
+<!-- mdformat-toc end -->
+
+## Overview
+
+Slurm-bridge uses [placeholder jobs](scheduler) to represent Kubernetes
+workloads in Slurm for scheduling purposes. Placeholder jobs are submitted using
+the JWT that is generated when the Slurm-bridge Helm chart is deployed. By
+default, placeholder jobs for Kubernetes workloads are submitted as `SlurmUser`,
+but they may be submitted as another user in Slurm by using the
+`slurmjob.slinky.slurm.net/user-id` annotation.
+
+Kubernetes workloads scheduled by Slurm-bridge always run within the user
+context that they are submitted with - Slurm-bridge does not interfere with a
+Kubernetes workloads' user context at runtime.
+
+## Using Kyverno Policies with Slurm-bridge
+
+### Overview of Kyverno
+
+[Kyverno] is a cloud-native policy engine, which can be used to enforce security
+policies using a Kubernetes admission controller. When running Slurm-bridge in a
+multi-tenant production environment, it may be desirable to prevent unprivileged
+users from modifying certain fields of Kubernetes objects, in order to prevent
+interference with the operations of Slurm-bridge.
+
+### Configuring Kyverno for use with Slurm-Bridge
+
+In order to use enforce policies related to a CRD, one must first use
+ClusterRoles to grant the Kyverno controller additional permissions related to
+that CRD. More information on that process can be found
+[here][kyverno-customize-permissions]. For example, the following ClusterRole
+provides the Kyverno controller with sufficient permissions to create Pods and
+PodGroups:
+
+```yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kyverno:create-pods-and-podgroups
+  labels:
+    rbac.kyverno.io/aggregate-to-background-controller: 'true'
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - create
+      - update
+  - apiGroups:
+      - podgroups.scheduling.x-k8s.io
+    resources:
+      - podgroups
+    verbs:
+      - create
+      - update
+```
+
+After applying this ClusterRole, confirm that the Kyverno background
+controller's top-level ClusterRole has successfully aggregated your new
+permissions:
+
+```bash
+kubectl get clusterrole kyverno:background-controller -o yaml
+```
+
+### Example Kyverno Policies for Slurm-bridge
+
+Below are some example Kyverno policies that can be used with Slurm-bridge.
+
+### Enforce Specification of `slurmjob.slinky.slurm.net/account` and `slurmjob.slinky.slurm.net/user-id`
+
+The following policy ensures that users specify the Slurm account and user-id
+under which the Slurm-bridge workload will be submitted.
+
+```yaml
+apiVersion: kyverno.io/v1
+# The `ClusterPolicy` kind applies to the entire cluster.
+kind: ClusterPolicy
+metadata:
+  name: require-user-and-account-annotations
+# The `spec` defines properties of the policy.
+spec:
+  # The `validationFailureAction` tells Kyverno if the resource being validated should be allowed but reported (`Audit`) or blocked (`Enforce`).
+  validationFailureAction: Enforce
+  # The `rules` is one or more rules which must be true.
+  rules:
+  - name: require-user-and-account-annotations
+    # The `match` statement sets the scope of what will be checked. In this case, it is any `Pod` or `PodGroup` resource.
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+          - PodGroup
+    # The `validate` statement tries to positively check what is defined. If the statement, when compared with the requested resource, is true, it is allowed. If false, it is blocked.
+    validate:
+      # The `message` is what gets displayed to a user if this rule fails validation.
+      message: "You must have annotations `slurmjob.slinky.slurm.net/account` and `slurmjob.slinky.slurm.net/user-id` with a string value set on all new Pods and PodGroups."
+      # The `pattern` object defines what pattern will be checked in the resource. In this case, it is looking for `metadata.annotations` with `slurmjob.slinky.slurm.net=account` and `slurmjob.slinky.slurm.net=user-id.
+      pattern:
+        metadata:
+          annotations:
+            slurmjob.slinky.slurm.net/account: "?*"
+            slurmjob.slinky.slurm.net/user-id: "?*"
+```
+
+A user attempting to apply a `Pod` or `PodGroup` without the specified
+annotations will be denied:
+
+```bash
+kubectl apply -f hack/examples/pod/sleep.yaml
+Error from server: error when creating "hack/examples/pod/sleep.yaml": admission webhook "validate.kyverno.svc-fail" denied the request:
+
+resource Pod/slurm-bridge/pod-single was blocked due to the following policies
+
+require-user-and-account-annotations:
+  require-user-and-account-annotations: 'validation error: You must have annotations
+    `slurmjob.slinky.slurm.net/account` and `slurmjob.slinky.slurm.net/user-id` with
+    a string value set on all new Pods and PodGroups. rule require-user-and-account-annotations
+    failed at path /metadata/annotations/slurmjob.slinky.slurm.net/account/'
+```
+
+While pods with the correct annotations are passed through to slurm-bridge's
+admission controller:
+
+```bash
+kubectl apply -f hack/examples/pod/sleep.yaml
+pod/pod-single created
+```
+
+### Prevent modification of Slinky annotations after workload submission
+
+The following Kyverno policy prevents users from modifying the value of certain
+Slinky annotations after workload submission, without interfering with the
+normal operation of Slurm-bridge.
+
+```yaml
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: block-updates-to-slinky-annotations
+spec:
+  validationFailureAction: Enforce
+  background: false
+  rules:
+  - name: block-updates-to-slinky-annotations
+    match:
+      any:
+      - resources:
+          kinds:
+          - Pod
+          - PodGroup
+    preconditions:
+      any:
+      - key: "{{request.operation}}"
+        operator: Equals
+        value: UPDATE
+    exclude:
+      any:
+      - subjects:
+        - kind: ServiceAccount
+          name: slurm-*
+          namespace: slinky
+      - clusterRoles:
+        - slurm-*
+    validate:
+      allowExistingViolations: false
+      message: Modifying the `slinky.slurm.net/slurm-node` annotation on a submitted pod or podgroup is not allowed.
+      pattern:
+        metadata:
+          "=(annotations)":
+            X(slinky.slurm.net/slurm-node): "*?"
+```
+
+<!-- Links -->
+
+[kyverno]: https://kyverno.io/
+[kyverno-customize-permissions]: https://kyverno.io/docs/installation/customization/#customizing-permissions