Hyperion on k8s

.dockerignore

@@ -5,8 +5,10 @@
 .ruff_cache
 **/__pycache__/
 
+# _version.py updated when image is built
+**/_version.py
+
 # virtualenv stuff - this gets built by the docker script
 .venv
 activate
-
 tmp

.github/workflows/_container.yml

@@ -48,10 +48,10 @@ jobs:
           images: |
             ghcr.io/${{ github.repository }}
           labels: |
-            org.opencontainers.image.title=mx-bluesky
+            org.opencontainers.image.title=hyperion
             org.opencontainers.image.description=Hyperion service for MX Unattended Data Collection
           annotations: |
-            org.opencontainers.image.title=mx-bluesky
+            org.opencontainers.image.title=hyperion
             org.opencontainers.image.description=Hyperion service for MX Unattended Data Collection
           tags: |
             type=ref,event=tag

.github/workflows/_helm.yml

@@ -27,9 +27,16 @@ jobs:
           TAG=${GITHUB_REF#refs/tags/}
           echo "version=$TAG" >> $GITHUB_OUTPUT
 
-      - name: Package and push chart
+      - name: Package and push mx-bluesky-blueapi chart
         if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }}
         run: |
           helm dependencies update helm/mx-bluesky-blueapi
           helm package helm/mx-bluesky-blueapi --version ${{ steps.meta.outputs.version }} --app-version ${{ steps.meta.outputs.version }} -d /tmp/
           helm push /tmp/mx-bluesky-blueapi-${{ steps.meta.outputs.version }}.tgz oci://ghcr.io/diamondlightsource/charts
+
+      - name: Package and push hyperion chart
+        if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }}
+        run: |
+          helm dependencies update helm/hyperion
+          helm package helm/hyperion --version ${{ steps.meta.outputs.version }} --app-version ${{ steps.meta.outputs.version }} -d /tmp/
+          helm push /tmp/hyperion-${{ steps.meta.outputs.version }}.tgz oci://ghcr.io/diamondlightsource/charts

Dockerfile.hyperion

@@ -1,46 +1,60 @@
-FROM python:3.11 AS build
+# This is the Dockerfile for building the image for hyperion-supervisor and hyperion-callbacks
+
+# The image will contain the directory structure:
+# /app/mx-bluesky/.venv
+# /app/mx-bluesky/.git
+# /app/dodal/.git
+# /app/mx-bluesky/scripts
+# but we don't include the source folder structure for mx-bluesky or dodal since these will be
+# deployed in a separate volume by the initContainer for the scratch folder
+
+FROM python:3.12 AS base-image
 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MX_BLUESKY
 
 # Need:
 # libgl, libglib2.0 for python cv2 dependency
-RUN apt update -y && \
-    apt upgrade -y && \
-    apt install -y libgl1 libglib2.0-0
+RUN apt-get update -y && \
+    apt-get upgrade -y && \
+    apt-get install -y libgl1 libglib2.0-0
+
+# Install matching uv binaries for our python distro
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
-RUN pip install --no-cache-dir --no-compile setuptools_scm
+# Create the hyperion user and group which will be needed in the deployment securityContext
+RUN groupadd -r -g 1000 hyperion && \
+    useradd -r -u 1000 -g hyperion -d /app hyperion
+
+ADD --chown=hyperion:hyperion .git /app/mx-bluesky/.git
+RUN mkdir -p /app/scripts && \
+    chown hyperion:hyperion /app/scripts
+
+#############################################################################
+FROM base-image as uv-staging
 
 # Copy the pyproject.toml and install dependencies for better caching when developing
 # & rerunning deployment scripts
-COPY pyproject.toml /app/hyperion/
-WORKDIR "/app/hyperion"
-RUN mkdir -p src/mx_bluesky
-
-# This enables us to cache the pip install without needing _version.py
-# see https://setuptools-scm.readthedocs.io/en/latest/usage/
-RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MX_BLUESKY=1.0.0 pip install \
-    --no-cache-dir --no-compile -e .
-
-# Check out and install dodal locally with no dependencies as this may be a different version to what
-# is referred to in the setup.cfg, but we don't care as it will be overridden by bind mounts in the
-# running container
-RUN mkdir ../dodal && \
-git clone https://github.com/DiamondLightSource/dodal.git ../dodal && \
-pip install --no-cache-dir --no-compile --no-deps -e ../dodal
-
-#
-# Everything above this line should be in the image cache unless pyproject.toml changes
-#
-ADD .git /app/hyperion/.git
-# Restore the repository at the current commit instead of copying, to exclude uncommitted changes
-# This is so that if you build a developer image from this dockerfile then _version.py will not
-# append the dirty workdir hash, which causes complications during deployments that mount from a clean folder.
-RUN git restore .
-
-# Regenerate _version.py with the correct version - this should run quickly since we already have our dependencies
-RUN rm src/mx_bluesky/_version.py
-RUN pip install --no-cache-dir --no-compile --no-deps -e .
-
-ENTRYPOINT ["/app/hyperion/utility_scripts/docker/entrypoint.sh"]
-
-EXPOSE 5005
+WORKDIR "/app/mx-bluesky"
+COPY pyproject.toml uv.lock ./
+RUN mkdir -p src/mx_bluesky && \
+    git config --global --add safe.directory /app/mx-bluesky && \
+    git restore . && \
+    uv sync --no-cache --locked
+
+#############################################################################
+FROM base-image as build
+
+WORKDIR "/app/mx-bluesky"
+RUN mkdir /scratch && \
+    chown hyperion:hyperion /scratch
+
+USER hyperion:hyperion
+
+RUN git clone --bare https://github.com/DiamondLightSource/dodal.git /app/dodal/.git
+
+COPY --chown=hyperion:hyperion --from=uv-staging /app/mx-bluesky/.venv /app/mx-bluesky/.venv
+RUN chmod -R o=rwX /app/mx-bluesky/.venv
+COPY --chown=hyperion:hyperion utility_scripts/docker/initialise_scratch.sh /app/scripts/
+
+EXPOSE 5006

docs/developer/general/explanations/containerised_mx_bluesky.rst

@@ -3,12 +3,32 @@ Containerised mx-bluesky
 
 There are currently two images associated with this repository which are pushed on release: hyperion, and mx-bluesky-blueapi.
 
-The Hyperion image exists because Hyperion was developed before BlueAPI was production-ready, and so doesn't use BlueAPI to schedule plans. This image is only really relevant for i03, and currently isn't used in production anywhere
+The Hyperion image is the image that provides ``hyperion-supervisor`` and ``hyperion-callbacks``. These are launched as applications in their own right; ``hyperion-supervisor`` makes use of BlueAPI client but does not expose blueapi plans. ``hyperion-callbacks`` does not use BlueAPI. Which application is launched depends on whether ``hyperion`` or ``hyperion-callbacks`` is specified in the container launch command.
 
-The ``mx-bluesky-blueapi`` image exists as a minor extension of BlueAPI's image. BlueAPI's image contains the dependencies of BlueAPI, as well as the dependencies of BlueAPI, which includes dodal. When the BlueAPI service is launched, it will do a ``pip install --no deps`` of the plan repository. For MX, this means ``mx-bluesky`` gets installed without any of its dependencies. For this reason, we have created an ``mx-bluesky-blueapi`` image which installs these extra dependencies. 
+The ``mx-bluesky-blueapi`` image exists as a minor extension of BlueAPI's image. BlueAPI's image contains the dependencies of BlueAPI, as well as the dependencies of BlueAPI, which includes dodal. When the BlueAPI service is launched, it will do a ``pip install --no deps`` of the plan repository. For MX, this means ``mx-bluesky`` gets installed without any of its dependencies. For this reason, we have created an ``mx-bluesky-blueapi`` image which installs these extra dependencies.
 
 This image can be used with BlueAPI's original helmchart, the only change required in the ``values.yaml`` is::
 
     image:
         repository: ghcr.io/diamondlightsource/mx-bluesky-blueapi
         tag: "{desired_version}"
+
+``hyperion-blueapi`` is launched as a standard ``mx-bluesky-blueapi`` image with configuration to load the hyperion plan and beamline modules.
+
+Notes on the hyperion k8s deployment
+------------------------------------
+
+The hyperion Kubernetes deployment consists of a singled pod in a deployment which has 4 containers:
+
+* ``hyperion-init``, an initContainer which runs before all other containers start.
+* ``hyperion-supervisor`` which launches the supervisor
+* ``hyperion-callbacks`` which launches the external callbacks
+* ``hyperion-scratch`` which is present to enable hotfixes to be applied
+
+The base ``hyperion`` container image contains only ``mx-bluesky`` and ``dodal`` bare git repos, plus the python virtual environment which provides all other library dependencies. When a new release of ``hyperion`` is first deployed to the cluster, it creates an empty Persistent Volume Claim (PVC).
+
+``hyperion-init`` runs on pod startup and is responsible for checking out ``dodal`` and ``mx-bluesky`` to the persistent volume if they do not already exist.
+
+``hyperion-supervisor`` and ``hyperion-callbacks`` then mount the persistent volume read-only under ``/scratch``, and then run ``hyperion`` from this.
+
+``hyperion-scratch`` is a container that has the persistent volume mounted read-write, its only purpose is to wait for VSCode to attach to it so that you can edit the writable PVC; since ordinary ephemeral containers cannot mount PVCs directly. In this manner, hot fixes to the code can be applied either by editing in VSCode, or via the console; when the pod is restarted the changes will be retained.

docs/developer/general/how-to/deploy-a-release.rst

@@ -3,7 +3,7 @@ Deploy a New Release
 
 .. warning::
 
-   This guide is for pre-containerised deployments. To see how deployments work using BlueAPI with kubernetes, please see `the updated guide <setup-blueapi-for-mx.html>`_.
+   This guide is for pre-containerised deployments. To see how deployments work using BlueAPI with kubernetes, please see `the updated guide <setup-blueapi-for-mx.html>`_. For containerised Hyperion deployments, please refer to `Deploying Hyperion on Kubernetes <deploy-hyperion-kk8s.html>`_.
 
 **Remember to discuss any new deployments with the appropriate beamline scientist.**
 

docs/developer/general/how-to/deploy-hyperion-k8s.rst

@@ -0,0 +1,38 @@
+Deploying Hyperion on Kubernetes
+================================
+
+Initial deployment
+------------------
+
+If deploying for the first time on a beamline, you will need to ensure that the following resources are present in the beamline namespace:
+
+* Secrets: Similar to the blueapi deployment, you will require the following secrets to be present:
+   * rmq-creds: Credentials for the zocalo rabbitmq instance
+   * ispyb-hyperion-cfg: Credentials required for connecting to Expe-Eye
+* A rabbitmq deployment <beamline>-rabbitmq - this is for internal communication between the supervisor, blueapi and the callbacks.
+
+See  `Notes on the Hyperion K8s Deployment <../explanations/containerised_mx_bluesky.html>`_ for a more detailed description of the deployment.
+
+Setting up beamline services and deployments repositories
+---------------------------------------------------------
+
+Deployment of hyperion is via ArgoCD and is enabled by creating an ``ixx-hyperion`` folder in the relevant ``ixx-services`` ``services/`` folder.
+
+For an example see the i03 deployment at https://gitlab.diamond.ac.uk/controls/containers/beamline/i03-services
+
+This will then need to be enabled by referencing it in the corresponding ``ixx-deployment`` ``apps/values.yaml``.
+
+Upgrading Hyperion to a newer version
+-------------------------------------
+
+In order to deploy a newer version of Hyperion you should make the following changes:
+  * ``services/ixx-hyperion/values.yaml`` - edit the ``hyperion.application.imageVersion`` with the updated hyperion container image version. Also update ``hyperion.initContainer.dodalBranch`` with the dodal git tag that the initContainer should check out to scratch.
+  * ``services/ixx-hyperion/Chart.yaml`` - Update dependencies.version for hyperion to the corresponding version of the ``hyperion`` helm chart.
+
+The above images and helm charts should be published on every release to `GHCR repository <https://github.com/orgs/DiamondLightSource/packages?repo_name=mx-bluesky>`_.
+
+Push these changes to the services repository. ArgoCD will detect the changes and deploy a new image.
+
+In the event that other configuration changes are made which don't require a new release, for most changes ArgoCD will detect a change in values.yaml and restart the deployment if necessary. However certain changes may result in the container configuration being updated live and so the pod will not restart. Hyperion will generally require a restart for such changes as it does not monitor the filesystem and for this it is sufficient to delete the pod from the Kubernetes dashboard which will result in the pod being recreated.
+
+When updating to a new version, note that the Persistent Volume Claim (PVC) for the scratch folder for the old version will not be removed - this is so that if reversion to the previous version is required the image is still present. In order to prevent a buildup of volumes, when they are no longer needed any live changes to the scratch folder should have PRs made for them if necessary and the PVCs manually deleted.

docs/developer/general/index.rst

@@ -31,6 +31,7 @@ Documentation is split into four categories, and each is also accessible from li
             how-to/contribute
             how-to/create-a-release
             how-to/deploy-a-release
+            how-to/deploy-hyperion-k8s
             how-to/dev-ops/dev-ops
 
         +++

helm/hyperion/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: hyperion
-description: Hyperion server
+description: Hyperion Supervisor and Callbacks
 type: application
 # version of the chart
 version: 0.0.1

helm/hyperion/templates/configmap.yaml

@@ -0,0 +1,33 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hyperion-supervisor-config
+data:
+  supervisor_config.yaml: |-
+    # Configures the supervisor BlueAPI context to access the baton
+    env:
+      sources:
+        - kind: deviceManager
+          module: dodal.beamlines.{{ .Values.application.beamline }}_supervisor
+          mock: {{ .Values.application.dev }}
+    logging:
+      graylog:
+        url: {{ .Values.graylog.url }}
+        enabled: true
+  client_config.yaml: |-
+    # Configuration for the BlueAPI client running in the hyperion supervisor
+    api:
+      url: {{ .Values.blueapi.url }}
+    stomp:
+      enabled: true
+      url: {{ .Values.stomp.url }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hyperion-callbacks-config
+data:
+  blueapi_callbacks.yml: |-
+    stomp:
+      enabled: true
+      url: {{ .Values.stomp.url }}