diff --git a/.dockerignore b/.dockerignore index ba7a552e17..61ead4d8b4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,8 +5,10 @@ .ruff_cache **/__pycache__/ +# _version.py updated when image is built +**/_version.py + # virtualenv stuff - this gets built by the docker script .venv activate - tmp diff --git a/.github/workflows/_container.yml b/.github/workflows/_container.yml index e8172726dc..596f92521d 100644 --- a/.github/workflows/_container.yml +++ b/.github/workflows/_container.yml @@ -48,10 +48,10 @@ jobs: images: | ghcr.io/${{ github.repository }} labels: | - org.opencontainers.image.title=mx-bluesky + org.opencontainers.image.title=hyperion org.opencontainers.image.description=Hyperion service for MX Unattended Data Collection annotations: | - org.opencontainers.image.title=mx-bluesky + org.opencontainers.image.title=hyperion org.opencontainers.image.description=Hyperion service for MX Unattended Data Collection tags: | type=ref,event=tag diff --git a/.github/workflows/_helm.yml b/.github/workflows/_helm.yml index ce2b69a057..80edab7692 100644 --- a/.github/workflows/_helm.yml +++ b/.github/workflows/_helm.yml @@ -27,9 +27,16 @@ jobs: TAG=${GITHUB_REF#refs/tags/} echo "version=$TAG" >> $GITHUB_OUTPUT - - name: Package and push chart + - name: Package and push mx-bluesky-blueapi chart if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }} run: | helm dependencies update helm/mx-bluesky-blueapi helm package helm/mx-bluesky-blueapi --version ${{ steps.meta.outputs.version }} --app-version ${{ steps.meta.outputs.version }} -d /tmp/ helm push /tmp/mx-bluesky-blueapi-${{ steps.meta.outputs.version }}.tgz oci://ghcr.io/diamondlightsource/charts + + - name: Package and push hyperion chart + if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }} + run: | + helm dependencies update helm/hyperion + helm package helm/hyperion --version ${{ steps.meta.outputs.version }} --app-version ${{ steps.meta.outputs.version }} -d /tmp/ + helm push /tmp/hyperion-${{ steps.meta.outputs.version }}.tgz oci://ghcr.io/diamondlightsource/charts diff --git a/Dockerfile.hyperion b/Dockerfile.hyperion index 3f84127d9d..41ea63d733 100644 --- a/Dockerfile.hyperion +++ b/Dockerfile.hyperion @@ -1,46 +1,60 @@ -FROM python:3.11 AS build +# This is the Dockerfile for building the image for hyperion-supervisor and hyperion-callbacks + +# The image will contain the directory structure: +# /app/mx-bluesky/.venv +# /app/mx-bluesky/.git +# /app/dodal/.git +# /app/mx-bluesky/scripts +# but we don't include the source folder structure for mx-bluesky or dodal since these will be +# deployed in a separate volume by the initContainer for the scratch folder + +FROM python:3.12 AS base-image ARG DEBIAN_FRONTEND=noninteractive +ARG SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MX_BLUESKY # Need: # libgl, libglib2.0 for python cv2 dependency -RUN apt update -y && \ - apt upgrade -y && \ - apt install -y libgl1 libglib2.0-0 +RUN apt-get update -y && \ + apt-get upgrade -y && \ + apt-get install -y libgl1 libglib2.0-0 + +# Install matching uv binaries for our python distro +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ -RUN pip install --no-cache-dir --no-compile setuptools_scm +# Create the hyperion user and group which will be needed in the deployment securityContext +RUN groupadd -r -g 1000 hyperion && \ + useradd -r -u 1000 -g hyperion -d /app hyperion + +ADD --chown=hyperion:hyperion .git /app/mx-bluesky/.git +RUN mkdir -p /app/scripts && \ + chown hyperion:hyperion /app/scripts + +############################################################################# +FROM base-image as uv-staging # Copy the pyproject.toml and install dependencies for better caching when developing # & rerunning deployment scripts -COPY pyproject.toml /app/hyperion/ -WORKDIR "/app/hyperion" -RUN mkdir -p src/mx_bluesky - -# This enables us to cache the pip install without needing _version.py -# see https://setuptools-scm.readthedocs.io/en/latest/usage/ -RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MX_BLUESKY=1.0.0 pip install \ - --no-cache-dir --no-compile -e . - -# Check out and install dodal locally with no dependencies as this may be a different version to what -# is referred to in the setup.cfg, but we don't care as it will be overridden by bind mounts in the -# running container -RUN mkdir ../dodal && \ -git clone https://github.com/DiamondLightSource/dodal.git ../dodal && \ -pip install --no-cache-dir --no-compile --no-deps -e ../dodal - -# -# Everything above this line should be in the image cache unless pyproject.toml changes -# -ADD .git /app/hyperion/.git -# Restore the repository at the current commit instead of copying, to exclude uncommitted changes -# This is so that if you build a developer image from this dockerfile then _version.py will not -# append the dirty workdir hash, which causes complications during deployments that mount from a clean folder. -RUN git restore . - -# Regenerate _version.py with the correct version - this should run quickly since we already have our dependencies -RUN rm src/mx_bluesky/_version.py -RUN pip install --no-cache-dir --no-compile --no-deps -e . - -ENTRYPOINT ["/app/hyperion/utility_scripts/docker/entrypoint.sh"] - -EXPOSE 5005 +WORKDIR "/app/mx-bluesky" +COPY pyproject.toml uv.lock ./ +RUN mkdir -p src/mx_bluesky && \ + git config --global --add safe.directory /app/mx-bluesky && \ + git restore . && \ + uv sync --no-cache --locked + +############################################################################# +FROM base-image as build + +WORKDIR "/app/mx-bluesky" +RUN mkdir /scratch && \ + chown hyperion:hyperion /scratch + +USER hyperion:hyperion + +RUN git clone --bare https://github.com/DiamondLightSource/dodal.git /app/dodal/.git + +COPY --chown=hyperion:hyperion --from=uv-staging /app/mx-bluesky/.venv /app/mx-bluesky/.venv +RUN chmod -R o=rwX /app/mx-bluesky/.venv +COPY --chown=hyperion:hyperion utility_scripts/docker/initialise_scratch.sh /app/scripts/ + +EXPOSE 5006 diff --git a/docs/developer/general/explanations/containerised_mx_bluesky.rst b/docs/developer/general/explanations/containerised_mx_bluesky.rst index 51bcf3d8b3..850b260b16 100644 --- a/docs/developer/general/explanations/containerised_mx_bluesky.rst +++ b/docs/developer/general/explanations/containerised_mx_bluesky.rst @@ -3,12 +3,32 @@ Containerised mx-bluesky There are currently two images associated with this repository which are pushed on release: hyperion, and mx-bluesky-blueapi. -The Hyperion image exists because Hyperion was developed before BlueAPI was production-ready, and so doesn't use BlueAPI to schedule plans. This image is only really relevant for i03, and currently isn't used in production anywhere +The Hyperion image is the image that provides ``hyperion-supervisor`` and ``hyperion-callbacks``. These are launched as applications in their own right; ``hyperion-supervisor`` makes use of BlueAPI client but does not expose blueapi plans. ``hyperion-callbacks`` does not use BlueAPI. Which application is launched depends on whether ``hyperion`` or ``hyperion-callbacks`` is specified in the container launch command. -The ``mx-bluesky-blueapi`` image exists as a minor extension of BlueAPI's image. BlueAPI's image contains the dependencies of BlueAPI, as well as the dependencies of BlueAPI, which includes dodal. When the BlueAPI service is launched, it will do a ``pip install --no deps`` of the plan repository. For MX, this means ``mx-bluesky`` gets installed without any of its dependencies. For this reason, we have created an ``mx-bluesky-blueapi`` image which installs these extra dependencies. +The ``mx-bluesky-blueapi`` image exists as a minor extension of BlueAPI's image. BlueAPI's image contains the dependencies of BlueAPI, as well as the dependencies of BlueAPI, which includes dodal. When the BlueAPI service is launched, it will do a ``pip install --no deps`` of the plan repository. For MX, this means ``mx-bluesky`` gets installed without any of its dependencies. For this reason, we have created an ``mx-bluesky-blueapi`` image which installs these extra dependencies. This image can be used with BlueAPI's original helmchart, the only change required in the ``values.yaml`` is:: image: repository: ghcr.io/diamondlightsource/mx-bluesky-blueapi tag: "{desired_version}" + +``hyperion-blueapi`` is launched as a standard ``mx-bluesky-blueapi`` image with configuration to load the hyperion plan and beamline modules. + +Notes on the hyperion k8s deployment +------------------------------------ + +The hyperion Kubernetes deployment consists of a singled pod in a deployment which has 4 containers: + +* ``hyperion-init``, an initContainer which runs before all other containers start. +* ``hyperion-supervisor`` which launches the supervisor +* ``hyperion-callbacks`` which launches the external callbacks +* ``hyperion-scratch`` which is present to enable hotfixes to be applied + +The base ``hyperion`` container image contains only ``mx-bluesky`` and ``dodal`` bare git repos, plus the python virtual environment which provides all other library dependencies. When a new release of ``hyperion`` is first deployed to the cluster, it creates an empty Persistent Volume Claim (PVC). + +``hyperion-init`` runs on pod startup and is responsible for checking out ``dodal`` and ``mx-bluesky`` to the persistent volume if they do not already exist. + +``hyperion-supervisor`` and ``hyperion-callbacks`` then mount the persistent volume read-only under ``/scratch``, and then run ``hyperion`` from this. + +``hyperion-scratch`` is a container that has the persistent volume mounted read-write, its only purpose is to wait for VSCode to attach to it so that you can edit the writable PVC; since ordinary ephemeral containers cannot mount PVCs directly. In this manner, hot fixes to the code can be applied either by editing in VSCode, or via the console; when the pod is restarted the changes will be retained. diff --git a/docs/developer/general/how-to/deploy-a-release.rst b/docs/developer/general/how-to/deploy-a-release.rst index 8af9565e4c..71466741c6 100644 --- a/docs/developer/general/how-to/deploy-a-release.rst +++ b/docs/developer/general/how-to/deploy-a-release.rst @@ -3,7 +3,7 @@ Deploy a New Release .. warning:: - This guide is for pre-containerised deployments. To see how deployments work using BlueAPI with kubernetes, please see `the updated guide `_. + This guide is for pre-containerised deployments. To see how deployments work using BlueAPI with kubernetes, please see `the updated guide `_. For containerised Hyperion deployments, please refer to `Deploying Hyperion on Kubernetes `_. **Remember to discuss any new deployments with the appropriate beamline scientist.** diff --git a/docs/developer/general/how-to/deploy-hyperion-k8s.rst b/docs/developer/general/how-to/deploy-hyperion-k8s.rst new file mode 100644 index 0000000000..0838d86426 --- /dev/null +++ b/docs/developer/general/how-to/deploy-hyperion-k8s.rst @@ -0,0 +1,38 @@ +Deploying Hyperion on Kubernetes +================================ + +Initial deployment +------------------ + +If deploying for the first time on a beamline, you will need to ensure that the following resources are present in the beamline namespace: + +* Secrets: Similar to the blueapi deployment, you will require the following secrets to be present: + * rmq-creds: Credentials for the zocalo rabbitmq instance + * ispyb-hyperion-cfg: Credentials required for connecting to Expe-Eye +* A rabbitmq deployment -rabbitmq - this is for internal communication between the supervisor, blueapi and the callbacks. + +See `Notes on the Hyperion K8s Deployment <../explanations/containerised_mx_bluesky.html>`_ for a more detailed description of the deployment. + +Setting up beamline services and deployments repositories +--------------------------------------------------------- + +Deployment of hyperion is via ArgoCD and is enabled by creating an ``ixx-hyperion`` folder in the relevant ``ixx-services`` ``services/`` folder. + +For an example see the i03 deployment at https://gitlab.diamond.ac.uk/controls/containers/beamline/i03-services + +This will then need to be enabled by referencing it in the corresponding ``ixx-deployment`` ``apps/values.yaml``. + +Upgrading Hyperion to a newer version +------------------------------------- + +In order to deploy a newer version of Hyperion you should make the following changes: + * ``services/ixx-hyperion/values.yaml`` - edit the ``hyperion.application.imageVersion`` with the updated hyperion container image version. Also update ``hyperion.initContainer.dodalBranch`` with the dodal git tag that the initContainer should check out to scratch. + * ``services/ixx-hyperion/Chart.yaml`` - Update dependencies.version for hyperion to the corresponding version of the ``hyperion`` helm chart. + +The above images and helm charts should be published on every release to `GHCR repository `_. + +Push these changes to the services repository. ArgoCD will detect the changes and deploy a new image. + +In the event that other configuration changes are made which don't require a new release, for most changes ArgoCD will detect a change in values.yaml and restart the deployment if necessary. However certain changes may result in the container configuration being updated live and so the pod will not restart. Hyperion will generally require a restart for such changes as it does not monitor the filesystem and for this it is sufficient to delete the pod from the Kubernetes dashboard which will result in the pod being recreated. + +When updating to a new version, note that the Persistent Volume Claim (PVC) for the scratch folder for the old version will not be removed - this is so that if reversion to the previous version is required the image is still present. In order to prevent a buildup of volumes, when they are no longer needed any live changes to the scratch folder should have PRs made for them if necessary and the PVCs manually deleted. diff --git a/docs/developer/general/index.rst b/docs/developer/general/index.rst index 2bff15c3a9..9652baf097 100644 --- a/docs/developer/general/index.rst +++ b/docs/developer/general/index.rst @@ -31,6 +31,7 @@ Documentation is split into four categories, and each is also accessible from li how-to/contribute how-to/create-a-release how-to/deploy-a-release + how-to/deploy-hyperion-k8s how-to/dev-ops/dev-ops +++ diff --git a/helm/hyperion/Chart.yaml b/helm/hyperion/Chart.yaml index cea620e9b7..e786d03398 100644 --- a/helm/hyperion/Chart.yaml +++ b/helm/hyperion/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: hyperion -description: Hyperion server +description: Hyperion Supervisor and Callbacks type: application # version of the chart version: 0.0.1 diff --git a/helm/hyperion/templates/configmap.yaml b/helm/hyperion/templates/configmap.yaml new file mode 100644 index 0000000000..def19432d4 --- /dev/null +++ b/helm/hyperion/templates/configmap.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: hyperion-supervisor-config +data: + supervisor_config.yaml: |- + # Configures the supervisor BlueAPI context to access the baton + env: + sources: + - kind: deviceManager + module: dodal.beamlines.{{ .Values.application.beamline }}_supervisor + mock: {{ .Values.application.dev }} + logging: + graylog: + url: {{ .Values.graylog.url }} + enabled: true + client_config.yaml: |- + # Configuration for the BlueAPI client running in the hyperion supervisor + api: + url: {{ .Values.blueapi.url }} + stomp: + enabled: true + url: {{ .Values.stomp.url }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: hyperion-callbacks-config +data: + blueapi_callbacks.yml: |- + stomp: + enabled: true + url: {{ .Values.stomp.url }} diff --git a/helm/hyperion/templates/deployment.yaml b/helm/hyperion/templates/deployment.yaml index 41eb7ca251..f4dda06bb8 100644 --- a/helm/hyperion/templates/deployment.yaml +++ b/helm/hyperion/templates/deployment.yaml @@ -6,131 +6,168 @@ spec: selector: matchLabels: app: hyperion + strategy: + type: Recreate replicas: 1 template: metadata: labels: app: hyperion spec: + terminationGracePeriodSeconds: {{ .Values.application.terminationGracePeriodSeconds }} securityContext: - # gda2 - runAsUser: {{ .Values.application.runAsUser }} - runAsGroup: {{ .Values.application.runAsGroup }} - supplementalGroups: {{ .Values.application.supplementalGroups }} + runAsGroup: {{ .Values.supervisor.runAsGroup }} + runAsUser: {{ .Values.supervisor.runAsUser }} volumes: - - name: dls-sw-bl - hostPath: - path: "/dls_sw/{{ .Values.application.beamline }}" - type: Directory - - name: dls-sw-dasc - hostPath: - path: "/dls_sw/dasc" - type: Directory - # Bind some source folders for easier debugging - - name: src - hostPath: - path: "{{ .Values.application.projectDir }}/src" - type: Directory - - name: tests - hostPath: - path: "{{ .Values.application.projectDir }}/tests" - type: Directory - - name: utility-scripts - hostPath: - path: "{{ .Values.application.projectDir }}/utility_scripts" - type: Directory - - name: dodal - hostPath: - path: "{{ .Values.dodal.projectDir | clean }}" - type: Directory - - name: logs - hostPath: - type: Directory - path: "{{ .Values.application.logDir }}" - - name: debuglogs - hostPath: - type: Directory - path: "{{ .Values.application.debugLogDir }}" - - name: data - hostPath: - type: Directory - path: "{{ .Values.application.dataDir }}" + - name: scratch + persistentVolumeClaim: + claimName: hyperion-supervisor-scratch-{{ .Values.application.imageVersion }} + - name: supervisor-config + configMap: + name: hyperion-supervisor-config + - name: callbacks-config + configMap: + name: hyperion-callbacks-config + {{- toYaml .Values.volumes.logging | nindent 8 }} + {{- toYaml .Values.volumes.data | nindent 8 }} + - name: expeye-secrets + secret: + secretName: ispyb-hyperion-cfg - name: zocalo-config configMap: - name: "{{ .Release.Name }}-zocalo-config" + name: zocalo-config + defaultMode: 420 - name: zocalo-secrets - projected: - defaultMode: 0444 - sources: - {{- if .Values.zocalo.secrets }} - {{- range .Values.zocalo.secrets }} - - secret: - name: {{ .secretName }} - {{- end -}} {{/* range */}} - {{- end -}} {{/* if */}} - - name: ispyb-secrets secret: - type: secret - secretName: ispyb-hyperion-cfg + secretName: rmq-creds + initContainers: + - name: hyperion-init + image: {{ .Values.application.imageRepository}}/hyperion:{{ .Values.application.imageVersion }} + command: ['/app/scripts/initialise_scratch.sh'] + volumeMounts: + - name: scratch + mountPath: "/scratch" + env: + - name: HOME + value: /tmp + - name: DODAL_BRANCH + value: {{ .Values.initContainer.dodalBranch }} containers: - - name: hyperion - image: {{ .Values.application.imageRepository}}/mx-bluesky:{{ .Values.application.appVersion }} - resources: - limits: - cpu: "1" - memory: "1Gi" - ports: - - name: hyperion-api - containerPort: {{ .Values.application.containerPort }} - protocol: TCP - env: - - name: LOG_DIR - value: /var/log/bluesky - - name: DEBUG_LOG_DIR - value: /var/log/bluesky-debug - - name: BEAMLINE - value: "{{ .Values.application.beamline }}" - {{- if not .Values.application.dev }} - - name: ZOCALO_GO_USER - value: "gda2" - - name: ZOCALO_GO_HOSTNAME - value: "{{ .Values.application.beamline }}-control" - - name: ZOCALO_CONFIG - value: "/zocalo/config/configuration.yaml" - - name: ISPYB_CONFIG_PATH - value: "/ispyb/secrets/ispyb.cfg" - {{- end }} - readinessProbe: - exec: - command: [ "/app/hyperion/utility_scripts/docker/healthcheck.sh" ] - periodSeconds: 5 - volumeMounts: - - mountPath: "/dls_sw/{{ .Values.application.beamline }}" - name: dls-sw-bl - readOnly: true - mountPropagation: HostToContainer - - mountPath: "/dls_sw/dasc" - name: dls-sw-dasc - readOnly: true - mountPropagation: HostToContainer - - mountPath: "/app/hyperion/src" - name: src - - mountPath: "/app/hyperion/tests" - name: tests - - mountPath: "/app/hyperion/utility_scripts" - name: utility-scripts - - mountPath: "/app/dodal" - name: dodal - - mountPath: "/var/log/bluesky" - name: logs - - mountPath: "/var/log/bluesky-debug" - name: debuglogs - - mountPath: "/dls/{{ .Values.application.beamline }}/data" - name: data - - mountPath: "/zocalo/config" - name: zocalo-config - - mountPath: "/zocalo/secrets" - name: zocalo-secrets - - mountPath: "/ispyb/secrets" - name: ispyb-secrets - hostNetwork: true + - name: hyperion-supervisor + image: {{ .Values.application.imageRepository}}/hyperion:{{ .Values.application.imageVersion }} + command: ['/bin/bash', '-c'] + args: + - | + . /app/mx-bluesky/.venv/bin/activate + hyperion --mode supervisor \ + --client-config /etc/hyperion/client_config.yaml \ + --supervisor-config /etc/hyperion/supervisor_config.yaml + {{- if .Values.application.dev }} \ + --dev + sleep 600 + {{- end }} + resources: + limits: + cpu: "1" + memory: "1Gi" + ports: + - name: rest-api + containerPort: {{ .Values.supervisor.containerPort }} + protocol: TCP + env: + - name: LOG_DIR + value: /var/log/bluesky + - name: DEBUG_LOG_DIR + value: /var/log/bluesky/debug + - name: BEAMLINE + value: "{{ .Values.application.beamline }}" + - name: CONFIG_SERVER_URL + value: "{{ .Values.configServer.url }}" + - name: PYTHONPATH + value: "/scratch/mx-bluesky/src:/scratch/dodal/src" + {{- range $key, $value := .Values.supervisor.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + readinessProbe: + httpGet: + path: "/status" + port: {{ .Values.supervisor.containerPort }} + initialDelaySeconds: 30 + periodSeconds: 5 + volumeMounts: + - mountPath: "/scratch" + name: scratch + readOnly: true + - mountPath: "/etc/hyperion" + name: supervisor-config + readOnly: true + - mountPath: "/var/log/bluesky" + name: logdir + - name: hyperion-callbacks + image: {{ .Values.application.imageRepository}}/hyperion:{{ .Values.application.imageVersion }} + command: ['/bin/bash', '-c'] + args: + - | + . /app/mx-bluesky/.venv/bin/activate + hyperion-callbacks \ + --watchdog-port {{ .Values.supervisor.containerPort }} \ + --stomp-config /etc/hyperion/blueapi_callbacks.yml + {{- if .Values.application.dev }} \ + --dev + sleep 600 + {{- end }} + securityContext: + fsGroup: {{ .Values.callbacks.runAsGroup }} + runAsGroup: {{ .Values.callbacks.runAsGroup }} + runAsUser: {{ .Values.callbacks.runAsUser }} + resources: + limits: + cpu: "1" + memory: "1Gi" + env: + - name: LOG_DIR + value: /var/log/bluesky + - name: DEBUG_LOG_DIR + value: /var/log/bluesky/debug + - name: BEAMLINE + value: "{{ .Values.application.beamline }}" + - name: CONFIG_SERVER_URL + value: "{{ .Values.configServer.url }}" + - name: PYTHONPATH + value: "/scratch/mx-bluesky/src:/scratch/dodal/src" + - name: ISPYB_CONFIG_PATH + value: "/etc/secrets/expeye/ispyb.cfg" + volumeMounts: + - mountPath: "/scratch" + name: scratch + readOnly: true + - mountPath: "/etc/hyperion" + name: callbacks-config + readOnly: true + - mountPath: "/etc/zocalo" + name: zocalo-config + readOnly: true + - mountPath: "/var/log/bluesky" + name: logdir + - mountPath: "/dls/{{ .Values.application.beamline }}/data" + name: data + - mountPath: "/etc/secrets/zocalo" + name: zocalo-secrets + - mountPath: "/etc/secrets/expeye" + name: expeye-secrets + - name: hyperion-scratch + # container provided in order for vscode to attach to a writable scratch + image: {{ .Values.application.imageRepository}}/hyperion:{{ .Values.application.imageVersion }} + restartPolicy: Always + volumeMounts: + - mountPath: "/scratch" + name: scratch + resources: + limits: + cpu: "1" + memory: "1Gi" + command: ['/bin/bash', '-c'] + args: + - | + while true; do sleep 60; done diff --git a/helm/hyperion/templates/ingress.yaml b/helm/hyperion/templates/ingress.yaml deleted file mode 100644 index 446cf4a9e1..0000000000 --- a/helm/hyperion/templates/ingress.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if not .Values.application.dev }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: hyperion-ingress -spec: - ingressClassName: nginx - tls: - - hosts: - - {{ .Values.application.externalHostname }} - rules: - - host: {{ .Values.application.externalHostname }} - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: hyperion-svc # this must match the name of the service you want to target - port: - number: {{ .Values.application.containerPort }} - {{- end }} - diff --git a/helm/hyperion/templates/service.yaml b/helm/hyperion/templates/service.yaml index ca11417892..222f70e195 100644 --- a/helm/hyperion/templates/service.yaml +++ b/helm/hyperion/templates/service.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: hyperion-svc + name: hyperion-supervisor-svc spec: {{- if .Values.application.dev }} type: LoadBalancer @@ -9,9 +9,9 @@ spec: type: ClusterIP {{- end }} ports: - - name: hyperion-api - port: {{ .Values.application.servicePort }} + - name: hyperion-supervisor-rest + port: {{ .Values.supervisor.servicePort }} protocol: TCP - targetPort: {{ .Values.application.containerPort }} + targetPort: {{ .Values.supervisor.containerPort }} selector: - app: hyperion + app: hyperion-supervisor diff --git a/helm/hyperion/templates/volumes.yaml b/helm/hyperion/templates/volumes.yaml new file mode 100644 index 0000000000..5cd83135ef --- /dev/null +++ b/helm/hyperion/templates/volumes.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: hyperion-supervisor-scratch-{{ .Values.application.imageVersion }} + annotations: + argocd.argoproj.io/sync-options: Prune=false,Delete=false + argocd.argoproj.io/compare-options: IgnoreExtraneous +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.scratch.volumeSize }} diff --git a/helm/hyperion/templates/zocalo-config.yaml b/helm/hyperion/templates/zocalo-config.yaml index 3fd1af8a36..0ce7498ad8 100644 --- a/helm/hyperion/templates/zocalo-config.yaml +++ b/helm/hyperion/templates/zocalo-config.yaml @@ -1,24 +1,22 @@ apiVersion: v1 kind: ConfigMap metadata: - name: "{{ .Release.Name }}-zocalo-config" + name: "zocalo-config" namespace: "{{ .Release.Namespace }}" labels: app.kubernetes.io/name: zocalo-config app.kubernetes.io/instance: "{{ .Release.Name }}-zocalo-config" app.kubernetes.io/version: "{{ .Chart.Version }}" - app.kubernetes.io/part-of: hyperion + app.kubernetes.io/part-of: hyperion data: configuration.yaml: |- version: 1 - graylog: plugin: graylog {{- .Values.zocalo.graylog | toYaml | nindent 6 }} - {{- if .Values.zocalo.secrets }} {{- range .Values.zocalo.secrets }} - {{ .name }}: "/zocalo/secrets/{{ .fileName }}" + {{ .name }}: "/etc/secrets/zocalo/{{ .fileName }}" {{- end -}} {{/* range */}} {{- end -}} {{/* if */}} diff --git a/helm/hyperion/values.yaml b/helm/hyperion/values.yaml index 0cd408b5c9..bd9ad3c393 100644 --- a/helm/hyperion/values.yaml +++ b/helm/hyperion/values.yaml @@ -1,25 +1,49 @@ application: - containerPort: 5005 - servicePort: 80 imageRepository: ghcr.io/diamondlightsource - # i03-hyperion user and group - runAsUser: 36101 - runAsGroup: 36101 - supplementalGroups: [] + imageName: hyperion + imageVersion: SET_ON_INSTALL beamline: i03 dev: false - logDir: "/dls_sw/i03/logs/bluesky/hyperion-k8s" - debugLogDir: "/dls/tmp/i03/logs/bluesky-k8s" - dataDir: "/dls/i03/data" - # These should be overridden at install time - projectDir: SET_ON_INSTALL - appVersion: SET_ON_INSTALL - externalHostname: i03-hyperion.diamond.ac.uk -dodal: - projectDir: SET_ON_INSTALL -service: - type: NodePort - + logDir: "/var/logs/mx-bluesky" + terminationGracePeriodSeconds: 30 +supervisor: + servicePort: 80 + containerPort: 5006 + runAsUser: 1000 + runAsGroup: 1000 + env: + EPICS_CA_NAME_SERVERS: "gateways:9064" + EPICS_PVA_NAME_SERVERS: "gateways:9065" +callbacks: + runAsUser: 1000 + runAsGroup: 1000 +initContainer: + dodalBranch: "main" +scratch: + volumeSize: "2Gi" +configServer: + url: http://i03-daq-config-server.diamond.ac.uk +graylog: + url: "tcp://graylog-log-target.diamond.ac.uk:12232" +stomp: + url: tcp://localhost:61613 +blueapi: + url: http://my.blueapi.service:5005 +volumes: + logging: + - name: logdir + emptyDir: {} +# In ixx-services, replace with path to relevant log folder, e.g. +# hostPath: +# path: "/dls_sw/i03/logs/bluesky/hyperion-k8s" +# type: Directory + data: + - name: data + emptyDir: {} +# In ixx-services, replace with path to relevant data folder, e.g. +# hostPath: +# type: Directory +# path: "/dls/i03/data" zocalo: defaultTransport: "PikaTransport" graylog: @@ -30,6 +54,3 @@ zocalo: - name: rabbitmq secretName: rmq-creds fileName: rabbitmq-credentials.yml - - name: rabbitmq-api-reader - secretName: rmq-api-reader - fileName: rabbitmq-api-reader.yml diff --git a/helm/rabbitmq/Chart.yaml b/helm/rabbitmq/Chart.yaml new file mode 100644 index 0000000000..0ea3296092 --- /dev/null +++ b/helm/rabbitmq/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: mx-bluesky-rabbitmq-test +description: Rabbitmq server for use in testing environments + +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 + +dependencies: + - name: rabbitmq + version: 2.3.0 + repository: https://groundhog2k.github.io/helm-charts/ diff --git a/helm/rabbitmq/README.md b/helm/rabbitmq/README.md new file mode 100644 index 0000000000..96edb237ab --- /dev/null +++ b/helm/rabbitmq/README.md @@ -0,0 +1 @@ +This helmchart is provided for development testing purposes only diff --git a/utility_scripts/build_docker_image.sh b/utility_scripts/build_docker_image.sh index 6854d10697..0c9ef468b8 100755 --- a/utility_scripts/build_docker_image.sh +++ b/utility_scripts/build_docker_image.sh @@ -1,8 +1,10 @@ #!/bin/bash -set -e -# builds the docker image +set -eo pipefail +# Script for building development docker images BUILD=1 -PUSH=1 +PUSH=0 +BUILD_UNCLEAN=0 +BUILD_HELM=0 PODMAN_FLAGS="" for option in "$@"; do case $option in @@ -10,22 +12,32 @@ for option in "$@"; do BUILD=0 shift ;; - --no-push) - PUSH=0 + --push) + PUSH=1 shift ;; --no-cache) PODMAN_FLAGS+=" --no-cache" shift ;; + --unclean) + BUILD_UNCLEAN=1 + shift + ;; + --helm) + BUILD_HELM=1 + shift + ;; --help|--info|--h) CMD=`basename $0` echo "$CMD [options]" - echo "Builds and/or pushes the docker container image to the repository" + echo "Builds a development hyperion docker image and optionally pushes the docker container image to the repository" echo " --help This help" echo " --no-build Do not build the image" - echo " --no-push Do not push the image" + echo " --push Push the image" echo " --no-cache Don't use the cache when building the image." + echo " --unclean Build with an unclean workspace" + echo " --helm Build (and optionally push) the helm chart" exit 0 ;; -*|--*) @@ -36,42 +48,54 @@ for option in "$@"; do done PROJECTDIR=`dirname $0`/.. -IMAGE=mx-bluesky +IMAGE=hyperion + +function extract_version() { + LATEST_VERSION=$(git tag | sed -E -n 's/^v?([[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+)$/\1/p' | sort -V -r | head -1) + GIT_HASH=$(git rev-parse --short HEAD) + echo $LATEST_VERSION-g$GIT_HASH +} -if ! git diff --cached --quiet; then - echo "Cannot build image from unclean workspace" - exit 1 +if [[ $BUILD_UNCLEAN == 0 ]]; then + if ! git diff --cached --quiet; then + echo "Cannot build image from unclean workspace" + exit 1 + fi fi +export BIGFILES_TMPDIR=/scratch/tmp +export TMPDIR=/tmp if [[ $BUILD == 1 ]]; then echo "Building initial image" - LATEST_TAG=$IMAGE:latest - TMPDIR=/tmp podman build \ - $PODMAN_FLAGS \ - -f $PROJECTDIR/Dockerfile.hyperion \ - --tag $LATEST_TAG \ - $PROJECTDIR - # Now extract the version from the built image and then rebuild with the label - IMAGE_VERSION=$(podman run --rm --entrypoint=hyperion $LATEST_TAG -c "--version" | \ - sed -e 's/[^a-zA-Z0-9 ._-]/_/g') - TAG=$IMAGE:$IMAGE_VERSION - echo "Labelling image with version $IMAGE_VERSION, tagging with tags $TAG $LATEST_TAG" - TMPDIR=/tmp podman build \ - -f $PROJECTDIR/Dockerfile.hyperion \ - --tag $TAG \ - --tag $LATEST_TAG \ - --label "version=$IMAGE_VERSION" \ - $PROJECTDIR + IMAGE_VERSION=$(extract_version) + MX_BLUESKY_VERSION=${IMAGE_VERSION/-/+} + LATEST_TAG=$IMAGE:dev + if [[ $BUILD_HELM == 0 ]]; then + podman build \ + $PODMAN_FLAGS \ + --build-arg SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MX_BLUESKY=$MX_BLUESKY_VERSION \ + -f $PROJECTDIR/Dockerfile.hyperion \ + --tag $LATEST_TAG \ + --tag $IMAGE:$IMAGE_VERSION \ + $PROJECTDIR + else + helm dependencies update helm/$IMAGE + helm package helm/$IMAGE --version ${IMAGE_VERSION} --app-version ${IMAGE_VERSION} -d /tmp/ + fi fi if [[ $PUSH == 1 ]]; then NAMESPACE=$(podman login --get-login ghcr.io | tr '[:upper:]' '[:lower:]') - if [[ $? != 0 ]]; then - echo "Not logged in to ghcr.io" - exit 1 + if [[ $BUILD_HELM == 0 ]]; then + if [[ $? != 0 ]]; then + echo "Not logged in to ghcr.io" + exit 1 + fi + echo "Pushing to ghcr.io/$NAMESPACE/$IMAGE:dev ..." + podman push $IMAGE:dev docker://ghcr.io/$NAMESPACE/$IMAGE:dev + podman push $IMAGE:dev docker://ghcr.io/$NAMESPACE/$IMAGE:$IMAGE_VERSION + else + helm push /tmp/hyperion-${IMAGE_VERSION}.tgz oci://ghcr.io/$NAMESPACE/charts fi - echo "Pushing to ghcr.io/$NAMESPACE/$IMAGE:latest ..." - podman push $IMAGE:latest docker://ghcr.io/$NAMESPACE/$IMAGE:latest - podman push $IMAGE:latest docker://ghcr.io/$NAMESPACE/$IMAGE:$IMAGE_VERSION fi diff --git a/utility_scripts/deploy/deploy_mx_bluesky_app_to_k8s.sh b/utility_scripts/deploy/deploy_mx_bluesky_app_to_k8s.sh index 59f70f8f2d..8a34955d99 100755 --- a/utility_scripts/deploy/deploy_mx_bluesky_app_to_k8s.sh +++ b/utility_scripts/deploy/deploy_mx_bluesky_app_to_k8s.sh @@ -1,7 +1,7 @@ #!/bin/bash # Installs helm package to kubernetes LOGIN=true - +LINT=false for option in "$@"; do case $option in -b=*|--beamline=*) @@ -12,28 +12,24 @@ for option in "$@"; do DEV=true shift ;; - --checkout-to-prod) - CHECKOUT=true - shift - ;; --repository=*) REPOSITORY="${option#*=}" shift ;; - --appVersion=*) - APP_VERSION="${option#*=}" + --imageVersion=*) + IMAGE_VERSION="${option#*=}" shift ;; --no-login) LOGIN=false shift ;; - --dry-run) - DRY_RUN=true + --lint) + LINT=true shift ;; - --bind-dir=*) - BIND_DIR="${option#*=}" + --dry-run) + DRY_RUN=true shift ;; --help|--info|--h) @@ -53,16 +49,15 @@ Arguments: Options: --help This help - --appVersion=version Version of the image to fetch from the repository otherwise it is deduced + --imageVersion=version Version of the image to fetch from the repository otherwise it is deduced from the setuptools_scm. Must be in the format x.y.z -b, --beamline=BEAMLINE Overrides the BEAMLINE environment variable with the given beamline - --checkout-to-prod Checkout source folders to the production folder using deploy_mx_bluesky.py --dev Install to a development kubernetes cluster (assumes project checked out under /home) (default cluster is argus in user namespace) --dry-run Do everything but don't do the final deploy to k8s --no-login Do not attempt to log in to kubernetes instead use the current namespace and cluster --repository=REPOSITORY Override the repository to fetch the image from - --bind-dir=DIR Override the directory to bind to, default calculated from app version + --lint Lint the helm chart EOM exit 0 ;; @@ -103,44 +98,6 @@ PROJECTDIR=$(readlink -e $(dirname $0)/../..) TOP_HELMCHART_DIR=${PROJECTDIR}/helm HELMCHART_DIR=${TOP_HELMCHART_DIR}/${APP_NAME} -if [[ -n $DEV ]]; then - if [[ -n $CHECKOUT ]]; then - echo "Cannot specify both --dev and --checkout-to-prod" - exit 1 - fi - CHECKED_OUT_VERSION=$(git describe --tag) -else - if [[ -z ${VIRTUAL_ENV#${PROJECTDIR}} ]]; then - echo "Virtual env not activated, activating" - . $PROJECTDIR/.venv/bin/activate - fi - - # First extract the version and location that will be deployed - DEPLOY_MX_BLUESKY="python $PROJECTDIR/utility_scripts/deploy/deploy_mx_bluesky.py" - MX_BLUESKY_BASE=$($DEPLOY_MX_BLUESKY --print-release-dir $BEAMLINE) - - if [[ -n $CHECKOUT ]]; then - if [[ -n $BIND_DIR ]]; then - echo "Cannot specify --bind-dir and --checkout-to-prod" - exit 1 - fi - echo "Running deploy_mx_bluesky.py to deploy to production folder..." - $DEPLOY_MX_BLUESKY --kubernetes $BEAMLINE - if [[ $? != 0 ]]; then - echo "Deployment failed, aborting." - exit 1 - fi - fi - - NEW_PROJECTDIR=$MX_BLUESKY_BASE/mx-bluesky - echo "Changing directory to $NEW_PROJECTDIR..." - cd $NEW_PROJECTDIR - PROJECTDIR=$NEW_PROJECTDIR - MX_BLUESKY_BASENAME=$(basename $MX_BLUESKY_BASE) - CHECKED_OUT_VERSION=${MX_BLUESKY_BASENAME#mx-bluesky_} -fi - - if [[ $LOGIN = true ]]; then if [[ -n $DEV ]]; then CLUSTER=argus @@ -151,73 +108,44 @@ if [[ $LOGIN = true ]]; then fi fi -ensure_version_py() { - # We require the _version.py to be created, this needs a minimal virtual environment - if [[ ! -d $PROJECTDIR/.venv ]]; then - echo "Creating _version.py" - echo "Virtual environment not found - creating" - module load python/3.11 && module load uv - uv venv $PROJECTDIR/.venv - source $PROJECTDIR/.venv/bin/activate - fi - uv pip install setuptools_scm -} - -app_version() { - . $PROJECTDIR/.venv/bin/activate - python -m setuptools_scm --force-write-version-files | sed -e 's/[^a-zA-Z0-9._-]/_/g' -} - if [[ -n $REPOSITORY ]]; then HELM_OPTIONS+="--set application.imageRepository=$REPOSITORY " fi -ensure_version_py -if [[ -z $APP_VERSION ]]; then - APP_VERSION=$(app_version) -fi - -echo "Checked out version that will be bind-mounted in $PROJECTDIR is $CHECKED_OUT_VERSION" echo "Container image version that will be pulled is $APP_VERSION" -if [[ $APP_VERSION != $CHECKED_OUT_VERSION ]]; then - echo "*****************************************************************" - echo "WARNING: Checked out version and container image versions differ!" - echo "*****************************************************************" -fi - +#application.runAsUser=$EUID,\ +#application.runAsGroup=$GID,\ +#application.supplementalGroups=[$SUPPLEMENTAL_GIDS],\ +#application.externalHostname=test-$APP_NAME.diamond.ac.uk " if [[ -n $DEV ]]; then GID=`id -g` SUPPLEMENTAL_GIDS=37904 HELM_OPTIONS+="--set \ application.dev=true,\ -application.runAsUser=$EUID,\ -application.runAsGroup=$GID,\ -application.supplementalGroups=[$SUPPLEMENTAL_GIDS],\ application.logDir=$PROJECTDIR/tmp,\ application.dataDir=$PROJECTDIR/tmp/data,\ -application.externalHostname=test-$APP_NAME.diamond.ac.uk " - mkdir -p $PROJECTDIR/tmp/data - DEPLOYMENT_DIR=$PROJECTDIR -else - DEPLOYMENT_DIR=/dls_sw/$BEAMLINE/software/bluesky/mx-bluesky_${APP_VERSION}/mx-bluesky -fi -if [[ -n $BIND_DIR ]]; then - DEPLOYMENT_DIR=$BIND_DIR +stomp.url=tcp://rabbitmq-test:61613 " fi -HELM_OPTIONS+="--set application.appVersion=v$APP_VERSION,\ -application.projectDir=$DEPLOYMENT_DIR,\ -dodal.projectDir=$DEPLOYMENT_DIR/../dodal " +HELM_OPTIONS+="--set application.imageVersion=$IMAGE_VERSION " module load helm +APP_VERSION=$IMAGE_VERSION + helm package $HELMCHART_DIR --app-version $APP_VERSION # Helm package generates a file suffixed with the chart version if [[ $LOGIN = true ]]; then module load $CLUSTER kubectl config set-context --current --namespace=$NAMESPACE fi -if [[ -z $DRY_RUN ]]; then +if [[ $DRY_RUN = true ]]; then + HELM_OPTIONS+="--dry-run=server --debug " +fi +if [[ $LINT = true ]]; then + helm template $RELEASE $HELMCHART_DIR $HELM_OPTIONS + helm lint $HELMCHART_DIR --strict --values $HELMCHART_DIR/values.yaml $HELM_OPTIONS +else helm upgrade --install $HELM_OPTIONS $RELEASE $APP_NAME-0.0.1.tgz fi diff --git a/utility_scripts/docker/entrypoint.sh b/utility_scripts/docker/entrypoint.sh deleted file mode 100755 index 9fba523722..0000000000 --- a/utility_scripts/docker/entrypoint.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -# Entry point for the production docker image that launches the external callbacks -# as well as the main server - -for option in "$@"; do - case $option in - --dev) - IN_DEV=true - ;; - --help|--info|--h) - echo "Arguments:" - echo " --dev start in development mode without external callbacks" - exit 0 - ;; - -*|--*) - echo "Unknown option ${option}. Use --help for info on option usage." - exit 1 - ;; - esac -done - -kill_active_apps () { - echo "Killing active instances of hyperion and hyperion-callbacks..." - pkill -e -f "python.*hyperion" - pkill -e -f "SCREEN.*hyperion" - echo "done." -} - -RELATIVE_SCRIPT_DIR=$( dirname -- "$0"; ) -cd ${RELATIVE_SCRIPT_DIR} - -echo "$(date) Logging to $LOG_DIR" -mkdir -p $LOG_DIR -start_log_path=$LOG_DIR/start_log.log -callback_start_log_path=$LOG_DIR/callback_start_log.log - -#Add future arguments here -declare -A h_and_cb_args=( ["IN_DEV"]="$IN_DEV" ) -declare -A h_and_cb_arg_strings=( ["IN_DEV"]="--dev" ) - -h_commands=() -cb_commands=() -for i in "${!h_and_cb_args[@]}" -do - if [ "${h_and_cb_args[$i]}" != false ]; then - h_commands+="${h_and_cb_arg_strings[$i]} "; - cb_commands+="${h_and_cb_arg_strings[$i]} "; - fi; -done - -trap kill_active_apps TERM - -hyperion-callbacks `echo $cb_commands;`>$callback_start_log_path 2>&1 & - -echo "$(date) Starting Hyperion..." -hyperion `echo $h_commands;`>$start_log_path 2>&1 diff --git a/utility_scripts/docker/initialise_scratch.sh b/utility_scripts/docker/initialise_scratch.sh new file mode 100755 index 0000000000..aa3330412f --- /dev/null +++ b/utility_scripts/docker/initialise_scratch.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# This script is run in the initContainer before the main hyperion pod is launched to initialise the scratch area +# if not already present. + +SCRATCH_ROOT=/scratch +APP_ROOT=/app + +git config --global --add safe.directory /app/dodal/.git +git config --global --add safe.directory /app/mx-bluesky/.git + +if [ ! -d $SCRATCH_ROOT/dodal/.git ]; then + git clone /app/dodal/.git $SCRATCH_ROOT/dodal + echo "Checking out dodal branch $DODAL_BRANCH" + git --git-dir=$SCRATCH_ROOT/dodal/.git checkout $DODAL_BRANCH +fi + +if [ ! -d $SCRATCH_ROOT/mx-bluesky/.git ]; then + CURRENT_BRANCH=$(git --git-dir=$APP_ROOT/mx-bluesky/.git rev-parse --abbrev-ref HEAD) + git clone /app/mx-bluesky/.git $SCRATCH_ROOT/mx-bluesky + ls -la $APP_ROOT/mx-bluesky/.git + echo "Checking out $CURRENT_BRANCH... as $UID" + ls -la $SCRATCH_ROOT/mx-bluesky/.git + git --git-dir=$SCRATCH_ROOT/mx-bluesky/.git checkout $CURRENT_BRANCH +fi + +# dodal and mx-bluesky are installed with uv pip install however since the writable layer for the .venv is ephemeral +# they still need to be added to PYTHONPATH in the container +cd $APP_ROOT/mx-bluesky +. ./.venv/bin/activate +uv pip install --no-cache -e $SCRATCH_ROOT/mx-bluesky +uv pip install --no-cache -e $SCRATCH_ROOT/dodal