Fix #858 by splitting up docker compose build (#860)

Author: mhucka

.github/workflows/ci_docker_tests.yaml

@@ -66,7 +66,13 @@ jobs:
           submodules: recursive
 
       - name: Build Docker images
-        run: docker compose build
+        run: |
+          # Running locally, a plain "docker compose build" works as expected.
+          # On GitHub, buildx tries to build all 3 images in parallel even if
+          # you set COMPOSE_PARALLEL_LIMIT or use --parallel 1. That fails b/c
+          # the qsim-base image is not available to the other two build jobs.
+          docker compose build qsim-base-image
+          docker compose build qsim-cxx-tests-image qsim-py-tests-image
 
       - name: Run C++ tests
         run: docker run --rm qsim-cxx-tests:latest
@@ -75,7 +81,7 @@ jobs:
         run: docker run --rm qsim-py-tests:latest
 
       - name: Run a sample simulation
-        run: docker run --rm qsim:latest -c /qsim/circuits/circuit_q24
+        run: docker run --rm qsim-base:latest -c /qsim/circuits/circuit_q24
 
       - name: Test installation process
         run: |

Dockerfile

@@ -1,18 +1,20 @@
 # Base OS
-FROM ubuntu:24.04
+FROM ubuntu:24.04 AS qsim-base
 
 # Allow passing this variable in from the outside.
 ARG CUDA_PATH
 ENV PATH="$CUDA_PATH/bin:$PATH"
 
 # Update package list & install some basic tools we'll need.
-RUN apt-get update
-RUN apt-get install -y python3-dev python3-pip python3-venv make g++ wget git
+# hadolint ignore=DL3009,DL3008
+RUN apt-get update && \
+    apt-get install -y make g++ wget git --no-install-recommends && \
+    apt-get install -y python3-dev python3-pip python3-venv --no-install-recommends
 
 # Ubuntu 24's version of CMake is 3.28. We need a newer version.
 RUN apt-get remove --purge --auto-remove cmake
-RUN wget -q https://github.com/Kitware/CMake/releases/download/v3.31.7/cmake-3.31.7-linux-x86_64.sh
-RUN sh cmake-3.31.7-linux-x86_64.sh --prefix=/usr/local --skip-license
+RUN wget -q https://github.com/Kitware/CMake/releases/download/v3.31.7/cmake-3.31.7-linux-x86_64.sh && \
+    sh cmake-3.31.7-linux-x86_64.sh --prefix=/usr/local --skip-license
 
 # Copy relevant files for simulation.
 COPY ./Makefile /qsim/Makefile
@@ -31,8 +33,9 @@ RUN python3 -m venv --upgrade-deps test_env
 ENV PATH="/test_env/bin:$PATH"
 
 # Install qsim requirements.
-RUN python3 -m pip install -r /qsim/requirements.txt
-RUN python3 -m pip install -r /qsim/dev-requirements.txt
+# hadolint ignore=DL3042
+RUN python3 -m pip install -r /qsim/requirements.txt && \
+    python3 -m pip install -r /qsim/dev-requirements.txt
 
 # Compile qsim.
 WORKDIR /qsim/

docker-compose.yml

@@ -1,23 +1,26 @@
 services:
-    qsim:
-        image: qsim
-        container_name: qsim
+    qsim-base-image:
+        image: qsim-base
+        platform: linux/amd64
         build:
             context: ./
             dockerfile: Dockerfile
-    qsim-cxx-tests:
+            target: qsim-base
+
+    qsim-cxx-tests-image:
         image: qsim-cxx-tests
-        container_name: qsim-cxx-tests
         build:
             context: ./
             dockerfile: tests/Dockerfile
+            target: qsim-cxx-tests
         depends_on:
-        - qsim
-    qsim-py-tests:
+          - qsim-base-image
+
+    qsim-py-tests-image:
         image: qsim-py-tests
-        container_name: qsim-py-tests
         build:
             context: ./
             dockerfile: pybind_interface/Dockerfile
+            target: qsim-py-tests
         depends_on:
-        - qsim
+          - qsim-base-image

install/tests/Dockerfile

@@ -6,13 +6,15 @@ ARG CUDA_PATH
 ENV PATH="$CUDA_PATH/bin:$PATH"
 
 # Update package list & install some basic tools we'll need.
-RUN apt-get update
-RUN apt-get install -y python3-dev python3-pip python3-venv make g++ wget git
+# hadolint ignore=DL3009,DL3008
+RUN apt-get update && \
+    apt-get install -y make g++ wget git --no-install-recommends && \
+    apt-get install -y python3-dev python3-pip python3-venv --no-install-recommends
 
 # Ubuntu 24's version of CMake is 3.28. We need a newer version.
 RUN apt-get remove --purge --auto-remove cmake
-RUN wget -q https://github.com/Kitware/CMake/releases/download/v3.31.7/cmake-3.31.7-linux-x86_64.sh
-RUN sh cmake-3.31.7-linux-x86_64.sh --prefix=/usr/local --skip-license
+RUN wget -q https://github.com/Kitware/CMake/releases/download/v3.31.7/cmake-3.31.7-linux-x86_64.sh && \
+    sh cmake-3.31.7-linux-x86_64.sh --prefix=/usr/local --skip-license
 
 # Copy qsim files from the outside-Docker location to an inside-Docker location.
 COPY ./ /qsim/
@@ -29,6 +31,7 @@ ENV PATH="/qsim/test_env/bin:$PATH"
 # Install qsim from sources.
 # Note: use pip3 here, not python3 -m pip. We need to make sure to get the pip
 # installed inside the venv, because that one has the correct sys.path.
+# hadolint ignore=DL3042
 RUN pip3 install -v /qsim/
 
 # Copy the tests to a non-qsim directory.

pybind_interface/Dockerfile

@@ -1,5 +1,5 @@
 # Base OS
-FROM qsim
+FROM qsim-base AS qsim-py-tests
 
 # Copy relevant files
 COPY ./pybind_interface/ /qsim/pybind_interface/

tests/Dockerfile

@@ -1,5 +1,5 @@
 # Base OS
-FROM qsim
+FROM qsim-base AS qsim-cxx-tests
 
 # Copy relevant files
 COPY ./tests/ /qsim/tests/

tests/Makefile

@@ -70,19 +70,19 @@ custatevec-tests: $(CUSTATEVEC_TARGETS)
 hip-tests: $(HIP_TARGETS)
 
 .PHONY: run-cxx-tests
-run-cxx-tests: cxx-tests
+run-cxx-tests: | $(GTEST_DIR)/build cxx-tests
 	for exe in $(CXX_TARGETS); do if ! ./$$exe; then exit 1; fi; done
 
 .PHONY: run-cuda-tests
-run-cuda-tests: cuda-tests
+run-cuda-tests: | $(GTEST_DIR)/build cuda-tests
 	for exe in $(CUDA_TARGETS); do if ! ./$$exe; then exit 1; fi; done
 
 .PHONY: run-custatevec-tests
-run-custatevec-tests: custatevec-tests
+run-custatevec-tests: | $(GTEST_DIR)/build custatevec-tests
 	for exe in $(CUSTATEVEC_TARGETS); do if ! ./$$exe; then exit 1; fi; done
 
 .PHONY: run-hip-tests
-run-hip-tests: hip-tests
+run-hip-tests: | $(GTEST_DIR)/build hip-tests
 	for exe in $(HIP_TARGETS); do if ! ./$$exe; then exit 1; fi; done
 
 $(GTEST_DIR)/build: