From 9bdf01dfc953b197e1acb748784e022d7907341b Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 03:18:08 +0000 Subject: [PATCH 01/19] Fix #858: tell buildx which platform to build for Docker builds on GitHub are failing with the problem that Docker build is trying to get qsim images from docker.io: ``` #1 [internal] load local bake definitions #1 reading from stdin 935B done #1 DONE 0.0s #2 [qsim internal] load build definition from Dockerfile #2 transferring dockerfile: 1.34kB done #2 DONE 0.0s #3 [qsim-py-tests internal] load build definition from Dockerfile #3 transferring dockerfile: 381B done #3 DONE 0.0s #4 [qsim-cxx-tests internal] load build definition from Dockerfile #4 transferring dockerfile: 208B done #4 DONE 0.0s #5 [qsim-cxx-tests internal] load metadata for #docker.io/library/qsim:latest ``` Apparently, this happens because the `buildx` plugin, which is the default builder in many modern Docker setups, can build images in parallel and uses an isolated builder environment. An image built for one service (like `qsim`): might not be immediately loaded into the local Docker daemon's image store. When the build for a dependent service (like `qsim-cxx-tests`) starts, it can't find the `qsim` base image locally and falls back to searching Docker Hub. While `depends_on` controls the runtime start order of containers, the build-time dependency inference can be tricky in this parallel execution environment. One way to fix this is to explicitly tell `buildx` which platform to build for. This forces buildx to build a single-platform image and load its layers into the local image store, making it available to subsequent build steps. --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 0b63bb418..b4d9cad20 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,13 @@ services: qsim: + platform: linux/amd64 image: qsim container_name: qsim build: context: ./ dockerfile: Dockerfile qsim-cxx-tests: + platform: linux/amd64 image: qsim-cxx-tests container_name: qsim-cxx-tests build: @@ -14,6 +16,7 @@ services: depends_on: - qsim qsim-py-tests: + platform: linux/amd64 image: qsim-py-tests container_name: qsim-py-tests build: From 399dd50c001e565fc8eb1de67dd5e11766c4611b Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 03:24:35 +0000 Subject: [PATCH 02/19] Error still persists, so trying another approach This should explicitly instruct buildx to export the finished build as a standard image in the local Docker daemon. --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index b4d9cad20..97cd045ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,8 @@ services: build: context: ./ dockerfile: Dockerfile + outputs: + - type=docker qsim-cxx-tests: platform: linux/amd64 image: qsim-cxx-tests From 9efe1cde25bb4aacf5212583272c43ed30902f94 Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 03:49:08 +0000 Subject: [PATCH 03/19] Remove outputs Turns out it's not valid. --- docker-compose.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 97cd045ec..8bff5d155 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,6 @@ services: build: context: ./ dockerfile: Dockerfile - outputs: - - type=docker qsim-cxx-tests: platform: linux/amd64 image: qsim-cxx-tests @@ -16,7 +14,7 @@ services: context: ./ dockerfile: tests/Dockerfile depends_on: - - qsim + - qsim qsim-py-tests: platform: linux/amd64 image: qsim-py-tests @@ -25,4 +23,4 @@ services: context: ./ dockerfile: pybind_interface/Dockerfile depends_on: - - qsim + - qsim From 81a50d0cc35ed5a803abf3b1922c3287af838fb3 Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 03:52:19 +0000 Subject: [PATCH 04/19] Name the first build stage something other than "qsim" It gets hard to know what is happening when everthing is called "qsim". --- docker-compose.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8bff5d155..0d703913f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,12 @@ services: - qsim: + qsim-base: platform: linux/amd64 image: qsim container_name: qsim build: context: ./ dockerfile: Dockerfile + qsim-cxx-tests: platform: linux/amd64 image: qsim-cxx-tests @@ -14,7 +15,8 @@ services: context: ./ dockerfile: tests/Dockerfile depends_on: - - qsim + - qsim-base + qsim-py-tests: platform: linux/amd64 image: qsim-py-tests @@ -23,4 +25,4 @@ services: context: ./ dockerfile: pybind_interface/Dockerfile depends_on: - - qsim + - qsim-base From 08288ca4ff38b5d2c87916d4b876c427ba92f12d Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 03:55:29 +0000 Subject: [PATCH 05/19] Try clearing the docker buildx cache --- .github/workflows/ci_docker_tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci_docker_tests.yaml b/.github/workflows/ci_docker_tests.yaml index 55698f401..e293812d1 100644 --- a/.github/workflows/ci_docker_tests.yaml +++ b/.github/workflows/ci_docker_tests.yaml @@ -65,6 +65,9 @@ jobs: fetch-depth: 1 submodules: recursive + - name: Clear the Docker buildx cache + run: docker buildx --force --all --verbose prune + - name: Build Docker images run: docker compose build From cbaa2b84b87426e9b807a8f6e8c851d7f7603220 Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 03:58:00 +0000 Subject: [PATCH 06/19] Fix syntax of prune command --- .github/workflows/ci_docker_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_docker_tests.yaml b/.github/workflows/ci_docker_tests.yaml index e293812d1..af90be9fa 100644 --- a/.github/workflows/ci_docker_tests.yaml +++ b/.github/workflows/ci_docker_tests.yaml @@ -66,7 +66,7 @@ jobs: submodules: recursive - name: Clear the Docker buildx cache - run: docker buildx --force --all --verbose prune + run: docker buildx prune --force --all --verbose - name: Build Docker images run: docker compose build From fba8268557ef5430183aaad023d5e7a6c90c34b2 Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 04:00:15 +0000 Subject: [PATCH 07/19] Prune did nothing; try building with --no-cache --- .github/workflows/ci_docker_tests.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_docker_tests.yaml b/.github/workflows/ci_docker_tests.yaml index af90be9fa..6770a236e 100644 --- a/.github/workflows/ci_docker_tests.yaml +++ b/.github/workflows/ci_docker_tests.yaml @@ -65,11 +65,11 @@ jobs: fetch-depth: 1 submodules: recursive - - name: Clear the Docker buildx cache - run: docker buildx prune --force --all --verbose - - name: Build Docker images - run: docker compose build + run: | + docker version + docker buildx version + docker compose build --no-cache - name: Run C++ tests run: docker run --rm qsim-cxx-tests:latest From 94cd0d2b611f4ce62c74224c5bced9390469da0a Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 04:18:26 +0000 Subject: [PATCH 08/19] Rewrite docker & docker-compose files --- .github/workflows/ci_docker_tests.yaml | 2 +- Dockerfile | 2 +- docker-compose.yml | 18 +++++++----------- pybind_interface/Dockerfile | 2 +- tests/Dockerfile | 2 +- 5 files changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci_docker_tests.yaml b/.github/workflows/ci_docker_tests.yaml index 6770a236e..821fad259 100644 --- a/.github/workflows/ci_docker_tests.yaml +++ b/.github/workflows/ci_docker_tests.yaml @@ -69,7 +69,7 @@ jobs: run: | docker version docker buildx version - docker compose build --no-cache + docker compose build - name: Run C++ tests run: docker run --rm qsim-cxx-tests:latest diff --git a/Dockerfile b/Dockerfile index 1d7fdcefd..4fbef1ce3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Base OS -FROM ubuntu:24.04 +FROM ubuntu:24.04 as qsim-base # Allow passing this variable in from the outside. ARG CUDA_PATH diff --git a/docker-compose.yml b/docker-compose.yml index 0d703913f..10b758942 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,28 +1,24 @@ services: - qsim-base: + qsim-base-image: platform: linux/amd64 - image: qsim - container_name: qsim + image: qsim-base build: context: ./ dockerfile: Dockerfile + target: qsim-base qsim-cxx-tests: - platform: linux/amd64 - image: qsim-cxx-tests - container_name: qsim-cxx-tests build: context: ./ dockerfile: tests/Dockerfile + target: qsim-cxx-tests depends_on: - - qsim-base + - qsim-base-image qsim-py-tests: - platform: linux/amd64 - image: qsim-py-tests - container_name: qsim-py-tests build: context: ./ dockerfile: pybind_interface/Dockerfile + target: qsim-py-tests depends_on: - - qsim-base + - qsim-base-image diff --git a/pybind_interface/Dockerfile b/pybind_interface/Dockerfile index 22f043480..98d73417b 100644 --- a/pybind_interface/Dockerfile +++ b/pybind_interface/Dockerfile @@ -1,5 +1,5 @@ # Base OS -FROM qsim +FROM qsim-base as qsim-py-tests # Copy relevant files COPY ./pybind_interface/ /qsim/pybind_interface/ diff --git a/tests/Dockerfile b/tests/Dockerfile index f83b25aa6..dde0b1a34 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,5 +1,5 @@ # Base OS -FROM qsim +FROM qsim-base as qsim-cxx-tests # Copy relevant files COPY ./tests/ /qsim/tests/ From bfed26489b4548d32f747b1dc8f41d0202b7e3ee Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 04:20:22 +0000 Subject: [PATCH 09/19] Fix casing --- Dockerfile | 2 +- pybind_interface/Dockerfile | 2 +- tests/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4fbef1ce3..85f272f37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Base OS -FROM ubuntu:24.04 as qsim-base +FROM ubuntu:24.04 AS qsim-base # Allow passing this variable in from the outside. ARG CUDA_PATH diff --git a/pybind_interface/Dockerfile b/pybind_interface/Dockerfile index 98d73417b..d22ba9911 100644 --- a/pybind_interface/Dockerfile +++ b/pybind_interface/Dockerfile @@ -1,5 +1,5 @@ # Base OS -FROM qsim-base as qsim-py-tests +FROM qsim-base AS qsim-py-tests # Copy relevant files COPY ./pybind_interface/ /qsim/pybind_interface/ diff --git a/tests/Dockerfile b/tests/Dockerfile index dde0b1a34..2e37212c6 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -1,5 +1,5 @@ # Base OS -FROM qsim-base as qsim-cxx-tests +FROM qsim-base AS qsim-cxx-tests # Copy relevant files COPY ./tests/ /qsim/tests/ From c8156d41bfedf0ad054f83b3a97dc01e44cf9f2b Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 15:04:50 +0000 Subject: [PATCH 10/19] Try another variation --- .github/workflows/ci_docker_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_docker_tests.yaml b/.github/workflows/ci_docker_tests.yaml index 821fad259..6d2ea73f1 100644 --- a/.github/workflows/ci_docker_tests.yaml +++ b/.github/workflows/ci_docker_tests.yaml @@ -69,7 +69,7 @@ jobs: run: | docker version docker buildx version - docker compose build + docker buildx bake --load --progress=plain -f docker-compose.yml - name: Run C++ tests run: docker run --rm qsim-cxx-tests:latest @@ -83,4 +83,4 @@ jobs: - name: Test installation process run: | cd install/tests - docker compose build + docker buildx bake --load --progress=plain -f docker-compose.yml From 1786db8efa37f1054a26963d9e78f2f4a41035be Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 15:26:03 +0000 Subject: [PATCH 11/19] Try another variation --- .github/workflows/ci_docker_tests.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_docker_tests.yaml b/.github/workflows/ci_docker_tests.yaml index 6d2ea73f1..4ff52a82c 100644 --- a/.github/workflows/ci_docker_tests.yaml +++ b/.github/workflows/ci_docker_tests.yaml @@ -69,13 +69,16 @@ jobs: run: | docker version docker buildx version - docker buildx bake --load --progress=plain -f docker-compose.yml + # First, build the base image that the other images depend on. + docker compose build qsim-base-image + # Now build the other images. + docker compose build - name: Run C++ tests - run: docker run --rm qsim-cxx-tests:latest + run: docker run --rm qsim-cxx-tests - name: Run Python tests - run: docker run --rm qsim-py-tests:latest + run: docker run --rm qsim-py-tests - name: Run a sample simulation run: docker run --rm qsim:latest -c /qsim/circuits/circuit_q24 @@ -83,4 +86,4 @@ jobs: - name: Test installation process run: | cd install/tests - docker buildx bake --load --progress=plain -f docker-compose.yml + docker compose build From 21ef9329d1015609fc344deabef3551d60e3ceb1 Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 15:32:10 +0000 Subject: [PATCH 12/19] Fix wrong image name --- .github/workflows/ci_docker_tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_docker_tests.yaml b/.github/workflows/ci_docker_tests.yaml index 4ff52a82c..5f781d101 100644 --- a/.github/workflows/ci_docker_tests.yaml +++ b/.github/workflows/ci_docker_tests.yaml @@ -75,13 +75,13 @@ jobs: docker compose build - name: Run C++ tests - run: docker run --rm qsim-cxx-tests + run: docker run --rm qsim-cxx-tests:latest - name: Run Python tests - run: docker run --rm qsim-py-tests + run: docker run --rm qsim-py-tests:latest - name: Run a sample simulation - run: docker run --rm qsim:latest -c /qsim/circuits/circuit_q24 + run: docker run --rm qsim-base-image:latest -c /qsim/circuits/circuit_q24 - name: Test installation process run: | From e9af9821e2154bb27f9d659f7b32055ad5e3d931 Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 17:36:43 +0000 Subject: [PATCH 13/19] Use less ambiguous names for the services --- docker-compose.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 10b758942..0cffe39ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,13 +1,14 @@ services: qsim-base-image: - platform: linux/amd64 image: qsim-base + platform: linux/amd64 build: context: ./ dockerfile: Dockerfile target: qsim-base - qsim-cxx-tests: + qsim-cxx-tests-image: + image: qsim-cxx-tests build: context: ./ dockerfile: tests/Dockerfile @@ -15,7 +16,8 @@ services: depends_on: - qsim-base-image - qsim-py-tests: + qsim-py-tests-image: + image: qsim-py-tests build: context: ./ dockerfile: pybind_interface/Dockerfile From b7c89c9c6c10eddd364281005af783302fd63341 Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 17:40:41 +0000 Subject: [PATCH 14/19] More tweaks --- .github/workflows/ci_docker_tests.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_docker_tests.yaml b/.github/workflows/ci_docker_tests.yaml index 5f781d101..4987fa801 100644 --- a/.github/workflows/ci_docker_tests.yaml +++ b/.github/workflows/ci_docker_tests.yaml @@ -65,14 +65,15 @@ jobs: fetch-depth: 1 submodules: recursive + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker + - name: Build Docker images run: | - docker version - docker buildx version - # First, build the base image that the other images depend on. docker compose build qsim-base-image - # Now build the other images. - docker compose build + docker compose build qsim-cxx-tests-image qsim-py-tests-image - name: Run C++ tests run: docker run --rm qsim-cxx-tests:latest @@ -81,7 +82,7 @@ jobs: run: docker run --rm qsim-py-tests:latest - name: Run a sample simulation - run: docker run --rm qsim-base-image:latest -c /qsim/circuits/circuit_q24 + run: docker run --rm qsim-base:latest -c /qsim/circuits/circuit_q24 - name: Test installation process run: | From 8b7f39410c47995e2c68b32da8a592605f784907 Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 19:17:39 +0000 Subject: [PATCH 15/19] Try one more --- .github/workflows/ci_docker_tests.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_docker_tests.yaml b/.github/workflows/ci_docker_tests.yaml index 4987fa801..99bd498e2 100644 --- a/.github/workflows/ci_docker_tests.yaml +++ b/.github/workflows/ci_docker_tests.yaml @@ -72,8 +72,12 @@ jobs: - name: Build Docker images run: | - docker compose build qsim-base-image - docker compose build qsim-cxx-tests-image qsim-py-tests-image + # Running locally, a plain "docker compose build" works as expected. + # On GitHub, buildx tries to build all 3 images in parallel even if + # you use env var COMPOSE_PARALLEL_LIMIT or option --parallel 1. + # The build then fails because the qsim-base image is not available + # to the other two build processes. + docker compose --parallel 1 build - name: Run C++ tests run: docker run --rm qsim-cxx-tests:latest From 18d9ea48fafb48c63088a142777b6a75b43ace3c Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 19:20:45 +0000 Subject: [PATCH 16/19] Revert & explain what's going on --- .github/workflows/ci_docker_tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_docker_tests.yaml b/.github/workflows/ci_docker_tests.yaml index 99bd498e2..21bea2613 100644 --- a/.github/workflows/ci_docker_tests.yaml +++ b/.github/workflows/ci_docker_tests.yaml @@ -74,10 +74,10 @@ jobs: run: | # Running locally, a plain "docker compose build" works as expected. # On GitHub, buildx tries to build all 3 images in parallel even if - # you use env var COMPOSE_PARALLEL_LIMIT or option --parallel 1. - # The build then fails because the qsim-base image is not available - # to the other two build processes. + # you set COMPOSE_PARALLEL_LIMIT or use --parallel 1. That fails b/c + # the qsim-base image is not available to the other two build jobs. docker compose --parallel 1 build + docker compose build qsim-cxx-tests-image qsim-py-tests-image - name: Run C++ tests run: docker run --rm qsim-cxx-tests:latest From c46833f023bfe346aa364d876f9831c6399eb6ff Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 19:22:00 +0000 Subject: [PATCH 17/19] Fix command --- .github/workflows/ci_docker_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_docker_tests.yaml b/.github/workflows/ci_docker_tests.yaml index 21bea2613..57ad67f5d 100644 --- a/.github/workflows/ci_docker_tests.yaml +++ b/.github/workflows/ci_docker_tests.yaml @@ -76,7 +76,7 @@ jobs: # On GitHub, buildx tries to build all 3 images in parallel even if # you set COMPOSE_PARALLEL_LIMIT or use --parallel 1. That fails b/c # the qsim-base image is not available to the other two build jobs. - docker compose --parallel 1 build + docker compose build qsim-base-image docker compose build qsim-cxx-tests-image qsim-py-tests-image - name: Run C++ tests From 74646cebbf1f36e0b999b658847a3c2ecb46ad62 Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 19:23:30 +0000 Subject: [PATCH 18/19] Pin GHA to hash --- .github/workflows/ci_docker_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_docker_tests.yaml b/.github/workflows/ci_docker_tests.yaml index 57ad67f5d..0a15d5140 100644 --- a/.github/workflows/ci_docker_tests.yaml +++ b/.github/workflows/ci_docker_tests.yaml @@ -66,7 +66,7 @@ jobs: submodules: recursive - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3 with: driver: docker From 4f1366cebbb03b314ed9ba287bb0695e7cefe672 Mon Sep 17 00:00:00 2001 From: mhucka Date: Sun, 10 Aug 2025 21:48:48 +0000 Subject: [PATCH 19/19] Try without the docker action --- .github/workflows/ci_docker_tests.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci_docker_tests.yaml b/.github/workflows/ci_docker_tests.yaml index 0a15d5140..77bf452c4 100644 --- a/.github/workflows/ci_docker_tests.yaml +++ b/.github/workflows/ci_docker_tests.yaml @@ -65,11 +65,6 @@ jobs: fetch-depth: 1 submodules: recursive - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3 - with: - driver: docker - - name: Build Docker images run: | # Running locally, a plain "docker compose build" works as expected.