From 3c61aefc6b2389f904ca133f6d53432fce23f118 Mon Sep 17 00:00:00 2001 From: JDBetteridge Date: Wed, 4 Mar 2026 23:53:12 +0000 Subject: [PATCH 1/6] ci: Add UID to the dockertag --- .github/workflows/pytest-core-mpi.yaml | 52 ++++++++++++++++++++++-- .github/workflows/pytest-core-nompi.yaml | 45 ++++++++++++++++++-- .github/workflows/pytest-gpu.yaml | 41 ++++++++++++++----- .github/workflows/tutorials.yaml | 32 ++++++++++++++- .pre-commit-config.yaml | 1 + 5 files changed, 150 insertions(+), 21 deletions(-) diff --git a/.github/workflows/pytest-core-mpi.yaml b/.github/workflows/pytest-core-mpi.yaml index 57c7db4c62..aa8a07c715 100644 --- a/.github/workflows/pytest-core-mpi.yaml +++ b/.github/workflows/pytest-core-mpi.yaml @@ -65,6 +65,8 @@ jobs: test-mpi-docker: name: pytest-mpi runs-on: ${{ matrix.os }} + outputs: + unique : ${{ steps.uniquetag.outputs.unique }} strategy: matrix: name: [gcc, gcc-arm, icx] @@ -89,15 +91,57 @@ jobs: - name: Checkout devito uses: actions/checkout@v6 + - name: Generate unique CI tag + id: uniquetag + run: | + UNIQUE=$(echo "${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" | cksum | cut -f 1 -d " ") + echo "Unique ID: ${UNIQUE}" + echo "unique=${UNIQUE}" >> "$GITHUB_OUTPUT" + - name: Build docker image + env: + UNIQUE: ${{ steps.uniquetag.outputs.unique }} run: | - docker build . --file docker/Dockerfile.devito --tag devito_img --build-arg base=devitocodes/bases:cpu-${{ matrix.arch }} + docker build \ + --file docker/Dockerfile.devito \ + --tag "devito_img${UNIQUE}" \ + --build-arg base=devitocodes/bases:cpu-${{ matrix.arch }} \ + . - name: Test with pytest + env: + UNIQUE: ${{ steps.uniquetag.outputs.unique }} run: | - docker run --init -t --rm -e CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} -e OMP_NUM_THREADS=1 --name testrun devito_img pytest tests/test_mpi.py + docker run \ + --init -t --rm \ + --env CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} \ + --env OMP_NUM_THREADS=1 \ + --name "testrun devito_img${UNIQUE}" \ + pytest tests/test_mpi.py - name: Test examples with MPI + env: + UNIQUE: ${{ steps.uniquetag.outputs.unique }} + run: | + docker run \ + --init -t --rm + ${{ matrix.mpiflag }} \ + --env DEVITO_MPI=1 \ + --env OMP_NUM_THREADS=1 \ + --name examplerun \ + "devito_img${UNIQUE}" \ + mpiexec -n 2 pytest examples/seismic/acoustic + # + docker run \ + --init -t --rm \ + --env DEVITO_MPI=1 \ + --env OMP_NUM_THREADS=1 \ + --name examplerun \ + "devito_img${UNIQUE}" \ + mpiexec -n 2 pytest examples/seismic/tti + + - name: Cleanup + env: + UNIQUE: ${{ steps.uniquetag.outputs.unique }} run: | - docker run --init -t --rm ${{ matrix.mpiflag }} -e DEVITO_MPI=1 -e OMP_NUM_THREADS=1 --name examplerun devito_img mpiexec -n 2 pytest examples/seismic/acoustic - docker run --init -t --rm -e DEVITO_MPI=1 -e OMP_NUM_THREADS=1 --name examplerun devito_img mpiexec -n 2 pytest examples/seismic/tti + docker image rm -f "devito_img${UNIQUE}" diff --git a/.github/workflows/pytest-core-nompi.yaml b/.github/workflows/pytest-core-nompi.yaml index 9b75ac714d..9fa6284f13 100644 --- a/.github/workflows/pytest-core-nompi.yaml +++ b/.github/workflows/pytest-core-nompi.yaml @@ -27,6 +27,9 @@ jobs: DEVITO_LANGUAGE: ${{ matrix.language }} OMP_NUM_THREADS: 2 + outputs: + unique : ${{ steps.uniquetag.outputs.unique }} + strategy: # Prevent all build to stop if a single one fails fail-fast: false @@ -146,15 +149,34 @@ jobs: python-version: ${{ matrix.python-version }} allow-prereleases: true + - name: Generate unique CI tag + id: uniquetag + run: | + UNIQUE=$(echo "${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" | cksum | cut -f 1 -d " ") + echo "Unique ID: ${UNIQUE}" + echo "unique=${UNIQUE}" >> "$GITHUB_OUTPUT" + - name: Build docker image if: contains(matrix.name, 'docker') + env: + UNIQUE: ${{ steps.uniquetag.outputs.unique }} run: | - docker build . --file docker/Dockerfile.devito --tag devito_img --build-arg base=devitocodes/bases:cpu-${{ matrix.arch }} + docker build \ + --file docker/Dockerfile.devito \ + --tag "devito_img${UNIQUE}" \ + --build-arg base=devitocodes/bases:cpu-${{ matrix.arch }} \ + . - name: Set run prefix + env: + UNIQUE: ${{ steps.uniquetag.outputs.unique }} run: | if [[ "${{ matrix.name }}" =~ "docker" ]]; then - echo "RUN_CMD=docker run --init -t --rm -e CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} --name testrun devito_img" >> "$GITHUB_ENV" + echo "RUN_CMD=docker run \ + --init -t --rm \ + --env CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} \ + --name testrun \ + devito_img${UNIQUE}" >> "$GITHUB_ENV" else echo "RUN_CMD=" >> "$GITHUB_ENV" fi @@ -207,11 +229,19 @@ jobs: - name: Check configuration run: | - ${{ env.RUN_CMD }} python3 -c "from devito import configuration; print(''.join(['%s: %s \n' % (k, v) for (k, v) in configuration.items()]))" + ${{ env.RUN_CMD }} python3 \ + -c "from devito import configuration; \ + print(''.join(['%s: %s \n' % (k, v) for (k, v) in configuration.items()]))" - name: Test with pytest run: | - ${{ env.RUN_CMD }} pytest -k "${{ matrix.test-set }}" -m "not parallel" --cov --cov-config=.coveragerc --cov-report=xml tests/ + ${{ env.RUN_CMD }} pytest \ + -k "${{ matrix.test-set }}" \ + -m "not parallel" \ + --cov \ + --cov-config=.coveragerc \ + --cov-report=xml \ + tests/ - name: Upload coverage to Codecov if: "!contains(matrix.name, 'docker')" @@ -219,3 +249,10 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} name: ${{ matrix.name }} + + - name: Cleanup Docker + if: "contains(matrix.name, 'docker')" + env: + UNIQUE: ${{ steps.uniquetag.outputs.unique }} + run: | + docker image rm -f "devito_img${UNIQUE}" diff --git a/.github/workflows/pytest-gpu.yaml b/.github/workflows/pytest-gpu.yaml index 5725bca980..993fc53b1e 100644 --- a/.github/workflows/pytest-gpu.yaml +++ b/.github/workflows/pytest-gpu.yaml @@ -37,6 +37,9 @@ jobs: - self-hosted - ${{ matrix.runner_label }} + outputs: + unique : ${{ steps.uniquetag.outputs.unique }} + strategy: fail-fast: false matrix: @@ -52,7 +55,7 @@ jobs: test_drive_cmd: "nvidia-smi" # Respect CUDA_VISIBLE_DEVICES and also hard-limit Docker to that device. # NOTE: CUDA_VISIBLE_DEVICES must be set by the runner (systemd drop-in etc.). - flags: >- + dockerflags: >- --init --rm -t --name ${CONTAINER_BASENAME} --gpus "device=${CUDA_VISIBLE_DEVICES:-all}" @@ -64,7 +67,7 @@ jobs: base: "devitocodes/bases:amd" test_drive_cmd: "rocm-smi" # Unchanged, still passes through required /dev nodes etc. - flags: >- + dockerflags: >- --init --network=host --device=/dev/kfd --device=/dev/dri --ipc=host @@ -77,9 +80,18 @@ jobs: - name: Checkout devito uses: actions/checkout@v6 + - name: Generate unique CI tag + id: uniquetag + run: | + UNIQUE=$(echo "${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" | cksum | cut -f 1 -d " ") + echo "Unique ID: ${UNIQUE}" + echo "unique=${UNIQUE}" >> "$GITHUB_OUTPUT" + - name: Set per-runner tags + env: + UNIQUE: ${{ steps.uniquetag.outputs.unique }} run: | - echo "DOCKER_IMAGE=${{ matrix.name }}-${RUNNER_NAME// /_}" >> "$GITHUB_ENV" + echo "DOCKER_IMAGE=${{ matrix.name }}-${RUNNER_NAME// /_}-${UNIQUE}" >> "$GITHUB_ENV" echo "CONTAINER_BASENAME=testrun-${{ matrix.name }}-${RUNNER_NAME// /_}-${{ github.sha }}" >> "$GITHUB_ENV" - name: Ensure buildx builder @@ -112,7 +124,7 @@ jobs: # Run a simple driver-probe command (nvidia-smi / rocm-smi) docker rm -f "${CONTAINER_BASENAME}" 2>/dev/null || true - docker run ${{ matrix.flags }} "${DOCKER_IMAGE}" ${{ matrix.test_drive_cmd }} + docker run ${{ matrix.dockerflags }} "${DOCKER_IMAGE}" ${{ matrix.test_drive_cmd }} - name: Test with pytest env: @@ -123,12 +135,13 @@ jobs: ci_env=$(bash <(curl -s https://codecov.io/env)) # Run the test suite using the matrix-defined flags - docker run ${{ matrix.flags }} \ + docker run \ + ${{ matrix.dockerflags }} \ "${ci_env}" \ - -e CI=true \ - -e PYTHONFAULTHANDLER=1 \ - -e DEVITO_LOGGING=DEBUG \ - -e CODECOV_TOKEN \ + --env CI=true \ + --env PYTHONFAULTHANDLER=1 \ + --env DEVITO_LOGGING=DEBUG \ + --env CODECOV_TOKEN \ "${DOCKER_IMAGE}" \ pytest -vvv --capture=no --showlocals \ --log-cli-level=DEBUG -o log_cli=true \ @@ -138,11 +151,17 @@ jobs: - name: Test examples run: | - docker run ${{ matrix.flags }} "${DOCKER_IMAGE}" pytest ${{ matrix.test_examples }} + docker run \ + ${{ matrix.dockerflags }} \ + "${DOCKER_IMAGE}" \ + pytest ${{ matrix.test_examples }} - name: Test examples with MPI run: | - docker run ${{ matrix.flags }} --env DEVITO_MPI=1 "${DOCKER_IMAGE}" \ + docker run \ + ${{ matrix.dockerflags }} \ + --env DEVITO_MPI=1 \ + "${DOCKER_IMAGE}" \ mpiexec -n 2 pytest ${{ matrix.test_examples }} - name: Builder & image cleanup (keep 3 days of cache) diff --git a/.github/workflows/tutorials.yaml b/.github/workflows/tutorials.yaml index 6dbc6010c8..575e4d07bb 100644 --- a/.github/workflows/tutorials.yaml +++ b/.github/workflows/tutorials.yaml @@ -26,6 +26,9 @@ jobs: DEVITO_ARCH: "${{ matrix.compiler }}" DEVITO_LANGUAGE: ${{ matrix.language }} + outputs: + unique : ${{ steps.uniquetag.outputs.unique }} + strategy: # Prevent all build to stop if a single one fails fail-fast: false @@ -59,6 +62,13 @@ jobs: - name: Checkout devito uses: actions/checkout@v6 + - name: Generate unique CI tag + id: uniquetag + run: | + UNIQUE=$(echo "${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" | cksum | cut -f 1 -d " ") + echo "Unique ID: ${UNIQUE}" + echo "unique=${UNIQUE}" >> "$GITHUB_OUTPUT" + - name: Set up Python ${{ matrix.pyver }} if: "!contains(matrix.name, 'docker')" uses: actions/setup-python@v6 @@ -72,13 +82,24 @@ jobs: - name: Build docker image if: "contains(matrix.name, 'docker')" + env: + UNIQUE: ${{ steps.uniquetag.outputs.unique }} run: | - docker build . --pull --file docker/Dockerfile.devito --tag devito_img + docker build \ + --pull \ + --file docker/Dockerfile.devito \ + --tag "devito_img${UNIQUE}" \ + . - name: Set run prefix + env: + UNIQUE: ${{ steps.uniquetag.outputs.unique }} run: | if [ "${{ matrix.name }}" == 'tutos-docker-gcc-py310' ]; then - echo "RUN_CMD=docker run --init -t --rm --name testrun devito_img" >> "$GITHUB_ENV" + echo "RUN_CMD=docker run \ + --init -t --rm \ + --name testrun \ + devito_img${UNIQUE}" >> "$GITHUB_ENV" else echo "RUN_CMD=" >> "$GITHUB_ENV" fi @@ -149,3 +170,10 @@ jobs: - name: Timestepping Notebooks run: | ${{ env.RUN_CMD }} py.test --nbval examples/timestepping + + - name: Cleanup Docker + if: "contains(matrix.name, 'docker')" + env: + UNIQUE: ${{ steps.uniquetag.outputs.unique }} + run: | + docker image rm -f "devito_img${UNIQUE}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8fbcc61d95..22160dbf9e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,6 +39,7 @@ repos: rev: v1.7.8 hooks: - id: actionlint-docker + args: [-oneline] - repo: https://github.com/hadolint/hadolint rev: v2.12.0 hooks: From 6afffaa6d7b978b36a4930d21a20cd9f246dc258 Mon Sep 17 00:00:00 2001 From: JDBetteridge Date: Thu, 5 Mar 2026 00:05:01 +0000 Subject: [PATCH 2/6] misc: Quoting --- .github/workflows/tutorials.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tutorials.yaml b/.github/workflows/tutorials.yaml index 575e4d07bb..90dc261fcd 100644 --- a/.github/workflows/tutorials.yaml +++ b/.github/workflows/tutorials.yaml @@ -99,7 +99,7 @@ jobs: echo "RUN_CMD=docker run \ --init -t --rm \ --name testrun \ - devito_img${UNIQUE}" >> "$GITHUB_ENV" + \"devito_img${UNIQUE}\"" >> "$GITHUB_ENV" else echo "RUN_CMD=" >> "$GITHUB_ENV" fi From 1d7a40a5927c65f8d34ea4e9612cb06a9f9af1ac Mon Sep 17 00:00:00 2001 From: JDBetteridge Date: Thu, 5 Mar 2026 00:13:31 +0000 Subject: [PATCH 3/6] misc: Quoting --- .github/workflows/tutorials.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tutorials.yaml b/.github/workflows/tutorials.yaml index 90dc261fcd..1315c9ecb4 100644 --- a/.github/workflows/tutorials.yaml +++ b/.github/workflows/tutorials.yaml @@ -96,12 +96,12 @@ jobs: UNIQUE: ${{ steps.uniquetag.outputs.unique }} run: | if [ "${{ matrix.name }}" == 'tutos-docker-gcc-py310' ]; then - echo "RUN_CMD=docker run \ + echo "RUN_CMD=docker run \ --init -t --rm \ - --name testrun \ + --name \"testrun\" \ \"devito_img${UNIQUE}\"" >> "$GITHUB_ENV" else - echo "RUN_CMD=" >> "$GITHUB_ENV" + echo "RUN_CMD=" >> "$GITHUB_ENV" fi id: set-run @@ -137,7 +137,7 @@ jobs: - name: Dask notebooks if: runner.os != 'macOS' run: | - ${{ env.RUN_CMD }} py.test --nbval examples/seismic/tutorials/*dask*.ipynb + ${{ env.RUN_CMD }} py.test --nbval examples/seismic/tutorials/*dask*.ipynb - name: Self-adjoint notebooks run: | From 116b1772109fcef6c6562e060c11aa3579de2116 Mon Sep 17 00:00:00 2001 From: JDBetteridge Date: Thu, 5 Mar 2026 00:31:21 +0000 Subject: [PATCH 4/6] misc: Editing the wrong file --- .github/workflows/pytest-core-mpi.yaml | 3 ++- .github/workflows/tutorials.yaml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest-core-mpi.yaml b/.github/workflows/pytest-core-mpi.yaml index aa8a07c715..6069727f50 100644 --- a/.github/workflows/pytest-core-mpi.yaml +++ b/.github/workflows/pytest-core-mpi.yaml @@ -116,7 +116,8 @@ jobs: --init -t --rm \ --env CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} \ --env OMP_NUM_THREADS=1 \ - --name "testrun devito_img${UNIQUE}" \ + --name testrun \ + "devito_img${UNIQUE}" \ pytest tests/test_mpi.py - name: Test examples with MPI diff --git a/.github/workflows/tutorials.yaml b/.github/workflows/tutorials.yaml index 1315c9ecb4..8562647c4c 100644 --- a/.github/workflows/tutorials.yaml +++ b/.github/workflows/tutorials.yaml @@ -98,8 +98,8 @@ jobs: if [ "${{ matrix.name }}" == 'tutos-docker-gcc-py310' ]; then echo "RUN_CMD=docker run \ --init -t --rm \ - --name \"testrun\" \ - \"devito_img${UNIQUE}\"" >> "$GITHUB_ENV" + --name testrun \ + devito_img${UNIQUE}" >> "$GITHUB_ENV" else echo "RUN_CMD=" >> "$GITHUB_ENV" fi From df0ae7b3b47cb8337604fcc8d3d438be24e8470a Mon Sep 17 00:00:00 2001 From: JDBetteridge Date: Thu, 5 Mar 2026 16:22:59 +0000 Subject: [PATCH 5/6] misc: Missing backslash --- .github/workflows/pytest-core-mpi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-core-mpi.yaml b/.github/workflows/pytest-core-mpi.yaml index 6069727f50..6923691723 100644 --- a/.github/workflows/pytest-core-mpi.yaml +++ b/.github/workflows/pytest-core-mpi.yaml @@ -125,7 +125,7 @@ jobs: UNIQUE: ${{ steps.uniquetag.outputs.unique }} run: | docker run \ - --init -t --rm + --init -t --rm \ ${{ matrix.mpiflag }} \ --env DEVITO_MPI=1 \ --env OMP_NUM_THREADS=1 \ From 5e1ab4763e2279d70db1ccea8e7863f850408098 Mon Sep 17 00:00:00 2001 From: JDBetteridge Date: Thu, 5 Mar 2026 18:28:11 +0000 Subject: [PATCH 6/6] misc: Kick CI