Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 49 additions & 4 deletions .github/workflows/pytest-core-mpi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ jobs:
test-mpi-docker:
name: pytest-mpi
runs-on: ${{ matrix.os }}
outputs:
unique : ${{ steps.uniquetag.outputs.unique }}
strategy:
matrix:
name: [gcc, gcc-arm, icx]
Expand All @@ -89,15 +91,58 @@ jobs:
- name: Checkout devito
uses: actions/checkout@v6

- name: Generate unique CI tag
id: uniquetag
run: |
UNIQUE=$(echo "${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" | cksum | cut -f 1 -d " ")
echo "Unique ID: ${UNIQUE}"
echo "unique=${UNIQUE}" >> "$GITHUB_OUTPUT"

- name: Build docker image
env:
UNIQUE: ${{ steps.uniquetag.outputs.unique }}
run: |
docker build . --file docker/Dockerfile.devito --tag devito_img --build-arg base=devitocodes/bases:cpu-${{ matrix.arch }}
docker build \
--file docker/Dockerfile.devito \
--tag "devito_img${UNIQUE}" \
--build-arg base=devitocodes/bases:cpu-${{ matrix.arch }} \
.

- name: Test with pytest
env:
UNIQUE: ${{ steps.uniquetag.outputs.unique }}
run: |
docker run --init -t --rm -e CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} -e OMP_NUM_THREADS=1 --name testrun devito_img pytest tests/test_mpi.py
docker run \
--init -t --rm \
--env CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} \
--env OMP_NUM_THREADS=1 \
--name testrun \
"devito_img${UNIQUE}" \
pytest tests/test_mpi.py

- name: Test examples with MPI
env:
UNIQUE: ${{ steps.uniquetag.outputs.unique }}
run: |
docker run \
--init -t --rm \
${{ matrix.mpiflag }} \
--env DEVITO_MPI=1 \
--env OMP_NUM_THREADS=1 \
--name examplerun \
"devito_img${UNIQUE}" \
mpiexec -n 2 pytest examples/seismic/acoustic
#
docker run \
--init -t --rm \
--env DEVITO_MPI=1 \
--env OMP_NUM_THREADS=1 \
--name examplerun \
"devito_img${UNIQUE}" \
mpiexec -n 2 pytest examples/seismic/tti

- name: Cleanup
env:
UNIQUE: ${{ steps.uniquetag.outputs.unique }}
run: |
docker run --init -t --rm ${{ matrix.mpiflag }} -e DEVITO_MPI=1 -e OMP_NUM_THREADS=1 --name examplerun devito_img mpiexec -n 2 pytest examples/seismic/acoustic
docker run --init -t --rm -e DEVITO_MPI=1 -e OMP_NUM_THREADS=1 --name examplerun devito_img mpiexec -n 2 pytest examples/seismic/tti
docker image rm -f "devito_img${UNIQUE}"
45 changes: 41 additions & 4 deletions .github/workflows/pytest-core-nompi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ jobs:
DEVITO_LANGUAGE: ${{ matrix.language }}
OMP_NUM_THREADS: 2

outputs:
unique : ${{ steps.uniquetag.outputs.unique }}

strategy:
# Prevent all build to stop if a single one fails
fail-fast: false
Expand Down Expand Up @@ -146,15 +149,34 @@ jobs:
python-version: ${{ matrix.python-version }}
allow-prereleases: true

- name: Generate unique CI tag
id: uniquetag
run: |
UNIQUE=$(echo "${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" | cksum | cut -f 1 -d " ")
echo "Unique ID: ${UNIQUE}"
echo "unique=${UNIQUE}" >> "$GITHUB_OUTPUT"

- name: Build docker image
if: contains(matrix.name, 'docker')
env:
UNIQUE: ${{ steps.uniquetag.outputs.unique }}
run: |
docker build . --file docker/Dockerfile.devito --tag devito_img --build-arg base=devitocodes/bases:cpu-${{ matrix.arch }}
docker build \
--file docker/Dockerfile.devito \
--tag "devito_img${UNIQUE}" \
--build-arg base=devitocodes/bases:cpu-${{ matrix.arch }} \
.

- name: Set run prefix
env:
UNIQUE: ${{ steps.uniquetag.outputs.unique }}
run: |
if [[ "${{ matrix.name }}" =~ "docker" ]]; then
echo "RUN_CMD=docker run --init -t --rm -e CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} --name testrun devito_img" >> "$GITHUB_ENV"
echo "RUN_CMD=docker run \
--init -t --rm \
--env CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} \
--name testrun \
devito_img${UNIQUE}" >> "$GITHUB_ENV"
else
echo "RUN_CMD=" >> "$GITHUB_ENV"
fi
Expand Down Expand Up @@ -207,15 +229,30 @@ jobs:

- name: Check configuration
run: |
${{ env.RUN_CMD }} python3 -c "from devito import configuration; print(''.join(['%s: %s \n' % (k, v) for (k, v) in configuration.items()]))"
${{ env.RUN_CMD }} python3 \
-c "from devito import configuration; \
print(''.join(['%s: %s \n' % (k, v) for (k, v) in configuration.items()]))"

- name: Test with pytest
run: |
${{ env.RUN_CMD }} pytest -k "${{ matrix.test-set }}" -m "not parallel" --cov --cov-config=.coveragerc --cov-report=xml tests/
${{ env.RUN_CMD }} pytest \
-k "${{ matrix.test-set }}" \
-m "not parallel" \
--cov \
--cov-config=.coveragerc \
--cov-report=xml \
tests/

- name: Upload coverage to Codecov
if: "!contains(matrix.name, 'docker')"
uses: codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
name: ${{ matrix.name }}

- name: Cleanup Docker
if: "contains(matrix.name, 'docker')"
env:
UNIQUE: ${{ steps.uniquetag.outputs.unique }}
run: |
docker image rm -f "devito_img${UNIQUE}"
41 changes: 30 additions & 11 deletions .github/workflows/pytest-gpu.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ jobs:
- self-hosted
- ${{ matrix.runner_label }}

outputs:
unique : ${{ steps.uniquetag.outputs.unique }}

strategy:
fail-fast: false
matrix:
Expand All @@ -52,7 +55,7 @@ jobs:
test_drive_cmd: "nvidia-smi"
# Respect CUDA_VISIBLE_DEVICES and also hard-limit Docker to that device.
# NOTE: CUDA_VISIBLE_DEVICES must be set by the runner (systemd drop-in etc.).
flags: >-
dockerflags: >-
--init --rm -t
--name ${CONTAINER_BASENAME}
--gpus "device=${CUDA_VISIBLE_DEVICES:-all}"
Expand All @@ -64,7 +67,7 @@ jobs:
base: "devitocodes/bases:amd"
test_drive_cmd: "rocm-smi"
# Unchanged, still passes through required /dev nodes etc.
flags: >-
dockerflags: >-
--init --network=host
--device=/dev/kfd --device=/dev/dri
--ipc=host
Expand All @@ -77,9 +80,18 @@ jobs:
- name: Checkout devito
uses: actions/checkout@v6

- name: Generate unique CI tag
id: uniquetag
run: |
UNIQUE=$(echo "${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" | cksum | cut -f 1 -d " ")
echo "Unique ID: ${UNIQUE}"
echo "unique=${UNIQUE}" >> "$GITHUB_OUTPUT"

- name: Set per-runner tags
env:
UNIQUE: ${{ steps.uniquetag.outputs.unique }}
run: |
echo "DOCKER_IMAGE=${{ matrix.name }}-${RUNNER_NAME// /_}" >> "$GITHUB_ENV"
echo "DOCKER_IMAGE=${{ matrix.name }}-${RUNNER_NAME// /_}-${UNIQUE}" >> "$GITHUB_ENV"
echo "CONTAINER_BASENAME=testrun-${{ matrix.name }}-${RUNNER_NAME// /_}-${{ github.sha }}" >> "$GITHUB_ENV"

- name: Ensure buildx builder
Expand Down Expand Up @@ -112,7 +124,7 @@ jobs:

# Run a simple driver-probe command (nvidia-smi / rocm-smi)
docker rm -f "${CONTAINER_BASENAME}" 2>/dev/null || true
docker run ${{ matrix.flags }} "${DOCKER_IMAGE}" ${{ matrix.test_drive_cmd }}
docker run ${{ matrix.dockerflags }} "${DOCKER_IMAGE}" ${{ matrix.test_drive_cmd }}

- name: Test with pytest
env:
Expand All @@ -123,12 +135,13 @@ jobs:
ci_env=$(bash <(curl -s https://codecov.io/env))

# Run the test suite using the matrix-defined flags
docker run ${{ matrix.flags }} \
docker run \
${{ matrix.dockerflags }} \
"${ci_env}" \
-e CI=true \
-e PYTHONFAULTHANDLER=1 \
-e DEVITO_LOGGING=DEBUG \
-e CODECOV_TOKEN \
--env CI=true \
--env PYTHONFAULTHANDLER=1 \
--env DEVITO_LOGGING=DEBUG \
--env CODECOV_TOKEN \
"${DOCKER_IMAGE}" \
pytest -vvv --capture=no --showlocals \
--log-cli-level=DEBUG -o log_cli=true \
Expand All @@ -138,11 +151,17 @@ jobs:

- name: Test examples
run: |
docker run ${{ matrix.flags }} "${DOCKER_IMAGE}" pytest ${{ matrix.test_examples }}
docker run \
${{ matrix.dockerflags }} \
"${DOCKER_IMAGE}" \
pytest ${{ matrix.test_examples }}

- name: Test examples with MPI
run: |
docker run ${{ matrix.flags }} --env DEVITO_MPI=1 "${DOCKER_IMAGE}" \
docker run \
${{ matrix.dockerflags }} \
--env DEVITO_MPI=1 \
"${DOCKER_IMAGE}" \
mpiexec -n 2 pytest ${{ matrix.test_examples }}

- name: Builder & image cleanup (keep 3 days of cache)
Expand Down
36 changes: 32 additions & 4 deletions .github/workflows/tutorials.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ jobs:
DEVITO_ARCH: "${{ matrix.compiler }}"
DEVITO_LANGUAGE: ${{ matrix.language }}

outputs:
unique : ${{ steps.uniquetag.outputs.unique }}

strategy:
# Prevent all build to stop if a single one fails
fail-fast: false
Expand Down Expand Up @@ -59,6 +62,13 @@ jobs:
- name: Checkout devito
uses: actions/checkout@v6

- name: Generate unique CI tag
id: uniquetag
run: |
UNIQUE=$(echo "${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" | cksum | cut -f 1 -d " ")
echo "Unique ID: ${UNIQUE}"
echo "unique=${UNIQUE}" >> "$GITHUB_OUTPUT"

- name: Set up Python ${{ matrix.pyver }}
if: "!contains(matrix.name, 'docker')"
uses: actions/setup-python@v6
Expand All @@ -72,15 +82,26 @@ jobs:

- name: Build docker image
if: "contains(matrix.name, 'docker')"
env:
UNIQUE: ${{ steps.uniquetag.outputs.unique }}
run: |
docker build . --pull --file docker/Dockerfile.devito --tag devito_img
docker build \
--pull \
--file docker/Dockerfile.devito \
--tag "devito_img${UNIQUE}" \
.

- name: Set run prefix
env:
UNIQUE: ${{ steps.uniquetag.outputs.unique }}
run: |
if [ "${{ matrix.name }}" == 'tutos-docker-gcc-py310' ]; then
echo "RUN_CMD=docker run --init -t --rm --name testrun devito_img" >> "$GITHUB_ENV"
echo "RUN_CMD=docker run \
--init -t --rm \
--name testrun \
devito_img${UNIQUE}" >> "$GITHUB_ENV"
else
echo "RUN_CMD=" >> "$GITHUB_ENV"
echo "RUN_CMD=" >> "$GITHUB_ENV"
fi
id: set-run

Expand Down Expand Up @@ -116,7 +137,7 @@ jobs:
- name: Dask notebooks
if: runner.os != 'macOS'
run: |
${{ env.RUN_CMD }} py.test --nbval examples/seismic/tutorials/*dask*.ipynb
${{ env.RUN_CMD }} py.test --nbval examples/seismic/tutorials/*dask*.ipynb

- name: Self-adjoint notebooks
run: |
Expand Down Expand Up @@ -149,3 +170,10 @@ jobs:
- name: Timestepping Notebooks
run: |
${{ env.RUN_CMD }} py.test --nbval examples/timestepping

- name: Cleanup Docker
if: "contains(matrix.name, 'docker')"
env:
UNIQUE: ${{ steps.uniquetag.outputs.unique }}
run: |
docker image rm -f "devito_img${UNIQUE}"
1 change: 1 addition & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ repos:
rev: v1.7.8
hooks:
- id: actionlint-docker
args: [-oneline]
- repo: https://github.com/hadolint/hadolint
rev: v2.12.0
hooks:
Expand Down
Loading