From fb4aa1c4e620fbe580ce929a0e4a1ac6d62154be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Tue, 25 Aug 2026 13:18:32 +0200 Subject: [PATCH 1/6] add ccache-buckets action --- .github/actions/ccache-buckets/action.yml | 91 +++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 .github/actions/ccache-buckets/action.yml diff --git a/.github/actions/ccache-buckets/action.yml b/.github/actions/ccache-buckets/action.yml new file mode 100644 index 000000000000..c4e9104bde73 --- /dev/null +++ b/.github/actions/ccache-buckets/action.yml @@ -0,0 +1,91 @@ +name: "ccache-buckets" +description: "Save/restore latest GitHub Actions ccache matching a key prefix to/from HF buckets" +inputs: + key: + description: "Cache key prefix to match and load" + required: true + folder: + description: "Bucket folder containing ccache files" + required: true + evict-old-files: + description: "Corresponds to the ccache --evict-older-than AGE option, where AGE is the number of seconds or days followed by the 's' or 'd' suffix respectively." + default: '' + save: + description: "Save ccache" + required: false + default: false + type: boolean + hf_bucket: + description: 'Hugging Face buckets path' + required: true + +runs: + using: "composite" + steps: + - name: Install Hugging Face Hub CLI + shell: bash + run: | + python3 -m venv .venv-hf + .venv-hf/bin/pip install -U huggingface_hub==1.28.0 + + - name: Restore ccache from buckets + if: ${{ inputs.save != 'true' }} + shell: bash + run: | + set +e -uo pipefail + source .venv-hf/bin/activate + CCACHE_DIR=$(ccache -k cache_dir) + if [[ -d "$CCACHE_DIR" ]]; then + CACHE_PATH=$(hf buckets list "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}" --json | jq -r '[.[] | select(.type == "file") | select(.path | startswith("${{ inputs.folder }}/${{ inputs.key }}") and endswith(".tar.gz"))] | sort_by(.path) | last | .path // ""') + if [[ -n "$CACHE_PATH" ]]; then + echo "Restoring ccache from '$CACHE_PATH'." + hf buckets cp "hf://buckets/${{ inputs.hf_bucket }}/$CACHE_PATH" ccache_bucket.tar.gz + mkdir -p ccache_bucket + if tar -xzf ccache_bucket.tar.gz -C ccache_bucket; then + rm -rf "$CCACHE_DIR" + mv ccache_bucket "$CCACHE_DIR" + ccache -z + fi + rm ccache_bucket.tar.gz + else + echo "No ccache found." + fi + else + echo "'$CCACHE_DIR' not found." + fi + + - name: Save ccache to buckets + if: ${{ inputs.save == 'true' }} + shell: bash + run: | + set +e -uo pipefail + source .venv-hf/bin/activate + CCACHE_DIR=$(ccache -k cache_dir) + if [[ -d "$CCACHE_DIR" ]]; then + ccache -s + if [[ -n "${{ inputs.evict-old-files }}" ]]; then + ccache --evict-older-than "${{ inputs.evict-old-files }}" + fi + DATESTAMP=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + CACHEFILE="${{ inputs.key }}-$DATESTAMP.tar.gz" + if tar -czf ccache_bucket.tar.gz -C "$CCACHE_DIR" .; then + hf buckets cp ccache_bucket.tar.gz "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}/$CACHEFILE" + fi + rm ccache_bucket.tar.gz + else + echo "'$CCACHE_DIR' not found." + fi + + - name: Remove old ccache files from buckets + if: ${{ inputs.save == 'true' }} + shell: bash + run: | + set +e -uo pipefail + source .venv-hf/bin/activate + CACHE_FILES=$(hf buckets list "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}" --json | jq -r '[.[] | select(.type == "file") | select(.path | startswith("${{ inputs.folder }}/${{ inputs.key }}") and endswith(".tar.gz"))] | sort_by(.path)[:-1] | .[] | [.path // ""] | @tsv') + if [[ -n "$CACHE_FILES" ]]; then + echo "Removing old ccache files..." + while IFS=$'\t' read -r CACHE_PATH; do + hf buckets rm "hf://buckets/${{ inputs.hf_bucket }}/$CACHE_PATH" -y + done <<< "$CACHE_FILES" + fi From a4a3e81f7309ddcd4bb1e536c0d9b932350d90d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Tue, 25 Aug 2026 13:27:26 +0200 Subject: [PATCH 2/6] use ccache-buckets --- .github/workflows/build-cuda-ubuntu.yml | 75 +++++++++++++++++-------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-cuda-ubuntu.yml b/.github/workflows/build-cuda-ubuntu.yml index ed5164756ae8..f4d395f81c7f 100644 --- a/.github/workflows/build-cuda-ubuntu.yml +++ b/.github/workflows/build-cuda-ubuntu.yml @@ -56,8 +56,16 @@ jobs: uses: ggml-org/ccache-action@v1.2.21 with: key: cuda-ubuntu-24.04-cuda - evict-old-files: 1d - save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: cuda-ubuntu-24.04-cuda + folder: llama.cpp + hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }} - name: Build with CMake # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project @@ -72,15 +80,16 @@ jobs: -DGGML_CUDA_CUB_3DOT2=ON cmake --build build - - name: ccache-clear - uses: ./.github/actions/ccache-clear + - name: ccache-buckets-save + uses: ./.github/actions/ccache-buckets env: - GH_TOKEN: ${{ github.token }} + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: key: cuda-ubuntu-24.04-cuda - older: 5m - min: 1 - dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }} + save: true hip: runs-on: ubuntu-22.04 @@ -101,8 +110,16 @@ jobs: uses: ggml-org/ccache-action@v1.2.21 with: key: cuda-ubuntu-22.04-hip - evict-old-files: 1d - save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: cuda-ubuntu-22.04-hip + folder: llama.cpp + hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }} - name: Build with native CMake HIP support id: cmake_build @@ -113,15 +130,16 @@ jobs: -DGGML_HIP=ON cmake --build build --config Release -j $(nproc) - - name: ccache-clear - uses: ./.github/actions/ccache-clear + - name: ccache-buckets-save + uses: ./.github/actions/ccache-buckets env: - GH_TOKEN: ${{ github.token }} + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: key: cuda-ubuntu-22.04-hip - older: 5m - min: 1 - dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }} + save: true musa: runs-on: ubuntu-22.04 @@ -142,8 +160,16 @@ jobs: uses: ggml-org/ccache-action@v1.2.21 with: key: cuda-ubuntu-22.04-musa - evict-old-files: 1d - save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + save: false + + - name: ccache-buckets-restore + uses: ./.github/actions/ccache-buckets + env: + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} + with: + key: cuda-ubuntu-22.04-musa + folder: llama.cpp + hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }} - name: Build with native CMake MUSA support id: cmake_build @@ -152,12 +178,13 @@ jobs: -DGGML_MUSA=ON time cmake --build build --config Release -j $(nproc) - - name: ccache-clear - uses: ./.github/actions/ccache-clear + - name: ccache-buckets-save + uses: ./.github/actions/ccache-buckets env: - GH_TOKEN: ${{ github.token }} + HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} with: key: cuda-ubuntu-22.04-musa - older: 5m - min: 1 - dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} + folder: llama.cpp + evict-old-files: 1d + hf_bucket: ${{ vars.HF_BUCKET_CACHE_OUTPUT }} + save: true From 3eb7b6147a0b0c4ba65f92231c544f4752d6cf22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Tue, 25 Aug 2026 13:33:03 +0200 Subject: [PATCH 3/6] only save on master --- .github/workflows/build-cuda-ubuntu.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-cuda-ubuntu.yml b/.github/workflows/build-cuda-ubuntu.yml index f4d395f81c7f..f7a23b9b3818 100644 --- a/.github/workflows/build-cuda-ubuntu.yml +++ b/.github/workflows/build-cuda-ubuntu.yml @@ -81,6 +81,7 @@ jobs: cmake --build build - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: ./.github/actions/ccache-buckets env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} @@ -131,6 +132,7 @@ jobs: cmake --build build --config Release -j $(nproc) - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: ./.github/actions/ccache-buckets env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} @@ -179,6 +181,7 @@ jobs: time cmake --build build --config Release -j $(nproc) - name: ccache-buckets-save + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: ./.github/actions/ccache-buckets env: HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }} From 0b656b4458aee9d262593f3f1d51f875a51359ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Tue, 25 Aug 2026 13:49:20 +0200 Subject: [PATCH 4/6] install python3-venv for hip --- .github/workflows/build-cuda-ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-cuda-ubuntu.yml b/.github/workflows/build-cuda-ubuntu.yml index f7a23b9b3818..9c3e571129b3 100644 --- a/.github/workflows/build-cuda-ubuntu.yml +++ b/.github/workflows/build-cuda-ubuntu.yml @@ -105,7 +105,7 @@ jobs: id: depends run: | sudo apt-get update - sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev + sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev python3-venv - name: ccache uses: ggml-org/ccache-action@v1.2.21 From f9a9e0b40d31e4021b448d73b6622cccf6b37edb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Tue, 25 Aug 2026 14:02:49 +0200 Subject: [PATCH 5/6] add jq and python3 for cuda --- .github/workflows/build-cuda-ubuntu.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-cuda-ubuntu.yml b/.github/workflows/build-cuda-ubuntu.yml index 9c3e571129b3..215299670ac4 100644 --- a/.github/workflows/build-cuda-ubuntu.yml +++ b/.github/workflows/build-cuda-ubuntu.yml @@ -50,7 +50,7 @@ jobs: DEBIAN_FRONTEND: noninteractive run: | apt update - apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev + apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev jq python3 python3-venv python3-pip - name: ccache uses: ggml-org/ccache-action@v1.2.21 @@ -105,7 +105,7 @@ jobs: id: depends run: | sudo apt-get update - sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev python3-venv + sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev jq python3-venv - name: ccache uses: ggml-org/ccache-action@v1.2.21 @@ -156,7 +156,7 @@ jobs: id: depends run: | apt-get update - apt-get install -y build-essential git cmake libssl-dev + apt-get install -y build-essential git cmake libssl-dev jq - name: ccache uses: ggml-org/ccache-action@v1.2.21 From b80336c5b7ab40061a474ac482ffd3bc53015380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Tue, 25 Aug 2026 14:34:17 +0200 Subject: [PATCH 6/6] only delete caches older than 5 minutes --- .github/actions/ccache-buckets/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/ccache-buckets/action.yml b/.github/actions/ccache-buckets/action.yml index c4e9104bde73..8eb65d275cb7 100644 --- a/.github/actions/ccache-buckets/action.yml +++ b/.github/actions/ccache-buckets/action.yml @@ -82,7 +82,7 @@ runs: run: | set +e -uo pipefail source .venv-hf/bin/activate - CACHE_FILES=$(hf buckets list "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}" --json | jq -r '[.[] | select(.type == "file") | select(.path | startswith("${{ inputs.folder }}/${{ inputs.key }}") and endswith(".tar.gz"))] | sort_by(.path)[:-1] | .[] | [.path // ""] | @tsv') + CACHE_FILES=$(hf buckets list "hf://buckets/${{ inputs.hf_bucket }}/${{ inputs.folder }}" --json | jq -r '[.[] | select(.type == "file") | select((.uploaded_at | .[:19]+"Z" | fromdateiso8601) < (now - 5 * 60)) | select(.path | startswith("${{ inputs.folder }}/${{ inputs.key }}") and endswith(".tar.gz"))] | sort_by(.path)[:-1] | .[] | [.path // ""] | @tsv') if [[ -n "$CACHE_FILES" ]]; then echo "Removing old ccache files..." while IFS=$'\t' read -r CACHE_PATH; do