mudler · mudler · Jun 17, 2026 · Jun 17, 2026 · Jun 17, 2026 · Jun 17, 2026
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -32,6 +32,45 @@ jobs:
         # job once a models bundle is published (Phase 4).
         run: ctest --test-dir build --output-on-failure -LE model
 
+  # -------------------------------------------------------------------------
+  # server-e2e: drive the real parakeet-server over HTTP.
+  #
+  # Runs on pull_request (merge gate) and manual workflow_dispatch. Not on every
+  # push: it downloads the ~125 MB tdt_ctc-110m-q4_k model via the alias path.
+  # Much lighter than closed-loop (no NeMo/Python venv) — it builds the server,
+  # then tests/server_e2e.sh starts it, transcribes tests/fixtures/speech.wav in
+  # json/text/verbose_json (with word timestamps), and checks the 400 paths.
+  # -------------------------------------------------------------------------
+  server-e2e:
+    if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y cmake build-essential curl ca-certificates
+
+      - name: Configure
+        run: |
+          cmake -B build \
+            -DPARAKEET_BUILD_SERVER=ON \
+            -DPARAKEET_BUILD_TESTS=ON \
+            -DGGML_NATIVE=OFF
+
+      - name: Build parakeet-server
+        run: cmake --build build --target parakeet-server -j
+
+      - name: Run server e2e
+        # PARAKEET_SERVER_E2E=1 flips the test from skip (77) to a real run.
+        env:
+          PARAKEET_SERVER_E2E: "1"
+        run: ctest --test-dir build --output-on-failure -R '^server_e2e$'
+
   # -------------------------------------------------------------------------
   # closed-loop: full end-to-end transcript assertion.
   #

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -1,12 +1,17 @@
 name: docker
 
-# Build the parakeet-cli container images and publish them to GitHub Container
-# Registry (ghcr.io/<owner>/parakeet.cpp-cli).
+# Build the parakeet container images and publish them to GitHub Container
+# Registry. Two images are shipped, one per binary:
+#   ghcr.io/<owner>/parakeet.cpp-cli     the command-line transcriber
+#   ghcr.io/<owner>/parakeet.cpp-server  the OpenAI-compatible HTTP server
+# Both come from the same Dockerfile (shared build stage, different runtime
+# target), so ggml is compiled once per build job.
 #
 # Each variant (cpu, cuda) is a multi-arch image (linux/amd64 + linux/arm64).
 # Every arch is built natively on its own runner (no QEMU): amd64 on
 # ubuntu-24.04, arm64 on ubuntu-24.04-arm. The per-arch images are pushed by
-# digest, then a merge job assembles one multi-arch manifest per variant.
+# digest, then a merge job assembles one multi-arch manifest per (image,
+# variant) pair.
 #
 # The CUDA images use the CUDA 13 base so ggml compiles the Blackwell
 # architectures (sm_120 + sm_121); that is what makes the arm64 CUDA image run
@@ -25,9 +30,11 @@ on:
 
 env:
   REGISTRY: ghcr.io
-  # Named for the binary it ships, so a future server image can live alongside
-  # it (e.g. parakeet.cpp-server). Resolves to <owner>/parakeet.cpp-cli.
-  IMAGE_NAME: ${{ github.repository }}-cli
+  # Each binary ships as its own image. Resolve to <owner>/parakeet.cpp-cli and
+  # <owner>/parakeet.cpp-server. Both are built from the same Dockerfile (shared
+  # build stage, different runtime target) in each build job below.
+  IMAGE_CLI: ${{ github.repository }}-cli
+  IMAGE_SERVER: ${{ github.repository }}-server
 
 jobs:
   # -------------------------------------------------------------------------
@@ -84,12 +91,17 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push by digest (${{ matrix.variant }}/${{ matrix.arch }})
-        id: build
+      # Both images come from the same Dockerfile. The cli build runs first and
+      # populates the gha cache for the shared `build` stage (which compiles
+      # ggml); the server build then reuses it via cache-from and only differs
+      # in its small runtime layer, so it is nearly free.
+      - name: Build and push cli by digest (${{ matrix.variant }}/${{ matrix.arch }})
+        id: build_cli
         uses: docker/build-push-action@v6
         with:
           context: .
           file: ./Dockerfile
+          target: runtime
           platforms: linux/${{ matrix.arch }}
           build-args: |
             BUILD_BASE=${{ matrix.build_base }}
@@ -98,23 +110,51 @@ jobs:
             CUDA_ARCHS=${{ matrix.cuda_archs }}
           # PRs: build only (cache-only, nothing pushed). Otherwise push the
           # image by digest so the merge job can stitch the arches together.
-          outputs: ${{ github.event_name != 'pull_request' && format('type=image,name={0}/{1},push-by-digest=true,name-canonical=true,push=true', env.REGISTRY, env.IMAGE_NAME) || 'type=cacheonly' }}
+          outputs: ${{ github.event_name != 'pull_request' && format('type=image,name={0}/{1},push-by-digest=true,name-canonical=true,push=true', env.REGISTRY, env.IMAGE_CLI) || 'type=cacheonly' }}
           cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }}
           cache-to: type=gha,mode=max,scope=${{ matrix.variant }}-${{ matrix.arch }}
 
-      - name: Export digest
+      - name: Build and push server by digest (${{ matrix.variant }}/${{ matrix.arch }})
+        id: build_server
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          target: runtime-server
+          platforms: linux/${{ matrix.arch }}
+          build-args: |
+            BUILD_BASE=${{ matrix.build_base }}
+            RUNTIME_BASE=${{ matrix.runtime_base }}
+            CMAKE_EXTRA_ARGS=${{ matrix.cmake_args }}
+            CUDA_ARCHS=${{ matrix.cuda_archs }}
+          outputs: ${{ github.event_name != 'pull_request' && format('type=image,name={0}/{1},push-by-digest=true,name-canonical=true,push=true', env.REGISTRY, env.IMAGE_SERVER) || 'type=cacheonly' }}
+          cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.variant }}-${{ matrix.arch }}
+
+      - name: Export digests
         if: github.event_name != 'pull_request'
         run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
+          mkdir -p /tmp/digests/cli /tmp/digests/server
+          cli="${{ steps.build_cli.outputs.digest }}"
+          srv="${{ steps.build_server.outputs.digest }}"
+          touch "/tmp/digests/cli/${cli#sha256:}"
+          touch "/tmp/digests/server/${srv#sha256:}"
 
-      - name: Upload digest
+      - name: Upload cli digest
         if: github.event_name != 'pull_request'
         uses: actions/upload-artifact@v4
         with:
-          name: digests-${{ matrix.variant }}-${{ matrix.arch }}
-          path: /tmp/digests/*
+          name: digests-cli-${{ matrix.variant }}-${{ matrix.arch }}
+          path: /tmp/digests/cli/*
+          if-no-files-found: error
+          retention-days: 1
+
+      - name: Upload server digest
+        if: github.event_name != 'pull_request'
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-server-${{ matrix.variant }}-${{ matrix.arch }}
+          path: /tmp/digests/server/*
           if-no-files-found: error
           retention-days: 1
 
@@ -131,18 +171,36 @@ jobs:
       packages: write
     strategy:
       fail-fast: false
+      # One manifest per (image, variant): cli + server, each cpu + cuda.
       matrix:
         include:
-          - variant: cpu
+          - image: cli
+            variant: cpu
             suffix: ""
-          - variant: cuda
+          - image: cli
+            variant: cuda
+            suffix: "-cuda"
+          - image: server
+            variant: cpu
+            suffix: ""
+          - image: server
+            variant: cuda
             suffix: "-cuda"
     steps:
+      - name: Resolve image name
+        id: img
+        run: |
+          if [ "${{ matrix.image }}" = "server" ]; then
+            echo "name=${{ env.IMAGE_SERVER }}" >> "$GITHUB_OUTPUT"
+          else
+            echo "name=${{ env.IMAGE_CLI }}" >> "$GITHUB_OUTPUT"
+          fi
+
       - name: Download digests
         uses: actions/download-artifact@v4
         with:
           path: /tmp/digests
-          pattern: digests-${{ matrix.variant }}-*
+          pattern: digests-${{ matrix.image }}-${{ matrix.variant }}-*
           merge-multiple: true
 
       - name: Set up Docker Buildx
@@ -159,7 +217,7 @@ jobs:
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          images: ${{ env.REGISTRY }}/${{ steps.img.outputs.name }}
           # cpu  -> latest, sha-xxxx, vX.Y.Z
           # cuda -> latest-cuda, sha-xxxx-cuda, vX.Y.Z-cuda
           flavor: |
@@ -174,9 +232,9 @@ jobs:
         run: |
           docker buildx imagetools create \
             $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
+            $(printf '${{ env.REGISTRY }}/${{ steps.img.outputs.name }}@sha256:%s ' *)
 
       - name: Inspect manifest
         run: |
           docker buildx imagetools inspect \
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest${{ matrix.suffix }}
+            ${{ env.REGISTRY }}/${{ steps.img.outputs.name }}:latest${{ matrix.suffix }}
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -9,6 +9,7 @@ endif()
 
 option(PARAKEET_BUILD_TESTS "Build ctest targets" OFF)
 option(PARAKEET_BUILD_CLI   "Build parakeet-cli" ON)
+option(PARAKEET_BUILD_SERVER "Build parakeet-server (OpenAI-compatible example)" ON)
 option(PARAKEET_SHARED      "Build libparakeet as shared" OFF)
 option(PARAKEET_GGML_CUDA   "Forward GGML_CUDA" OFF)
 option(PARAKEET_GGML_METAL  "Forward GGML_METAL" OFF)
@@ -102,6 +103,9 @@ target_link_libraries(parakeet PUBLIC ggml)
 if(PARAKEET_BUILD_CLI)
   add_subdirectory(examples/cli)
 endif()
+if(PARAKEET_BUILD_SERVER)
+  add_subdirectory(examples/server)
+endif()
 if(PARAKEET_BUILD_TESTS)
   enable_testing()
   add_subdirectory(tests)

diff --git a/Dockerfile b/Dockerfile
@@ -1,8 +1,11 @@
 # parakeet.cpp container image.
 #
-# Multi-stage build: a fat build stage compiles parakeet-cli (and the ggml
-# backends it links against), then a slim runtime stage carries only the
-# binary plus the ggml shared libraries.
+# Multi-stage build: a fat build stage compiles parakeet-cli and
+# parakeet-server (and the ggml backends they link against), then slim runtime
+# stages carry only one binary plus the ggml shared libraries. Two runtime
+# targets are exposed:
+#   --target runtime         the cli image (default)
+#   --target runtime-server  the OpenAI-compatible HTTP server image
 #
 # The same Dockerfile produces the CPU and CUDA variants. Select with build
 # args:
@@ -60,32 +63,56 @@ RUN cmake -B build \
         -DCMAKE_BUILD_TYPE=Release \
         -DGGML_NATIVE=OFF \
         -DPARAKEET_BUILD_CLI=ON \
+        -DPARAKEET_BUILD_SERVER=ON \
         -DPARAKEET_BUILD_TESTS=OFF \
         ${CMAKE_EXTRA_ARGS} \
         ${CUDA_ARCHS:+"-DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHS}"} \
     && cmake --build build -j"$(nproc)"
 
-# Stage the binary and every backend shared library (CPU, and CUDA when built)
-# into a clean prefix the runtime stage can copy wholesale.
+# Stage both binaries and every backend shared library (CPU, and CUDA when
+# built) into a clean prefix the runtime stages copy from. The cli and server
+# images each pick only the binary they ship.
 RUN mkdir -p /install/bin /install/lib \
     && cp build/examples/cli/parakeet-cli /install/bin/ \
+    && cp build/examples/server/parakeet-server /install/bin/ \
     && find build -name '*.so*' -exec cp -av {} /install/lib/ \;
 
 # ---------------------------------------------------------------------------
-# runtime: slim image with just the binary and its shared libraries.
+# runtime-base: shared slim layer with the ggml backend libraries. The cli and
+# server targets below add their own binary and entrypoint on top.
 # ---------------------------------------------------------------------------
-FROM ${RUNTIME_BASE} AS runtime
+FROM ${RUNTIME_BASE} AS runtime-base
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y --no-install-recommends \
         libgomp1 \
         ca-certificates \
     && rm -rf /var/lib/apt/lists/*
 
-COPY --from=build /install/bin/ /usr/local/bin/
 COPY --from=build /install/lib/ /usr/local/lib/
 RUN ldconfig
 
 WORKDIR /work
+
+# ---------------------------------------------------------------------------
+# runtime-server: the OpenAI-compatible HTTP server. Binds 0.0.0.0 so the
+# published port is reachable from outside the container; curl is added so
+# `--model <alias>` can fetch a published model on first run.
+# ---------------------------------------------------------------------------
+FROM runtime-base AS runtime-server
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=build /install/bin/parakeet-server /usr/local/bin/
+EXPOSE 8080
+ENTRYPOINT ["parakeet-server", "--host", "0.0.0.0"]
+CMD ["--help"]
+
+# ---------------------------------------------------------------------------
+# runtime: the cli image. Kept last so a plain `docker build .` (no --target)
+# still produces the cli image exactly as before.
+# ---------------------------------------------------------------------------
+FROM runtime-base AS runtime
+COPY --from=build /install/bin/parakeet-cli /usr/local/bin/
 ENTRYPOINT ["parakeet-cli"]
 CMD ["--help"]
diff --git a/README.md b/README.md
@@ -128,26 +128,35 @@ The CLI auto-selects the first GPU device the ggml registry reports (including i
 
 ## Docker
 
-Prebuilt images are published to GitHub Container Registry on every push to `master`. They contain just the `parakeet-cli` binary, so mount a converted `.gguf` model and your audio at runtime. Both the CPU and CUDA images are multi-arch (`linux/amd64` and `linux/arm64`), so the right one is pulled for your host automatically:
+Two prebuilt images are published to GitHub Container Registry on every push to `master`, one per binary:
+
+- `ghcr.io/mudler/parakeet.cpp-cli`: the command-line transcriber.
+- `ghcr.io/mudler/parakeet.cpp-server`: the [OpenAI-compatible server](#openai-compatible-server).
+
+Each comes in a CPU and a CUDA variant (the CUDA tag is suffixed `-cuda`), and both are multi-arch (`linux/amd64` and `linux/arm64`), so the right one is pulled for your host automatically. They contain just the binary, so mount a converted `.gguf` model (and, for the cli, your audio) at runtime:
 
 ```sh
-# CPU
+# CLI, CPU
 docker run --rm \
   -v "$PWD/models:/models:ro" \
   -v "$PWD/audio:/audio:ro" \
   ghcr.io/mudler/parakeet.cpp-cli:latest \
   transcribe --model /models/parakeet-tdt_ctc-110m-q5_k.gguf --input /audio/speech.wav --decoder tdt
 
-# CUDA (needs the nvidia container toolkit on the host)
+# CLI, CUDA (needs the nvidia container toolkit on the host)
 docker run --rm --gpus all \
   -v "$PWD/models:/models:ro" -v "$PWD/audio:/audio:ro" \
   ghcr.io/mudler/parakeet.cpp-cli:latest-cuda \
   transcribe --model /models/parakeet-tdt_ctc-110m-q5_k.gguf --input /audio/speech.wav --decoder tdt
+
+# Server: binds 0.0.0.0 and exposes 8080. Fetch a model by alias on first run,
+# or mount a local .gguf. Add --gpus all with the :latest-cuda tag for GPU.
+docker run --rm -p 8080:8080 ghcr.io/mudler/parakeet.cpp-server:latest --model tdt_ctc-110m
 ```
 
 The CUDA image is built on CUDA 13, so it covers everything from Turing up through Blackwell, including GB10 / Grace-Blackwell (DGX Spark) on arm64.
 
-To build the image yourself, see the build args at the top of the [`Dockerfile`](Dockerfile). The CPU image is the portable `GGML_NATIVE=OFF` build, so it runs on any amd64 or arm64 host.
+To build the images yourself, see the build args at the top of the [`Dockerfile`](Dockerfile); the cli is the default target and the server is `--target runtime-server`. The CPU image is the portable `GGML_NATIVE=OFF` build, so it runs on any amd64 or arm64 host.
 
 ---
 
@@ -240,6 +249,39 @@ The `parakeet-cli` binary lands at `build/examples/cli/parakeet-cli`.
 
 ---
 
+## OpenAI-compatible server
+
+`parakeet-server` is a small HTTP server that speaks the OpenAI transcription
+API, so any OpenAI client works by pointing its `base_url` at it. It is built by
+default (`PARAKEET_BUILD_SERVER=ON`) and lands at `build/examples/server/parakeet-server`.
+
+```sh
+# Serve a model. --model takes a local .gguf, an http(s) URL, a <name>.gguf in
+# mudler/parakeet-cpp-gguf, or an alias (downloaded and cached on first run).
+parakeet-server --model tdt_ctc-110m --port 8080
+
+# Transcribe over HTTP
+curl -F file=@audio.wav -F response_format=verbose_json \
+  http://localhost:8080/v1/audio/transcriptions
+```
+
+```python
+from openai import OpenAI
+client = OpenAI(base_url="http://localhost:8080/v1", api_key="not-needed")
+with open("audio.wav", "rb") as f:
+    print(client.audio.transcriptions.create(model="parakeet", file=f).text)
+```
+
+It supports `response_format` `json` / `text` / `verbose_json` and
+`timestamp_granularities[]=word`. This is a single-model, one-request-at-a-time
+example that accepts WAV uploads only; see [`examples/server/README.md`](examples/server/README.md)
+for the full list of options and known simplifications. **For a production
+deployment, use [LocalAI](https://localai.io)**, which embeds parakeet.cpp as a
+backend and adds a model gallery, concurrency, multi-model serving, the full
+OpenAI API surface, auth, and metrics.
+
+---
+
 ## Batching
 
 Single-clip transcription is the default and needs no flags: every `transcribe` call runs one clip at a time, byte-for-byte identical to before. Batching is an opt-in path for decoding several clips together, which matters when you serve many concurrent requests on a GPU.