[RHDHBUGS-2804] Fix Developer Lightspeed Safety Guard

.github/workflows/test.yml

@@ -52,6 +52,8 @@ jobs:
             cliArgs: "-f compose.yaml -f developer-lightspeed/compose.yaml"
           - name: "developer-lightspeed"
             cliArgs: "-f compose.yaml -f developer-lightspeed/compose-with-ollama.yaml"
+          - name: "developer-lightspeed-safety-guard"
+            cliArgs: "-f compose.yaml -f developer-lightspeed/compose-with-ollama.yaml -f developer-lightspeed/compose-with-safety-guard-ollama.yaml"
 
     name: "${{ matrix.tool }} compose - ${{ matrix.composeConfig.name }}${{ matrix.os != 'ubuntu-24.04' && format(' - {0}', matrix.os) || '' }}${{ matrix.userConfig == 'true' && ' - user config' || '' }}"
     runs-on: ${{ matrix.os }}

default.env

@@ -106,9 +106,13 @@ VERTEX_AI_PROJECT=
 # ------------------------------------------------------------------------------
 # Llama Guard Settings
 # ------------------------------------------------------------------------------
-## Defaults to llama-guard3:8b if not set
+# The safety provider uses inline::llama-guard, so SAFETY_MODEL must be a
+# llama-guard variant (e.g. llama-guard3:1b).
+# Defaults to llama-guard3:1b if not set.
 SAFETY_MODEL=
-## Defaults to http://host.docker.internal:11434/v1 if not set
+# SAFETY_URL is auto-configured by the compose files.
+# You should not need to set this manually.
 SAFETY_URL=
-## Only required for non-local environments with a api key
+# Only required if connecting to a remote safety endpoint that requires auth.
+# Not needed when using the local safety-ollama container (the default).
 SAFETY_API_KEY=

developer-lightspeed/README.md

@@ -62,7 +62,7 @@ Follow these steps to configure and launch Developer Lightspeed.
 
    **Step 2: Choose Safety Guard**
    - **No safety guard**: Allows any type of questions - Developer Lightspeed acts as a general-purpose assistant with no safety content filtering.
-   - **With safety guard**: Enables Llama Guard for content safety filtering - filters questions for safety content. Defaults to `llama-guard3:8b` with Ollama if no `SAFETY_MODEL` is configured.
+   - **With safety guard**: Enables Llama Guard for content safety filtering. A local Ollama container is automatically provisioned to run the safety model (`llama-guard3:1b` by default). No additional configuration is needed - the safety guard only supports llama-guard model variants.
 
    > [!NOTE]
    > **What does "Bring Your Own Model" mean?**
@@ -99,9 +99,9 @@ Follow these steps to configure and launch Developer Lightspeed.
     > [!IMPORTANT]
     > **Configuration Requirements:**
     > - **If you selected Ollama in step 1**: You don't need to modify any environment variables. The defaults work out of the box.
-    > - **If you selected Ollama + "With safety guard"**: No configuration needed! The setup automatically pulls `llama-guard3:8b` and configures the safety URL.
+    > - **If you selected Ollama + "With safety guard"**: No configuration needed! The setup automatically pulls `llama-guard3:1b` and configures the safety URL.
     > - **If you selected "Bring Your Own Model" in step 1**: You **must** configure at least one external LLM provider below before starting the application.
-    > - **If you selected "Bring Your Own Model" + "With safety guard"**: Optionally configure `SAFETY_MODEL`, `SAFETY_URL`, and `SAFETY_API_KEY`. Defaults to `llama-guard3:8b` via Ollama on host at `http://host.docker.internal:11434/v1`.
+    > - **If you selected "Bring Your Own Model" + "With safety guard"**: No safety configuration needed! A dedicated local Ollama container (`safety-ollama`) is automatically provisioned to run `llama-guard3:1b`. You only need to configure your external LLM provider.
 
     ### Quick Reference: Setup Combinations → Required Configuration
 
@@ -110,7 +110,7 @@ Follow these steps to configure and launch Developer Lightspeed.
     | **Ollama** | No safety guard | None (defaults work) | `compose.yaml` + `developer-lightspeed/compose-with-ollama.yaml` |
     | **Ollama** | With safety guard | None (auto-configured) | `compose.yaml` + `developer-lightspeed/compose-with-ollama.yaml` + `developer-lightspeed/compose-with-safety-guard-ollama.yaml` |
     | **Bring your own model** | No safety guard | At least one: vLLM, OpenAI, or Vertex AI | `compose.yaml` + `developer-lightspeed/compose.yaml` |
-    | **Bring your own model** | With safety guard | At least one provider + optional `SAFETY_*` vars | `compose.yaml` + `developer-lightspeed/compose.yaml` + `developer-lightspeed/compose-with-safety-guard.yaml` |
+    | **Bring your own model** | With safety guard | At least one: vLLM, OpenAI, or Vertex AI | `compose.yaml` + `developer-lightspeed/compose.yaml` + `developer-lightspeed/compose-with-safety-guard.yaml` |
 
     ---
 
@@ -202,35 +202,30 @@ Follow these steps to configure and launch Developer Lightspeed.
 
     ---
 
-    ### Configure Safety Guard (Only for "Bring Your Own Model")
+    ### Safety Guard Configuration
 
     > [!TIP]
-    > **If you selected Ollama provider + "With safety guard"**: No configuration needed! The setup automatically:
-    > - Pulls the `llama-guard3:8b` model into the containerized Ollama
-    > - Configures the safety URL to point to the containerized Ollama
-    > - Just works out of the box!
+    > **Safety guard is auto-configured for all provider combinations.** No manual safety configuration is needed regardless of whether you use Ollama or an external provider.
 
-    **If you selected "Bring Your Own Model" + "With safety guard"**, you can optionally configure the safety guard model. If not configured, it defaults to `llama-guard3:8b` via Ollama on your host machine.
+    When you select "With safety guard", the setup automatically provisions a local Ollama container to run the safety model:
+
+    - **Ollama provider**: The safety model (`llama-guard3:1b`) is pulled into the same Ollama container used for inference.
+    - **Bring Your Own Model**: A dedicated `safety-ollama` container is spun up solely for the safety model. Your external provider handles inference while the local Ollama handles safety filtering.
+
+    > [!IMPORTANT]
+    > **The safety provider uses `inline::llama-guard`**, which means `SAFETY_MODEL` **must** be a llama-guard variant (e.g. `llama-guard3:1b`). Other model families will not work for safety filtering.
+
+    > [!NOTE]
+    > **Safety model latency:** The default safety model is now `llama-guard3:1b`. Depending on your local CPU/GPU setup and container runtime, safety checks may still add noticeable latency, and in some environments may be slower than expected.
 
     ```env
-    # OPTIONAL: Safety guard model (default: llama-guard3:8b)
-    # This model is used by Llama Guard for content safety filtering
-    SAFETY_MODEL=llama-guard3:8b
-
-    # OPTIONAL: URL for the safety guard model endpoint (default: http://host.docker.internal:11434/v1)
-    # Points to Ollama on the host machine by default
-    # For external providers, set to your provider's OpenAI-compatible endpoint
-    SAFETY_URL=http://host.docker.internal:11434/v1
-
-    # OPTIONAL: API key for safety guard endpoint (only needed for non-local providers)
-    SAFETY_API_KEY=
+    # OPTIONAL: Override the default safety guard model (default: llama-guard3:1b)
+    # Must be a llama-guard variant
+    SAFETY_MODEL=llama-guard3:1b
     ```
 
-    > [!NOTE]
-    > **Safety Guard for External Providers:**
-    > - If no `SAFETY_MODEL` is configured, it defaults to `llama-guard3:8b`
-    > - If no `SAFETY_URL` is configured, it defaults to `http://host.docker.internal:11434/v1` (Ollama on host)
-    > - Ensure Ollama is running on your host machine with the llama-guard model, or configure an external safety endpoint
+    > [!WARNING]
+    > **Performance impact:** Running the llama-guard model locally can increase response latency, especially on machines with limited CPU, memory, or GPU resources. Every user message is evaluated by the safety model before being forwarded to the inference model, so constrained environments may experience noticeably slower responses. If you encounter this, consider increasing the memory allocated to your container runtime (see [Running Larger Models with Ollama](#running-larger-models-with-ollama)) or running without the guard.
 
 5. **Start the application**
 
@@ -249,7 +244,7 @@ Follow these steps to configure and launch Developer Lightspeed.
    > **Before starting:**
    > - **Ollama provider**: No configuration needed - works out of the box (including with safety guard)
    > - **Bring Your Own Model**: Ensure you've configured at least one external LLM provider in your `.env` file (see step 4 above)
-   > - **Bring Your Own Model + Safety Guard**: Optionally configure `SAFETY_MODEL`, `SAFETY_URL`, and `SAFETY_API_KEY`, or ensure Ollama is running on your host machine
+   > - **Bring Your Own Model + Safety Guard**: Only your external LLM provider needs configuration. The safety guard model is automatically provisioned locally via a dedicated `safety-ollama` container
 
 
 ---

developer-lightspeed/compose-with-ollama.yaml

@@ -47,7 +47,7 @@ services:
     entrypoint: [ "sh", "-c" ]
     restart: unless-stopped
     healthcheck:
-      test: ["CMD-SHELL", "ollama list | grep -q $$OLLAMA_MODEL && [ -f /tmp/ollama-model-ready ]"]
+      test: ollama list | grep -q $$OLLAMA_MODEL && [ -f /tmp/ollama-model-ready ]
       interval: 10s
       timeout: 10s
       retries: 40
@@ -80,6 +80,7 @@ services:
         required: false
     environment:
       ENABLE_OLLAMA: 'true'
+      OLLAMA_URL: http://ollama:11434
     healthcheck:
       test: ["CMD-SHELL", "curl -f http://localhost:8321/v1/models || exit 1"]
       interval: 5s

developer-lightspeed/compose-with-safety-guard-ollama.yaml

@@ -7,10 +7,10 @@ services:
     command: >
       "ollama serve & sleep 5 && 
       ollama pull ${OLLAMA_MODEL:-llama3.2:1b} && 
-      ollama pull ${SAFETY_MODEL:-llama-guard3:8b} && 
+      ollama pull ${SAFETY_MODEL:-llama-guard3:1b} && 
       touch /tmp/ollama-model-ready && wait"
     healthcheck:
-      test: ["CMD-SHELL", "ollama list | grep -q $$OLLAMA_MODEL && ollama list | grep -q $${SAFETY_MODEL:-llama-guard3} && [ -f /tmp/ollama-model-ready ]"]
+      test: ollama list | grep -q $$OLLAMA_MODEL && ollama list | grep -q $${SAFETY_MODEL:-llama-guard3:1b} && [ -f /tmp/ollama-model-ready ]
       interval: 10s
       timeout: 10s
       retries: 40

developer-lightspeed/compose-with-safety-guard.yaml

@@ -1,8 +1,49 @@
+# Safety Guard configuration for BYO (Bring Your Own) model provider
+# Spins up a dedicated Ollama container solely for the llama-guard safety model
+# Used when: BYO model + With safety guard is selected in start-lightspeed.sh
 services:
+  # Dedicated Ollama instance for the llama-guard safety model only
+  # Uses a separate container name to avoid collision with the inference Ollama service
+  safety-ollama:
+    image: docker.io/ollama/ollama:0.10.0
+    container_name: safety-ollama
+    volumes:
+      - safety_ollama_data:/root/.ollama
+    env_file:
+      - path: "./default.env"
+        required: true
+      - path: "./.env"
+        required: false
+    command: >
+      "ollama serve & sleep 5 &&
+      ollama pull ${SAFETY_MODEL:-llama-guard3:1b} &&
+      touch /tmp/ollama-model-ready && wait"
+    entrypoint: [ "sh", "-c" ]
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "ollama list | grep -q $${SAFETY_MODEL:-llama-guard3:1b} && [ -f /tmp/ollama-model-ready ]"]
+      interval: 10s
+      timeout: 10s
+      retries: 40
+      start_period: 90s
+
   llama-stack:
+    depends_on:
+      safety-ollama:
+        condition: service_healthy
     volumes:
       - ./developer-lightspeed/configs/extra-files/run.yaml:/app-root/run.yaml:Z
       - rag_embeddings:/rag-content/embeddings_model
       - rag_vector_db:/rag-content/vector_db
       # Set VERTEX_AI_CREDENTIALS_PATH in your .env file to your Google Cloud credentials JSON file path
       - ${VERTEX_AI_CREDENTIALS_PATH:-./developer-lightspeed/configs/extra-files/templates/placeholder.json}:/app-root/credentials.json:Z
+    environment:
+      SAFETY_URL: http://safety-ollama:11434/v1
+
+  lightspeed-core-service:
+    depends_on:
+      safety-ollama:
+        condition: service_healthy
+
+volumes:
+  safety_ollama_data:

developer-lightspeed/configs/extra-files/run-no-guard.yaml

@@ -1,7 +1,7 @@
 # ==============================================================================
 # Llama Stack Configuration - Without Safety Guard
 # ==============================================================================
-# This configuration file is used when safety guard is diasbeld
+# This configuration file is used when safety guard is disabled
 # (selected "No safety guard" in step 2 of start-lightspeed.sh)
 # It configures Llama Stack to provide LLM inference and RAG capabilities
 # without safety guard to filter user questions for safety content

developer-lightspeed/configs/extra-files/run.yaml

@@ -125,16 +125,16 @@ registered_resources:
       model_type: embedding
       provider_id: sentence-transformers
       provider_model_id: /rag-content/embeddings_model
-    - model_id: ${env.SAFETY_MODEL:=llama-guard3:8b}
+    - model_id: ${env.SAFETY_MODEL:=llama-guard3:1b}
       provider_id: safety-guard
-      provider_model_id: ${env.SAFETY_MODEL:=llama-guard3:8b}
+      provider_model_id: ${env.SAFETY_MODEL:=llama-guard3:1b}
       model_type: llm
       metadata: {}
   # Safety shields that apply safety shields to user questions
   shields:
     - shield_id: llama-guard-shield
       provider_id: llama-guard
-      provider_shield_id: safety-guard/${env.SAFETY_MODEL:=llama-guard3:8b}
+      provider_shield_id: safety-guard/${env.SAFETY_MODEL:=llama-guard3:1b}
   # Defines groups of tools available to the LLM
   tool_groups:
     - provider_id: rag-runtime

developer-lightspeed/scripts/start-lightspeed.sh

@@ -56,9 +56,9 @@ show_safety_guard_menu() {
     echo "   - Recommended for users who know what they are doing"
     echo ""
     echo "2) With safety guard"
-    echo "   - Filters questions for safety content only"
-    echo "   - Defaults to llama-guard3:8b with Ollama"
-    echo "   - Recommended for users who want to be safe"
+    echo "   - Filters questions for safety content using Llama Guard"
+    echo "   - Automatically provisions llama-guard3:1b locally via Ollama"
+    echo "   - No additional configuration needed"
     echo ""
 }
 

developer-lightspeed/scripts/stop-lightspeed.sh

@@ -62,13 +62,18 @@ detect_compose_config() {
     local runtime=$1
 
     local has_ollama=false
+    local has_safety_ollama=false
     local has_lightspeed=false
 
     # Check for Lightspeed-specific containers
     if is_container_running "$runtime" "ollama"; then
         has_ollama=true
     fi
 
+    if is_container_running "$runtime" "safety-ollama"; then
+        has_safety_ollama=true
+    fi
+
     if is_container_running "$runtime" "lightspeed-core-service"; then
         has_lightspeed=true
     fi
@@ -79,14 +84,23 @@ detect_compose_config() {
         return
     fi
 
-    # Determine provider type (Ollama vs external)
+    # Determine provider type:
+    # - ollama container (inference) running → Ollama provider
+    # - safety-ollama container (no ollama) → BYO provider with local safety guard
+    # - neither → BYO provider without safety guard
     local provider=""
     if [[ "$has_ollama" == true ]]; then
         provider="ollama"
     else
         provider="external"
     fi
 
+    # safety-ollama is the dedicated safety container used in BYO + Safety Guard mode
+    if [[ "$has_safety_ollama" == true && "$has_ollama" == false ]]; then
+        echo "external-guard"
+        return
+    fi
+
     # Check safety guard mode by inspecting the mounted run.yaml file
     local safety_guard_mode
     safety_guard_mode=$(check_safety_guard_mode "$runtime")