feat(gallery): add multi-select usecase filtering, backend-aware filters, and async estimates

Move gallery model listing to a cached implementation with background
refresh so the page loads instantly. Add per-model VRAM/size estimate
endpoint so estimates load asynchronously without blocking the gallery.
Introduce backend capabilities map as a single source of truth for what
each backend supports, powering a new /api/backends/usecases endpoint
that lets the UI grey out unavailable filter buttons when a backend is
selected. Gallery filters now support multi-select (comma-separated
tags with AND-match filtering) and a dedicated "multimodal" filter.
Includes Playwright e2e tests for the new filter behaviors.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: richiejp

core/config/backend_capabilities.go

@@ -0,0 +1,116 @@
+package config
+
+import (
+	"slices"
+	"strings"
+	"testing"
+)
+
+func TestBackendCapabilities_AllHaveUsecases(t *testing.T) {
+	for name, cap := range BackendCapabilities {
+		if len(cap.PossibleUsecases) == 0 {
+			t.Errorf("backend %q has no possible usecases", name)
+		}
+		if len(cap.DefaultUsecases) == 0 {
+			t.Errorf("backend %q has no default usecases", name)
+		}
+		if len(cap.GRPCMethods) == 0 {
+			t.Errorf("backend %q has no gRPC methods", name)
+		}
+	}
+}
+
+func TestBackendCapabilities_DefaultsSubsetOfPossible(t *testing.T) {
+	for name, cap := range BackendCapabilities {
+		for _, d := range cap.DefaultUsecases {
+			if !slices.Contains(cap.PossibleUsecases, d) {
+				t.Errorf("backend %q: default %q not in possible %v", name, d, cap.PossibleUsecases)
+			}
+		}
+	}
+}
+
+func TestBackendCapabilities_UsecasesMatchFlags(t *testing.T) {
+	allFlags := GetAllModelConfigUsecases()
+	for name, cap := range BackendCapabilities {
+		for _, u := range cap.PossibleUsecases {
+			info, ok := UsecaseInfoMap[u]
+			if !ok {
+				t.Errorf("backend %q: usecase %q not in UsecaseInfoMap", name, u)
+				continue
+			}
+			flagName := "FLAG_" + strings.ToUpper(u)
+			if _, ok := allFlags[flagName]; !ok {
+				// Try without transform — some names differ
+				found := false
+				for _, flag := range allFlags {
+					if flag == info.Flag {
+						found = true
+						break
+					}
+				}
+				if !found {
+					t.Errorf("backend %q: usecase %q flag %d not in GetAllModelConfigUsecases", name, u, info.Flag)
+				}
+			}
+		}
+	}
+}
+
+func TestUsecaseInfoMap_AllHaveFlags(t *testing.T) {
+	for name, info := range UsecaseInfoMap {
+		if info.Flag == FLAG_ANY {
+			t.Errorf("usecase %q has FLAG_ANY (zero) — should have a real flag", name)
+		}
+		if info.GRPCMethod == "" {
+			t.Errorf("usecase %q has no gRPC method", name)
+		}
+	}
+}
+
+func TestGetBackendCapability(t *testing.T) {
+	cap := GetBackendCapability("llama-cpp")
+	if cap == nil {
+		t.Fatal("llama-cpp should be known")
+	}
+	if !slices.Contains(cap.PossibleUsecases, "chat") {
+		t.Error("llama-cpp should support chat")
+	}
+}
+
+func TestGetBackendCapability_Normalize(t *testing.T) {
+	cap := GetBackendCapability("llama.cpp")
+	if cap == nil {
+		t.Fatal("llama.cpp should normalize to llama-cpp")
+	}
+}
+
+func TestGetBackendCapability_Unknown(t *testing.T) {
+	cap := GetBackendCapability("nonexistent")
+	if cap != nil {
+		t.Error("unknown backend should return nil")
+	}
+}
+
+func TestIsValidUsecaseForBackend(t *testing.T) {
+	if !IsValidUsecaseForBackend("piper", "tts") {
+		t.Error("piper should support tts")
+	}
+	if IsValidUsecaseForBackend("piper", "chat") {
+		t.Error("piper should not support chat")
+	}
+	// Unknown backend is permissive
+	if !IsValidUsecaseForBackend("unknown", "anything") {
+		t.Error("unknown backend should allow any usecase")
+	}
+}
+
+func TestAllBackendNames(t *testing.T) {
+	names := AllBackendNames()
+	if len(names) < 30 {
+		t.Errorf("expected 30+ backends, got %d", len(names))
+	}
+	if !slices.IsSorted(names) {
+		t.Error("should be sorted")
+	}
+}

core/config/backend_capabilities_test.go

@@ -565,11 +565,39 @@ const (
 	FLAG_VAD              ModelConfigUsecase = 0b010000000000
 	FLAG_VIDEO            ModelConfigUsecase = 0b100000000000
 	FLAG_DETECTION        ModelConfigUsecase = 0b1000000000000
+	FLAG_VISION           ModelConfigUsecase = 0b10000000000000
 
 	// Common Subsets
 	FLAG_LLM ModelConfigUsecase = FLAG_CHAT | FLAG_COMPLETION | FLAG_EDIT
 )
 
+// ModalityGroups defines groups of usecases that belong to the same modality.
+// Flags within the same group are NOT orthogonal (e.g., chat and completion are
+// both text/language). A model is multimodal when its usecases span 2+ groups.
+var ModalityGroups = []ModelConfigUsecase{
+	FLAG_CHAT | FLAG_COMPLETION | FLAG_EDIT, // text/language
+	FLAG_VISION | FLAG_DETECTION,            // visual understanding
+	FLAG_TRANSCRIPT,                         // speech input
+	FLAG_TTS | FLAG_SOUND_GENERATION,        // audio output
+	FLAG_IMAGE | FLAG_VIDEO,                 // visual generation
+}
+
+// IsMultimodal returns true if the given usecases span two or more orthogonal
+// modality groups. For example chat+vision is multimodal, but chat+completion
+// is not (both belong to the text/language group).
+func IsMultimodal(usecases ModelConfigUsecase) bool {
+	groupCount := 0
+	for _, group := range ModalityGroups {
+		if usecases&group != 0 {
+			groupCount++
+			if groupCount >= 2 {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func GetAllModelConfigUsecases() map[string]ModelConfigUsecase {
 	return map[string]ModelConfigUsecase{
 		// Note: FLAG_ANY is intentionally excluded from this map
@@ -588,6 +616,7 @@ func GetAllModelConfigUsecases() map[string]ModelConfigUsecase {
 		"FLAG_LLM":              FLAG_LLM,
 		"FLAG_VIDEO":            FLAG_VIDEO,
 		"FLAG_DETECTION":        FLAG_DETECTION,
+		"FLAG_VISION":           FLAG_VISION,
 	}
 }
 

core/config/model_config.go

@@ -7,6 +7,8 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/lithammer/fuzzysearch/fuzzy"
@@ -92,6 +94,34 @@ func (gm GalleryElements[T]) Search(term string) GalleryElements[T] {
 	return filteredModels
 }
 
+// FilterGalleryModelsByUsecase returns models whose known_usecases include all
+// the bits set in usecase. For example, passing FLAG_CHAT matches any model
+// with the chat usecase; passing FLAG_CHAT|FLAG_VISION matches only models
+// that have both.
+func FilterGalleryModelsByUsecase(models GalleryElements[*GalleryModel], usecase config.ModelConfigUsecase) GalleryElements[*GalleryModel] {
+	var filtered GalleryElements[*GalleryModel]
+	for _, m := range models {
+		u := m.GetKnownUsecases()
+		if u != nil && (*u&usecase) == usecase {
+			filtered = append(filtered, m)
+		}
+	}
+	return filtered
+}
+
+// FilterGalleryModelsByMultimodal returns models whose known_usecases span two
+// or more orthogonal modality groups (e.g. chat+vision, tts+transcript).
+func FilterGalleryModelsByMultimodal(models GalleryElements[*GalleryModel]) GalleryElements[*GalleryModel] {
+	var filtered GalleryElements[*GalleryModel]
+	for _, m := range models {
+		u := m.GetKnownUsecases()
+		if u != nil && config.IsMultimodal(*u) {
+			filtered = append(filtered, m)
+		}
+	}
+	return filtered
+}
+
 func (gm GalleryElements[T]) FilterByTag(tag string) GalleryElements[T] {
 	var filtered GalleryElements[T]
 	for _, m := range gm {
@@ -267,6 +297,69 @@ func AvailableGalleryModels(galleries []config.Gallery, systemState *system.Syst
 	return models, nil
 }
 
+var (
+	availableModelsMu    sync.RWMutex
+	availableModelsCache GalleryElements[*GalleryModel]
+	refreshing           atomic.Bool
+)
+
+// AvailableGalleryModelsCached returns gallery models from an in-memory cache.
+// Local-only fields (installed status) are refreshed on every call. A background
+// goroutine is triggered to re-fetch the full model list (including network
+// calls) so subsequent requests pick up changes without blocking the caller.
+// The first call with an empty cache blocks until the initial load completes.
+func AvailableGalleryModelsCached(galleries []config.Gallery, systemState *system.SystemState) (GalleryElements[*GalleryModel], error) {
+	availableModelsMu.RLock()
+	cached := availableModelsCache
+	availableModelsMu.RUnlock()
+
+	if cached != nil {
+		// Refresh installed status under write lock to avoid races with
+		// concurrent readers and the background refresh goroutine.
+		availableModelsMu.Lock()
+		for _, m := range cached {
+			_, err := os.Stat(filepath.Join(systemState.Model.ModelsPath, fmt.Sprintf("%s.yaml", m.GetName())))
+			m.SetInstalled(err == nil)
+		}
+		availableModelsMu.Unlock()
+		// Trigger a background refresh if one is not already running.
+		triggerGalleryRefresh(galleries, systemState)
+		return cached, nil
+	}
+
+	// No cache yet — must do a blocking load.
+	models, err := AvailableGalleryModels(galleries, systemState)
+	if err != nil {
+		return nil, err
+	}
+
+	availableModelsMu.Lock()
+	availableModelsCache = models
+	availableModelsMu.Unlock()
+
+	return models, nil
+}
+
+// triggerGalleryRefresh starts a background goroutine that refreshes the
+// gallery model cache. Only one refresh runs at a time; concurrent calls
+// are no-ops.
+func triggerGalleryRefresh(galleries []config.Gallery, systemState *system.SystemState) {
+	if !refreshing.CompareAndSwap(false, true) {
+		return
+	}
+	go func() {
+		defer refreshing.Store(false)
+		models, err := AvailableGalleryModels(galleries, systemState)
+		if err != nil {
+			xlog.Error("background gallery refresh failed", "error", err)
+			return
+		}
+		availableModelsMu.Lock()
+		availableModelsCache = models
+		availableModelsMu.Unlock()
+	}()
+}
+
 // List available backends
 func AvailableBackends(galleries []config.Gallery, systemState *system.SystemState) (GalleryElements[*GalleryBackend], error) {
 	return availableBackendsWithFilter(galleries, systemState, true)

core/gallery/gallery.go

@@ -52,3 +52,23 @@ func (m *GalleryModel) GetTags() []string {
 func (m *GalleryModel) GetDescription() string {
 	return m.Description
 }
+
+// GetKnownUsecases extracts known_usecases from the model's Overrides and
+// returns the parsed usecase flags. Returns nil when no usecases are declared.
+func (m *GalleryModel) GetKnownUsecases() *config.ModelConfigUsecase {
+	raw, ok := m.Overrides["known_usecases"]
+	if !ok {
+		return nil
+	}
+	list, ok := raw.([]any)
+	if !ok {
+		return nil
+	}
+	strs := make([]string, 0, len(list))
+	for _, v := range list {
+		if s, ok := v.(string); ok {
+			strs = append(strs, s)
+		}
+	}
+	return config.GetUsecasesFromYAML(strs)
+}