feat: improve search response cleansing with expanded allowlists

internal/output/cleanse.go

@@ -27,6 +27,16 @@ func CleanseSearchResponse(resp any) (any, error) {
 
 	if results, ok := result["results"].([]any); ok {
 		result["results"] = filterEmptyResults(results)
+		for _, r := range result["results"].([]any) {
+			if m, ok := r.(map[string]any); ok {
+				if snippets, ok := m["snippets"].([]any); ok {
+					m["snippets"] = filterEmptySnippets(snippets)
+					if len(m["snippets"].([]any)) == 0 {
+						delete(m, "snippets")
+					}
+				}
+			}
+		}
 	}
 
 	return result, nil
@@ -57,20 +67,26 @@ var documentAllowlist = allowlist{
 }
 
 var metadataAllowlist = allowlist{
-	"datasource": nil,
-	"objectType": nil,
-	"author":     personAllowlist,
-	"updateTime": nil,
-	"createTime": nil,
+	"datasource":   nil,
+	"objectType":   nil,
+	"author":       personAllowlist,
+	"owner":        personAllowlist,
+	"assignedTo":   personAllowlist,
+	"updatedBy":    personAllowlist,
+	"updateTime":   nil,
+	"createTime":   nil,
+	"status":       nil,
+	"priority":     nil,
+	"container":    nil,
+	"datasourceId": nil,
 }
 
 var personAllowlist = allowlist{
-	"name":  nil,
-	"email": nil,
+	"name": nil,
 }
 
 var snippetAllowlist = allowlist{
-	"snippet":  nil,
+	"text":     nil,
 	"mimeType": nil,
 }
 
@@ -171,6 +187,21 @@ func filterEmptyResults(results []any) []any {
 	return out
 }
 
+// filterEmptySnippets removes snippets where the text field is empty or missing.
+func filterEmptySnippets(snippets []any) []any {
+	out := make([]any, 0, len(snippets))
+	for _, s := range snippets {
+		m, ok := s.(map[string]any)
+		if !ok {
+			continue
+		}
+		if text, ok := m["text"].(string); ok && text != "" {
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
 // filterSlice applies the allowlist to each element in a slice.
 func filterSlice(s []any, al allowlist) []any {
 	out := make([]any, len(s))

internal/output/cleanse_snapshot_test.go

@@ -0,0 +1,135 @@
+package output
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCleanseSearchResponse_Snapshots(t *testing.T) {
+	fixtures := []struct {
+		name     string
+		rawFile  string
+		wantFile string
+	}{
+		{"people query", "raw_people.json", "cleansed_people.json"},
+		{"jira query", "raw_jira.json", "cleansed_jira.json"},
+		{"github query", "raw_github.json", "cleansed_github.json"},
+		{"mixed query", "raw_mixed.json", "cleansed_mixed.json"},
+	}
+
+	for _, tt := range fixtures {
+		t.Run(tt.name, func(t *testing.T) {
+			rawBytes, err := os.ReadFile(filepath.Join("testdata", tt.rawFile))
+			require.NoError(t, err)
+
+			wantBytes, err := os.ReadFile(filepath.Join("testdata", tt.wantFile))
+			require.NoError(t, err)
+
+			var rawInput map[string]any
+			require.NoError(t, json.Unmarshal(rawBytes, &rawInput))
+
+			got, err := CleanseSearchResponse(rawInput)
+			require.NoError(t, err)
+
+			gotBytes, err := json.MarshalIndent(got, "", "  ")
+			require.NoError(t, err)
+
+			var want any
+			require.NoError(t, json.Unmarshal(wantBytes, &want))
+			wantNorm, err := json.MarshalIndent(want, "", "  ")
+			require.NoError(t, err)
+
+			if !assert.JSONEq(t, string(wantNorm), string(gotBytes)) {
+				// On failure, write the actual output for easy diffing
+				actualPath := filepath.Join("testdata", "actual_"+tt.wantFile)
+				_ = os.WriteFile(actualPath, append(gotBytes, '\n'), 0600)
+				t.Logf("Actual output written to %s for diffing", actualPath)
+			}
+		})
+	}
+}
+
+func TestCleanseSearchResponse_SnapshotStructure(t *testing.T) {
+	allowedResponse := map[string]bool{"results": true, "cursor": true, "hasMoreResults": true, "requestID": true}
+	allowedResult := map[string]bool{"title": true, "url": true, "snippets": true, "document": true}
+	allowedDocument := map[string]bool{"title": true, "url": true, "datasource": true, "docType": true, "metadata": true}
+	allowedMetadata := map[string]bool{
+		"datasource": true, "objectType": true, "author": true, "owner": true,
+		"assignedTo": true, "updatedBy": true, "updateTime": true, "createTime": true,
+		"status": true, "priority": true, "container": true, "datasourceId": true,
+	}
+	allowedPerson := map[string]bool{"name": true}
+	allowedSnippet := map[string]bool{"text": true, "mimeType": true}
+
+	fixtures := []string{"raw_people.json", "raw_jira.json", "raw_github.json", "raw_mixed.json"}
+
+	for _, fixture := range fixtures {
+		t.Run(fixture, func(t *testing.T) {
+			rawBytes, err := os.ReadFile(filepath.Join("testdata", fixture))
+			require.NoError(t, err)
+
+			var rawInput map[string]any
+			require.NoError(t, json.Unmarshal(rawBytes, &rawInput))
+
+			got, err := CleanseSearchResponse(rawInput)
+			require.NoError(t, err)
+
+			m := got.(map[string]any)
+
+			// Response level
+			for k := range m {
+				assert.True(t, allowedResponse[k], "disallowed response key: %s", k)
+			}
+
+			results, _ := m["results"].([]any)
+			for i, r := range results {
+				rm := r.(map[string]any)
+
+				// Result level
+				for k := range rm {
+					assert.True(t, allowedResult[k], "result[%d] disallowed key: %s", i, k)
+				}
+
+				// Document level
+				if doc, ok := rm["document"].(map[string]any); ok {
+					for k := range doc {
+						assert.True(t, allowedDocument[k], "result[%d].document disallowed key: %s", i, k)
+					}
+
+					// Metadata level
+					if meta, ok := doc["metadata"].(map[string]any); ok {
+						for k := range meta {
+							assert.True(t, allowedMetadata[k], "result[%d].metadata disallowed key: %s", i, k)
+						}
+
+						// Person fields
+						for _, pf := range []string{"author", "owner", "assignedTo", "updatedBy"} {
+							if p, ok := meta[pf].(map[string]any); ok {
+								for k := range p {
+									assert.True(t, allowedPerson[k], "result[%d].metadata.%s disallowed key: %s", i, pf, k)
+								}
+							}
+						}
+					}
+				}
+
+				// Snippet level
+				if snippets, ok := rm["snippets"].([]any); ok {
+					for j, s := range snippets {
+						sm := s.(map[string]any)
+						for k := range sm {
+							assert.True(t, allowedSnippet[k], "result[%d].snippets[%d] disallowed key: %s", i, j, k)
+						}
+						text, _ := sm["text"].(string)
+						assert.NotEmpty(t, text, "result[%d].snippets[%d] has empty text", i, j)
+					}
+				}
+			}
+		})
+	}
+}