feat: migrate session storage from JSON to append-only JSONL

internal/tui/session.go

@@ -2,13 +2,13 @@
 package tui
 
 import (
+	"bufio"
 	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
 
 	"github.com/gleanwork/glean-cli/internal/debug"
-	"github.com/gleanwork/glean-cli/internal/fileutil"
 )
 
 var sessionLog = debug.New("session:persist")
@@ -31,6 +31,7 @@ type Source struct {
 // Session holds conversation history and can be persisted to disk.
 type Session struct {
 	Turns []Turn `json:"turns"`
+	path  string // resolved path to the session file
 }
 
 // sessionsDir returns ~/.glean/sessions/.
@@ -43,55 +44,144 @@ func sessionsDir() (string, error) {
 }
 
 // LoadLatest loads the last saved session, or returns an empty session if none exists.
+// It reads JSONL format first, falling back to legacy JSON if no JSONL file exists.
 func LoadLatest() *Session {
 	dir, err := sessionsDir()
 	if err != nil {
 		sessionLog.Log("load: sessions dir error: %v", err)
 		return &Session{}
 	}
-	path := filepath.Join(dir, "latest.json")
-	data, err := os.ReadFile(path)
+
+	jsonlPath := filepath.Join(dir, "latest.jsonl")
+	if s, ok := loadJSONL(jsonlPath); ok {
+		return s
+	}
+
+	jsonPath := filepath.Join(dir, "latest.json")
+	if s, ok := migrateFromJSON(jsonPath, jsonlPath); ok {
+		return s
+	}
+
+	return &Session{}
+}
+
+func loadJSONL(path string) (*Session, bool) {
+	f, err := os.Open(path)
 	if err != nil {
-		sessionLog.Log("load: %v", err)
-		return &Session{}
+		return nil, false
 	}
-	var s Session
-	if err := json.Unmarshal(data, &s); err != nil {
-		sessionLog.Log("load: parse error: %v", err)
-		return &Session{}
+	defer f.Close()
+
+	var turns []Turn
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		var turn Turn
+		if err := json.Unmarshal(scanner.Bytes(), &turn); err != nil {
+			sessionLog.Log("load: skipping malformed line: %v", err)
+			continue
+		}
+		turns = append(turns, turn)
+	}
+	if err := scanner.Err(); err != nil {
+		sessionLog.Log("load: scanner error: %v", err)
 	}
-	sessionLog.Log("loaded %d turns from %s", len(s.Turns), path)
-	return &s
+	sessionLog.Log("loaded %d turns from %s", len(turns), path)
+	return &Session{Turns: turns, path: path}, true
 }
 
-// Save persists the session to ~/.glean/sessions/latest.json.
-func (s *Session) Save() error {
-	dir, err := sessionsDir()
+func migrateFromJSON(jsonPath, jsonlPath string) (*Session, bool) {
+	data, err := os.ReadFile(jsonPath)
 	if err != nil {
-		return fmt.Errorf("could not locate sessions dir: %w", err)
+		return nil, false
 	}
-	if err := os.MkdirAll(dir, 0700); err != nil {
-		return fmt.Errorf("could not create sessions dir: %w", err)
+	var legacy Session
+	if err := json.Unmarshal(data, &legacy); err != nil {
+		sessionLog.Log("load: legacy parse error: %v", err)
+		return nil, false
 	}
-	data, err := json.MarshalIndent(s, "", "  ")
+	sessionLog.Log("migrating %d turns from %s to %s", len(legacy.Turns), jsonPath, jsonlPath)
+
+	legacy.path = jsonlPath
+	for _, turn := range legacy.Turns {
+		if err := appendTurnToFile(jsonlPath, turn); err != nil {
+			sessionLog.Log("migration write error: %v", err)
+			return &legacy, true
+		}
+	}
+	os.Remove(jsonPath)
+	return &legacy, true
+}
+
+func appendTurnToFile(path string, turn Turn) error {
+	data, err := json.Marshal(turn)
 	if err != nil {
 		return err
 	}
-	return fileutil.WriteFileAtomic(filepath.Join(dir, "latest.json"), data, 0600)
+	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	_, err = f.Write(append(data, '\n'))
+	return err
+}
+
+// ensurePath resolves and caches the session file path.
+func (s *Session) ensurePath() (string, error) {
+	if s.path != "" {
+		return s.path, nil
+	}
+	dir, err := sessionsDir()
+	if err != nil {
+		return "", fmt.Errorf("could not locate sessions dir: %w", err)
+	}
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return "", fmt.Errorf("could not create sessions dir: %w", err)
+	}
+	s.path = filepath.Join(dir, "latest.jsonl")
+	return s.path, nil
 }
 
-// AddTurn appends a turn to the session and saves immediately.
+// AddTurn appends a turn to the session and persists it immediately.
 func (s *Session) AddTurn(role, content string, sources []Source) error {
 	return s.AppendTurn(Turn{Role: role, Content: content, Sources: sources})
 }
 
-// AppendTurn appends a complete Turn (including Elapsed and any other fields)
-// to the session and saves immediately.
+// AppendTurn appends a complete Turn to the session and persists it immediately.
 func (s *Session) AppendTurn(turn Turn) error {
 	s.Turns = append(s.Turns, turn)
-	if err := s.Save(); err != nil {
+	path, err := s.ensurePath()
+	if err != nil {
+		sessionLog.Log("save failed: %v", err)
+		return err
+	}
+	if err := appendTurnToFile(path, turn); err != nil {
 		sessionLog.Log("save failed: %v", err)
 		return err
 	}
 	return nil
 }
+
+// Save rewrites the entire session to disk. Used only for migration or
+// exceptional cases — normal operation uses AppendTurn for O(1) writes.
+func (s *Session) Save() error {
+	path, err := s.ensurePath()
+	if err != nil {
+		return err
+	}
+	f, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	for _, turn := range s.Turns {
+		data, err := json.Marshal(turn)
+		if err != nil {
+			return err
+		}
+		if _, err := f.Write(append(data, '\n')); err != nil {
+			return err
+		}
+	}
+	return nil
+}

internal/tui/session_test.go

@@ -0,0 +1,88 @@
+package tui
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAppendTurn_JSONL(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.jsonl")
+
+	s := &Session{path: path}
+	require.NoError(t, s.AppendTurn(Turn{Role: "user", Content: "hello"}))
+	require.NoError(t, s.AppendTurn(Turn{Role: "assistant", Content: "hi there"}))
+	require.NoError(t, s.AppendTurn(Turn{Role: "user", Content: "bye"}))
+
+	assert.Len(t, s.Turns, 3)
+
+	loaded, ok := loadJSONL(path)
+	require.True(t, ok)
+	assert.Len(t, loaded.Turns, 3)
+	assert.Equal(t, "hello", loaded.Turns[0].Content)
+	assert.Equal(t, "hi there", loaded.Turns[1].Content)
+	assert.Equal(t, "bye", loaded.Turns[2].Content)
+}
+
+func TestLoadJSONL_SkipsCorruptLines(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.jsonl")
+
+	lines := []string{
+		`{"role":"user","content":"first"}`,
+		`{not valid json`,
+		`{"role":"assistant","content":"second"}`,
+	}
+	require.NoError(t, os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0600))
+
+	s, ok := loadJSONL(path)
+	require.True(t, ok)
+	assert.Len(t, s.Turns, 2)
+	assert.Equal(t, "first", s.Turns[0].Content)
+	assert.Equal(t, "second", s.Turns[1].Content)
+}
+
+func TestMigrateFromJSON(t *testing.T) {
+	dir := t.TempDir()
+	jsonPath := filepath.Join(dir, "latest.json")
+	jsonlPath := filepath.Join(dir, "latest.jsonl")
+
+	legacy := Session{
+		Turns: []Turn{
+			{Role: "user", Content: "old question"},
+			{Role: "assistant", Content: "old answer"},
+		},
+	}
+	data, err := json.MarshalIndent(legacy, "", "  ")
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(jsonPath, data, 0600))
+
+	s, ok := migrateFromJSON(jsonPath, jsonlPath)
+	require.True(t, ok)
+	assert.Len(t, s.Turns, 2)
+
+	// JSONL file should exist
+	_, err = os.Stat(jsonlPath)
+	assert.NoError(t, err)
+
+	// Legacy JSON file should be removed
+	_, err = os.Stat(jsonPath)
+	assert.True(t, os.IsNotExist(err))
+
+	// Verify JSONL is readable
+	loaded, ok := loadJSONL(jsonlPath)
+	require.True(t, ok)
+	assert.Len(t, loaded.Turns, 2)
+	assert.Equal(t, "old question", loaded.Turns[0].Content)
+}
+
+func TestLoadJSONL_NonExistent(t *testing.T) {
+	_, ok := loadJSONL("/nonexistent/path.jsonl")
+	assert.False(t, ok)
+}