Skip to content

feat: implement context-window pruning middleware for semantic workspaces #295

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
muntasir-396 wants to merge 3 commits into
base: main
Choose a base branch
from
+336 −10
Open

feat: implement context-window pruning middleware for semantic workspaces #295

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e3fa07f
add context-window pruning middleware
muntasir-396 Jul 16, 2026
13315ca
fix: address CodeRabbit review comments
muntasir-396 Jul 16, 2026
a8feb91
Add unit tests for MeetingFormatter service
muntasir-396 Jul 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 19 additions & 10 deletions lib/services/ai_service.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import 'package:http/http.dart' as http;
import 'package:intl/intl.dart';
import 'package:ell_ena/services/supabase_service.dart';
import 'package:ell_ena/services/meeting_formatter.dart';
import 'package:ell_ena/services/context_pruning_service.dart';

class AIService {
static final AIService _instance = AIService._internal();
Expand Down Expand Up @@ -67,17 +68,25 @@ class AIService {
bool isMeetingQuery = _isMeetingRelatedQuery(userMessage);
String meetingContext = "";

// If it's a meeting query, retrieve relevant meeting summaries
if (isMeetingQuery) {
final meetingSummaries = await getRelevantMeetingSummaries(userMessage);

if (meetingSummaries.isNotEmpty) {
meetingContext = "\nRelevant meeting information:\n\n";
meetingContext += MeetingFormatter.formatMeetingSummaries(meetingSummaries);
}
}

try {
// If it's a meeting query, retrieve relevant meeting summaries
if (isMeetingQuery) {
final meetingSummaries = await getRelevantMeetingSummaries(userMessage);

// Apply Context-Window Pruning and Token Truncation Middleware
final contextPruner = ContextPruningService();
final optimizedMeetingSummaries = contextPruner.prune(
query: userMessage,
meetings: meetingSummaries,
maxTokens: 1500 // Strict budget for meeting context
);

if (optimizedMeetingSummaries.isNotEmpty) {
meetingContext = "\nRelevant meeting information:\n\n";
meetingContext += MeetingFormatter.formatMeetingSummaries(optimizedMeetingSummaries);
}
}

// Define function declarations for the model
final List<Map<String, dynamic>> functionDeclarations = [
{
Expand Down
124 changes: 124 additions & 0 deletions lib/services/context_pruning_service.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
import 'dart:core';
import 'package:flutter/foundation.dart';
import 'package:ell_ena/services/meeting_formatter.dart';

class ContextPruningService {
/// Estimates the number of tokens based on a character heuristic (approx. 4 chars per token).
int _estimateTokens(String text) {
return (text.length / 4).ceil();
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

/// Calculates a lightweight keyword overlap score as a fallback
/// if the vector search similarity score is unavailable.
double _calculateKeywordOverlap(String query, String text) {
final punctuationRegExp = RegExp(r'[^\w\s]+');
final queryWords = query.toLowerCase().replaceAll(punctuationRegExp, '').split(RegExp(r'\s+')).where((w) => w.isNotEmpty).toSet();
final textWords = text.toLowerCase().replaceAll(punctuationRegExp, '').split(RegExp(r'\s+')).where((w) => w.isNotEmpty).toSet();

if (queryWords.isEmpty || textWords.isEmpty) return 0.0;

final intersection = queryWords.intersection(textWords);
return intersection.length / queryWords.length;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}

/// Prunes the list of meeting contexts based on relevance and a token budget.
List<Map<String, dynamic>> prune({
required String query,
required List<Map<String, dynamic>> meetings,
required int maxTokens,
}) {
final stopwatch = Stopwatch()..start();

// Create deep copies to avoid modifying cached references
List<Map<String, dynamic>> chunks = meetings.map((m) => Map<String, dynamic>.from(m)).toList();

int originalTokens = 0;
int chunksRemoved = 0;

// 1. Scoring & Initial Token Counting
for (var chunk in chunks) {
final summaryStr = chunk['summary']?.toString() ?? '';

// Calculate token budget based on fully rendered meeting context
final renderedChunk = MeetingFormatter.formatMeetingSummaries([chunk]);
final tokenCount = _estimateTokens(renderedChunk);

originalTokens += tokenCount;
chunk['_token_count'] = tokenCount;

// Use existing similarity if provided by vector search, else fallback to keyword overlap
if (chunk.containsKey('similarity') && chunk['similarity'] != null) {
chunk['_relevance_score'] = (chunk['similarity'] as num).toDouble();
} else {
chunk['_relevance_score'] = _calculateKeywordOverlap(query, summaryStr);
}
}

// 2. Rank contexts from highest relevance to lowest
chunks.sort((a, b) {
final scoreA = a['_relevance_score'] as double;
final scoreB = b['_relevance_score'] as double;
return scoreB.compareTo(scoreA); // Descending order
});

// 3. Sliding-Window Pruning (Remove lowest-ranked chunks until under budget)
List<Map<String, dynamic>> optimizedChunks = [];
// Reserve overhead for prepended content: "\nRelevant meeting information:\n\n"
int currentTokens = _estimateTokens("\nRelevant meeting information:\n\n");

for (var chunk in chunks) {
final tokenCount = chunk['_token_count'] as int;

if (currentTokens + tokenCount <= maxTokens) {
optimizedChunks.add(chunk);
currentTokens += tokenCount;
} else {
// Enforce strict truncation for the last partially fitting chunk
int remainingTokens = maxTokens - currentTokens;

if (remainingTokens > 100 && chunk['summary'] != null) {
var partialSummary = Map<String, dynamic>.from(chunk['summary']);

if (partialSummary['overall_summary'] != null) {
String overall = partialSummary['overall_summary'].toString();
int allowedChars = remainingTokens * 4;

if (overall.length > allowedChars) {
partialSummary['overall_summary'] =
overall.substring(0, allowedChars) + '\n... [TRUNCATED DUE TO TOKEN BUDGET]';
}
}
chunk['summary'] = partialSummary;
optimizedChunks.add(chunk);
currentTokens += remainingTokens;
}
}
}

// Accumulate skipped chunks in the count
chunksRemoved = chunks.length - optimizedChunks.length;

stopwatch.stop();

// 4. Debug Logging & Reduction Metrics
final double reductionPct = originalTokens > 0
? ((originalTokens - currentTokens) / originalTokens) * 100
: 0.0;

debugPrint('--- Context-Window Pruning Metrics ---');
debugPrint('Original Tokens: $originalTokens');
debugPrint('Final Tokens: $currentTokens');
debugPrint('Reduction: ${reductionPct.toStringAsFixed(2)}%');
debugPrint('Chunks Removed: $chunksRemoved');
debugPrint('Latency: ${stopwatch.elapsedMilliseconds} ms');
debugPrint('--------------------------------------');

// Clean up internal metric keys before returning
for (var chunk in optimizedChunks) {
chunk.remove('_token_count');
chunk.remove('_relevance_score');
}

return optimizedChunks;
}
}
193 changes: 193 additions & 0 deletions test/services/meeting_formatter_test.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:ell_ena/services/meeting_formatter.dart';

void main() {
group('MeetingFormatter', () {
group('formatMeetingSummary', () {
test('formats complete summary correctly (Normal/valid input)', () {
final Map<String, dynamic> summary = <String, dynamic>{
'key_discussion_points': <String>['Point 1', 'Point 2'],
'important_decisions': <String>['Decision 1'],
'action_items': <Map<String, dynamic>>[
<String, dynamic>{'item': 'Task 1', 'owner': 'Alice', 'deadline': 'Tomorrow'},
<String, dynamic>{'item': 'Task 2'} // Missing owner and deadline to test fallbacks
],
'follow_up_tasks': <String>['Follow up 1'],
'overall_summary': 'Great meeting'
};

final String result = MeetingFormatter.formatMeetingSummary(
title: 'Team Sync',
date: '2026-07-25 at 10:00',
summary: summary,
);

final String expected = '📅 *Team Sync*\n'
'🕒 2026-07-25 at 10:00\n'
'\n'
'Key Points:\n'
'-> Point 1\n'
'-> Point 2\n'
'\n'
'Decisions:\n'
'-> Decision 1\n'
'\n'
'Action Items:\n'
'-> Task 1 (Owner: Alice, Deadline: Tomorrow)\n'
'-> Task 2 (Owner: Unassigned, Deadline: No deadline)\n'
'\n'
'Follow-Up Tasks:\n'
'-> Follow up 1\n'
'\n'
'Summary:\n'
'Great meeting\n';

expect(result, expected);
});

test('formats partial input correctly (Partial input)', () {
final Map<String, dynamic> summary = <String, dynamic>{
'key_discussion_points': <String>['Point 1'],
};

final String result = MeetingFormatter.formatMeetingSummary(
title: 'Partial Sync',
date: '2026-07-25 at 10:00',
summary: summary,
);

final String expected = '📅 *Partial Sync*\n'
'🕒 2026-07-25 at 10:00\n'
'\n'
'Key Points:\n'
'-> Point 1\n'
'\n';

expect(result, expected);
});

test('formats empty summary map correctly (Empty input)', () {
final String result = MeetingFormatter.formatMeetingSummary(
title: 'Empty Sync',
date: '2026-07-25 at 10:00',
summary: <String, dynamic>{},
);

final String expected = '📅 *Empty Sync*\n'
'🕒 2026-07-25 at 10:00\n'
'\n';

expect(result, expected);
});

test('formats null summary correctly (Null values)', () {
final String result = MeetingFormatter.formatMeetingSummary(
title: 'Null Sync',
date: '2026-07-25 at 10:00',
summary: null,
);

final String expected = '📅 *Null Sync*\n'
'🕒 2026-07-25 at 10:00\n'
'\n';

expect(result, expected);
});

test('formats empty lists correctly (Empty lists/maps)', () {
final Map<String, dynamic> summary = <String, dynamic>{
'key_discussion_points': <String>[],
'important_decisions': <String>[],
'action_items': <Map<String, dynamic>>[],
'follow_up_tasks': <String>[],
};

final String result = MeetingFormatter.formatMeetingSummary(
title: 'Empty Lists Sync',
date: '2026-07-25 at 10:00',
summary: summary,
);

// Based on current implementation, empty key points and decisions generate headers,
// but action items and follow up tasks are skipped due to .isNotEmpty check.
final String expected = '📅 *Empty Lists Sync*\n'
'🕒 2026-07-25 at 10:00\n'
'\n'
'Key Points:\n'
'\n'
'Decisions:\n'
'\n';

expect(result, expected);
});

test('throws Error when action items list contains null (Malformed input)', () {
final Map<String, dynamic> summary = <String, dynamic>{
'action_items': <dynamic>[null],
};

expect(
() => MeetingFormatter.formatMeetingSummary(
title: 'Malformed Sync',
date: '2026-07-25 at 10:00',
summary: summary,
),
throwsA(isA<Error>()), // Catches NoSuchMethodError or TypeError depending on Dart version
);
Comment on lines +124 to +136

@coderabbitai coderabbitai Bot Jul 25, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win

Do not codify a crash for malformed action items.

This accepts a runtime failure when externally supplied summary data contains a null/non-map item. MeetingFormatter indexes every item as a map, so one malformed entry aborts formatting the entire meeting context. Validate or skip invalid entries in the formatter, then assert a safe fallback here instead.

As per path instructions, **/*.dart requires proper input validation and error handling.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@test/services/meeting_formatter_test.dart` around lines 124 - 136, Update
MeetingFormatter.formatMeetingSummary to validate action_items entries before
treating them as maps, skipping null or otherwise malformed items while
continuing to format valid entries. Replace the test’s throwsA expectation with
an assertion for the formatter’s safe fallback output, while preserving normal
behavior for valid action items.

Source: Path instructions

});
});

group('formatMeetingSummaries', () {
test('returns default message for empty list (Empty list)', () {
final String result = MeetingFormatter.formatMeetingSummaries(<Map<String, dynamic>>[]);
expect(result, 'No relevant meetings found.');
});

test('formats multiple meetings correctly (Multiple items, Date/String formatting)', () {
final List<Map<String, dynamic>> meetings = <Map<String, dynamic>>[
<String, dynamic>{
'title': 'Meeting 1',
'meeting_date': '2026-07-25 14:30:00', // Use local string without Z to be completely safe from timezone parsing bugs
'summary': <String, dynamic>{
'overall_summary': 'Summary 1',
},
},
<String, dynamic>{
// Missing title and date
'summary': null,
}
];

final String result = MeetingFormatter.formatMeetingSummaries(meetings);

final String expected = '📅 *Meeting 1*\n'
'🕒 2026-07-25 at 14:30\n'
'\n'
'Summary:\n'
'Summary 1\n'
'\n'
'----------------------------------------\n'
'\n'
'📅 *Untitled Meeting*\n'
'🕒 Unknown date\n'
'\n';

expect(result, expected);
});

test('throws FormatException for malformed date string (Edge cases)', () {
final List<Map<String, dynamic>> meetings = <Map<String, dynamic>>[
<String, dynamic>{
'title': 'Meeting 1',
'meeting_date': 'not-a-valid-date', // Will fail DateTime.parse()
}
];

expect(
() => MeetingFormatter.formatMeetingSummaries(meetings),
throwsFormatException,
);
});
});
});
}