From e15704143f4321a4f60572bd0275b56ea1c01965 Mon Sep 17 00:00:00 2001 From: jharris1679 Date: Mon, 8 Dec 2025 15:23:59 -0500 Subject: [PATCH] feat: add agent behavior metrics (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement computed metrics that analyze HOW an agent works: - totalTokens: tokens consumed for the case - toolCount: number of tool calls - costUsd: total cost - explorationRatio: research vs action balance - cacheHitRatio: context reuse efficiency - tokensPerTool: tokens per tool call - tokensPerRead: tokens per file read - Raw cache token breakdown (input/read/write) Uses case as the unit (not SDK-specific "turns"). Metrics are displayed after interview runs and stored in baselines. Also: - Default model set to claude-haiku-4-5-20251001 - Show model name in completion output - Fix TUI interleaving when text streams between tool calls 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/agents/claude-code.ts | 2 +- src/agents/types.ts | 31 +++++++++++++ src/cli/commands/interview.ts | 57 +++++++++++++++++------- src/metrics/behavior.ts | 83 +++++++++++++++++++++++++++++++++++ src/metrics/index.ts | 7 +++ 5 files changed, 163 insertions(+), 17 deletions(-) create mode 100644 src/metrics/behavior.ts create mode 100644 src/metrics/index.ts diff --git a/src/agents/claude-code.ts b/src/agents/claude-code.ts index c0f29a6..8107906 100644 --- a/src/agents/claude-code.ts +++ b/src/agents/claude-code.ts @@ -123,7 +123,7 @@ export class ClaudeCodeAgent implements AgentWrapper { disallowedTools: options.disallowedTools, maxBudgetUsd: options.maxBudgetUsd, maxTurns: options.maxTurns, - model: options.model, + model: options.model || 'claude-haiku-4-5-20251001', includePartialMessages: options.includePartialMessages ?? true, env: options.env, // Don't load user/project settings - isolation mode diff --git a/src/agents/types.ts b/src/agents/types.ts index 70506e6..5f0e251 100644 --- a/src/agents/types.ts +++ b/src/agents/types.ts @@ -198,6 +198,37 @@ export interface AgentRegistry { findAvailable(): Promise; } +/** + * Computed behavior metrics from agent execution + * These analyze HOW the agent works, not just what it produces + */ +export interface BehaviorMetrics { + /** Total tokens consumed */ + totalTokens: number; + /** Number of tool calls */ + toolCount: number; + /** Total cost in USD */ + costUsd: number; + /** (Read+Glob+Grep calls) / total tools - research vs action */ + explorationRatio: number; + /** Cache read tokens / total input tokens - context reuse */ + cacheHitRatio: number; + /** Average tool execution time in ms (0 if timing unavailable) */ + avgToolDurationMs: number; + /** Tokens per tool call */ + tokensPerTool: number; + /** Tokens per Read tool call (file read efficiency) */ + tokensPerRead: number; + /** Number of Read tool calls */ + readCount: number; + /** Raw input tokens (non-cached) */ + inputTokens: number; + /** Raw cache read tokens */ + cacheReadTokens: number; + /** Raw cache write tokens */ + cacheWriteTokens: number; +} + /** * Create empty token usage object */ diff --git a/src/cli/commands/interview.ts b/src/cli/commands/interview.ts index 38bbda8..ec59bce 100644 --- a/src/cli/commands/interview.ts +++ b/src/cli/commands/interview.ts @@ -16,6 +16,7 @@ import { box } from '../../utils/ui'; import { loadCases, getDefaultCasesDir } from '../../cases'; import { Case } from '../../cases/types'; import { getAgent, AgentWrapper, AgentResult, AgentEvent } from '../../agents'; +import { computeBehaviorMetrics, formatBehaviorMetrics } from '../../metrics'; /** * Exploration status messages - cycles through these while agent works @@ -111,6 +112,20 @@ interface Baseline { gradedAt: string; gradedBy: string; notes?: string; + behaviorMetrics?: { + totalTokens: number; + toolCount: number; + costUsd: number; + explorationRatio: number; + cacheHitRatio: number; + avgToolDurationMs: number; + tokensPerTool: number; + tokensPerRead: number; + readCount: number; + inputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + }; } interface BaselineStore { @@ -341,12 +356,12 @@ async function runInterviewQuestion( detail = `${subagentType} ${desc}`.trim(); exploration.toolCalls.push(`› ${event.tool.name}`); - exploration.spinner.stop(); - console.log(chalk.yellow(` ⚡ Task ${chalk.bold(subagentType)} ${chalk.dim(desc)}`)); + if (!textOutputStarted) exploration.spinner.stop(); + console.log(chalk.yellow(`\n ⚡ Task ${chalk.bold(subagentType)} ${chalk.dim(desc)}`)); if (prompt) { console.log(chalk.dim(` "${prompt}${String(input.prompt).length > 60 ? '...' : ''}"`)); } - exploration.spinner.start(); + if (!textOutputStarted) exploration.spinner.start(); } else { // Extract most useful input field for display if (input.file_path) detail = String(input.file_path).split('/').slice(-2).join('/'); @@ -358,15 +373,17 @@ async function runInterviewQuestion( const toolInfo = detail ? `${event.tool.name} ${chalk.dim(detail)}` : event.tool.name; exploration.toolCalls.push(`› ${event.tool.name}`); - // Stop spinner and show tool call - exploration.spinner.stop(); - console.log(chalk.cyan(` › ${toolInfo}`)); - exploration.spinner.start(); + // Stop spinner, show tool call, restart only if text hasn't started + if (!textOutputStarted) exploration.spinner.stop(); + console.log(chalk.cyan(`${textOutputStarted ? '\n' : ''} › ${toolInfo}`)); + if (!textOutputStarted) exploration.spinner.start(); } - const state = EXPLORATION_STATES[exploration.toolCalls.length % EXPLORATION_STATES.length]; - const baseText = `${chalk.bold.hex('#D97706')(agent.displayName)} ${state.color(state.text)}`; - exploration.spinner.text = `${baseText} ${chalk.dim(`(${exploration.toolCalls.length} tools)`)}`; + if (!textOutputStarted) { + const state = EXPLORATION_STATES[exploration.toolCalls.length % EXPLORATION_STATES.length]; + const baseText = `${chalk.bold.hex('#D97706')(agent.displayName)} ${state.color(state.text)}`; + exploration.spinner.text = `${baseText} ${chalk.dim(`(${exploration.toolCalls.length} tools)`)}`; + } break; } @@ -379,13 +396,13 @@ async function runInterviewQuestion( // Show thinking/reasoning output between tool calls const text = event.text.trim(); if (text) { - exploration.spinner.stop(); + if (!textOutputStarted) exploration.spinner.stop(); // Show first line or first 150 chars of thinking const firstLine = text.split('\n')[0]; const display = firstLine.length > 150 ? firstLine.substring(0, 150) + '...' : firstLine; - console.log(chalk.magenta(` 💭 ${display}`)); + console.log(chalk.magenta(`${textOutputStarted ? '\n' : ''} 💭 ${display}`)); // If there's more content, indicate it if (text.includes('\n') || text.length > 150) { const lineCount = text.split('\n').length; @@ -393,7 +410,7 @@ async function runInterviewQuestion( console.log(chalk.dim(` (${lineCount} lines of reasoning)`)); } } - exploration.spinner.start(); + if (!textOutputStarted) exploration.spinner.start(); } break; } @@ -412,8 +429,10 @@ async function runInterviewQuestion( } case 'status': { - // Show status updates in spinner - exploration.spinner.text = `${chalk.bold.hex('#D97706')(agent.displayName)} ${chalk.cyan(event.message)}`; + // Only update spinner if text hasn't started + if (!textOutputStarted) { + exploration.spinner.text = `${chalk.bold.hex('#D97706')(agent.displayName)} ${chalk.cyan(event.message)}`; + } break; } @@ -446,13 +465,18 @@ async function runInterviewQuestion( return { grade: 0, skipped: true, durationMs: result.durationMs, rl }; } - console.log(chalk.green(`\n ✓ ${agent.displayName} completed in ${durationSec}s`)); + console.log(chalk.green(`\n ✓ ${agent.displayName} completed in ${durationSec}s`) + chalk.dim(` (${result.model})`)); // Show tools used if available if (result.toolsUsed && result.toolsUsed.length > 0) { console.log(chalk.dim(`\n Tools used: ${result.toolsUsed.join(', ')}`)); } + // Show behavior metrics + const behaviorMetrics = computeBehaviorMetrics(result); + console.log(chalk.bold('\n Behavior Metrics:')); + console.log(formatBehaviorMetrics(behaviorMetrics)); + // Recreate readline after agent run - stdin may have been disrupted // by the spawned claude process if (!isReadlineOpen(rl)) { @@ -480,6 +504,7 @@ async function runInterviewQuestion( gradedAt: new Date().toISOString(), gradedBy: 'human', notes: notes || undefined, + behaviorMetrics, }; saveBaselines(projectRoot, store); diff --git a/src/metrics/behavior.ts b/src/metrics/behavior.ts new file mode 100644 index 0000000..0b82443 --- /dev/null +++ b/src/metrics/behavior.ts @@ -0,0 +1,83 @@ +/** + * Behavior metrics computation + * + * Analyzes HOW an agent works based on raw execution data. + * These metrics help understand agent efficiency and patterns. + */ + +import { AgentResult, BehaviorMetrics } from '../agents/types.js'; + +/** Tools considered "exploration" (read-only research) */ +const EXPLORATION_TOOLS = ['Read', 'Glob', 'Grep', 'WebFetch', 'WebSearch']; + +/** + * Compute behavior metrics from an agent result + */ +export function computeBehaviorMetrics(result: AgentResult): BehaviorMetrics { + const { tokens, costUsd, toolCalls } = result; + + const safeToolCount = Math.max(toolCalls.length, 1); + + // Exploration ratio: what fraction of tool calls are read-only research + const explorationCalls = toolCalls.filter((t) => + EXPLORATION_TOOLS.includes(t.name) + ).length; + const explorationRatio = explorationCalls / safeToolCount; + + // Cache hit ratio: fraction of input tokens that came from cache + // In Claude API: input_tokens = new tokens, cache_read_input_tokens = cached tokens + // Total input = input_tokens + cache_read_input_tokens + const totalInputTokens = tokens.inputTokens + tokens.cacheReadTokens; + const cacheHitRatio = totalInputTokens > 0 + ? tokens.cacheReadTokens / totalInputTokens + : 0; + + + // Average tool duration (may be 0 if SDK doesn't provide timing) + const totalToolDuration = toolCalls.reduce( + (sum, t) => sum + (t.durationMs || 0), + 0 + ); + const avgToolDurationMs = totalToolDuration / safeToolCount; + + // Tokens per tool call + const tokensPerTool = tokens.totalTokens / safeToolCount; + + // Count Read tool calls and compute tokens per read + const readCount = toolCalls.filter((t) => t.name === 'Read').length; + const tokensPerRead = readCount > 0 ? tokens.totalTokens / readCount : 0; + + return { + totalTokens: tokens.totalTokens, + toolCount: toolCalls.length, + costUsd: Math.round(costUsd * 10000) / 10000, // 4 decimals + explorationRatio: Math.round(explorationRatio * 100) / 100, // 2 decimals + cacheHitRatio: Math.round(cacheHitRatio * 100) / 100, + avgToolDurationMs: Math.round(avgToolDurationMs), + tokensPerTool: Math.round(tokensPerTool), + tokensPerRead: Math.round(tokensPerRead), + readCount, + inputTokens: tokens.inputTokens, + cacheReadTokens: tokens.cacheReadTokens, + cacheWriteTokens: tokens.cacheWriteTokens, + }; +} + +/** + * Format behavior metrics for display + */ +export function formatBehaviorMetrics(metrics: BehaviorMetrics): string { + const lines = [ + ` Tokens: ${metrics.totalTokens.toLocaleString()} ` + + `Tools: ${metrics.toolCount} ` + + `Cost: $${metrics.costUsd.toFixed(4)}`, + ` Tokens/tool: ${metrics.tokensPerTool.toLocaleString()} ` + + `Tokens/read: ${metrics.tokensPerRead.toLocaleString()} (${metrics.readCount} reads)`, + ` Exploration Ratio: ${Math.round(metrics.explorationRatio * 100)}% ` + + `Cache hits: ${Math.round(metrics.cacheHitRatio * 100)}%`, + ` Cache: input=${metrics.inputTokens.toLocaleString()} ` + + `read=${metrics.cacheReadTokens.toLocaleString()} ` + + `write=${metrics.cacheWriteTokens.toLocaleString()}`, + ]; + return lines.join('\n'); +} diff --git a/src/metrics/index.ts b/src/metrics/index.ts new file mode 100644 index 0000000..7fd1287 --- /dev/null +++ b/src/metrics/index.ts @@ -0,0 +1,7 @@ +/** + * Metrics module + * + * Provides computed metrics for analyzing agent behavior and performance. + */ + +export * from './behavior.js';