From 375665052545d4106132914e4cb780f09e502e00 Mon Sep 17 00:00:00 2001
From: jharris1679 <j_harris@live.ca>
Date: Fri, 5 Dec 2025 15:44:48 -0500
Subject: [PATCH 1/5] feat: Add agent wrapper infrastructure and Claude Code
 integration
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Create AgentWrapper interface for abstracting coding agents
- Implement ClaudeCodeAgent using `claude -p` for non-interactive execution
- Add agent registry for managing multiple agent types
- Integrate real Claude Code agent into interview command
- Replace placeholder responses with actual agent execution
- Add timeout handling, progress streaming, and error recovery
- Parse tool usage from agent output

ANS-441

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 src/agents/claude-code.ts     | 204 ++++++++++++++++++++++++++++++++++
 src/agents/index.ts           |   9 ++
 src/agents/registry.ts        |  86 ++++++++++++++
 src/agents/types.ts           | 102 +++++++++++++++++
 src/cli/commands/interview.ts | 102 ++++++++++++-----
 5 files changed, 478 insertions(+), 25 deletions(-)
 create mode 100644 src/agents/claude-code.ts
 create mode 100644 src/agents/index.ts
 create mode 100644 src/agents/registry.ts
 create mode 100644 src/agents/types.ts

diff --git a/src/agents/claude-code.ts b/src/agents/claude-code.ts
new file mode 100644
index 0000000..75d3169
--- /dev/null
+++ b/src/agents/claude-code.ts
@@ -0,0 +1,204 @@
+/**
+ * Claude Code agent wrapper
+ *
+ * Wraps the Claude Code CLI (`claude`) to run prompts programmatically.
+ * Uses the --print (-p) flag for non-interactive single-prompt execution.
+ */
+
+import { spawn } from 'child_process';
+import { AgentWrapper, AgentResult, AgentRunOptions } from './types';
+
+/**
+ * Claude Code agent wrapper
+ */
+export class ClaudeCodeAgent implements AgentWrapper {
+  name = 'claude-code';
+  displayName = 'Claude Code';
+
+  /** Path to claude CLI (defaults to 'claude' in PATH) */
+  private cliPath: string;
+
+  constructor(cliPath: string = 'claude') {
+    this.cliPath = cliPath;
+  }
+
+  /**
+   * Check if Claude Code CLI is available
+   */
+  async isAvailable(): Promise<boolean> {
+    try {
+      const version = await this.getVersion();
+      return version !== null;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Get Claude Code version
+   */
+  async getVersion(): Promise<string | null> {
+    return new Promise((resolve) => {
+      const proc = spawn(this.cliPath, ['--version'], {
+        timeout: 5000,
+      });
+
+      let stdout = '';
+      proc.stdout?.on('data', (data) => {
+        stdout += data.toString();
+      });
+
+      proc.on('close', (code) => {
+        if (code === 0 && stdout.trim()) {
+          resolve(stdout.trim());
+        } else {
+          resolve(null);
+        }
+      });
+
+      proc.on('error', () => {
+        resolve(null);
+      });
+    });
+  }
+
+  /**
+   * Run a prompt through Claude Code
+   *
+   * Uses `claude -p "prompt"` for non-interactive execution.
+   * The agent will explore the codebase and provide an answer.
+   */
+  async run(prompt: string, options: AgentRunOptions): Promise<AgentResult> {
+    const startTime = Date.now();
+    const timeoutMs = options.timeoutMs || 300000; // 5 min default
+
+    return new Promise((resolve) => {
+      // Build command args
+      // -p: print mode (non-interactive, single prompt)
+      // --output-format: get structured output if available
+      const args = ['-p', prompt];
+
+      const proc = spawn(this.cliPath, args, {
+        cwd: options.cwd,
+        env: {
+          ...process.env,
+          ...options.env,
+          // Ensure non-interactive
+          CI: 'true',
+        },
+        timeout: timeoutMs,
+      });
+
+      let stdout = '';
+      let stderr = '';
+      let timedOut = false;
+
+      proc.stdout?.on('data', (data) => {
+        const chunk = data.toString();
+        stdout += chunk;
+        options.onOutput?.(chunk);
+      });
+
+      proc.stderr?.on('data', (data) => {
+        stderr += data.toString();
+      });
+
+      // Handle timeout
+      const timer = setTimeout(() => {
+        timedOut = true;
+        proc.kill('SIGTERM');
+        // Give it a moment to clean up, then force kill
+        setTimeout(() => {
+          proc.kill('SIGKILL');
+        }, 5000);
+      }, timeoutMs);
+
+      proc.on('close', (code) => {
+        clearTimeout(timer);
+        const durationMs = Date.now() - startTime;
+
+        // Parse the output to extract the answer
+        // Claude Code's -p mode outputs the response directly
+        const answer = this.parseAnswer(stdout);
+
+        // Try to extract tool usage from output
+        const toolsUsed = this.parseToolsUsed(stdout);
+
+        resolve({
+          answer,
+          success: code === 0 && !timedOut,
+          error: timedOut ? 'Timed out' : (code !== 0 ? `Exit code: ${code}` : undefined),
+          timedOut,
+          durationMs,
+          toolsUsed,
+          stdout,
+          stderr,
+          exitCode: code,
+        });
+      });
+
+      proc.on('error', (err) => {
+        clearTimeout(timer);
+        const durationMs = Date.now() - startTime;
+
+        resolve({
+          answer: '',
+          success: false,
+          error: err.message,
+          timedOut: false,
+          durationMs,
+          stdout,
+          stderr,
+          exitCode: null,
+        });
+      });
+    });
+  }
+
+  /**
+   * Parse the answer from Claude Code output
+   *
+   * The -p flag outputs the response directly, but there may be
+   * some formatting or metadata to strip.
+   */
+  private parseAnswer(stdout: string): string {
+    // For now, return the full output
+    // TODO: Parse out any metadata/formatting if needed
+    return stdout.trim();
+  }
+
+  /**
+   * Parse tools used from output
+   *
+   * Claude Code shows tool usage in its output. Try to extract them.
+   */
+  private parseToolsUsed(stdout: string): string[] {
+    const tools: Set<string> = new Set();
+
+    // Look for common tool patterns in Claude Code output
+    const toolPatterns = [
+      /Read\s+\S+/g,        // Read file
+      /Edit\s+\S+/g,        // Edit file
+      /Write\s+\S+/g,       // Write file
+      /Bash\s*\([^)]+\)/g,  // Bash command
+      /Grep\s+\S+/g,        // Grep search
+      /Glob\s+\S+/g,        // Glob search
+    ];
+
+    for (const pattern of toolPatterns) {
+      const matches = stdout.match(pattern);
+      if (matches) {
+        matches.forEach((m) => tools.add(m.split(/\s+/)[0]));
+      }
+    }
+
+    return Array.from(tools);
+  }
+}
+
+/**
+ * Create a Claude Code agent instance
+ */
+export function createClaudeCodeAgent(cliPath?: string): ClaudeCodeAgent {
+  return new ClaudeCodeAgent(cliPath);
+}
diff --git a/src/agents/index.ts b/src/agents/index.ts
new file mode 100644
index 0000000..53e0262
--- /dev/null
+++ b/src/agents/index.ts
@@ -0,0 +1,9 @@
+/**
+ * Agent wrappers for sniffbench
+ *
+ * Provides interfaces to various coding agents (Claude Code, Cursor, Aider, etc.)
+ */
+
+export * from './types';
+export * from './claude-code';
+export * from './registry';
diff --git a/src/agents/registry.ts b/src/agents/registry.ts
new file mode 100644
index 0000000..273aa38
--- /dev/null
+++ b/src/agents/registry.ts
@@ -0,0 +1,86 @@
+/**
+ * Agent registry
+ *
+ * Manages available agent wrappers and provides discovery.
+ */
+
+import { AgentWrapper, AgentRegistry } from './types';
+import { createClaudeCodeAgent } from './claude-code';
+
+/**
+ * Default agent registry implementation
+ */
+class DefaultAgentRegistry implements AgentRegistry {
+  private agents: Map<string, AgentWrapper> = new Map();
+
+  constructor() {
+    // Register built-in agents
+    this.register(createClaudeCodeAgent());
+  }
+
+  get(name: string): AgentWrapper | undefined {
+    return this.agents.get(name);
+  }
+
+  list(): AgentWrapper[] {
+    return Array.from(this.agents.values());
+  }
+
+  register(agent: AgentWrapper): void {
+    this.agents.set(agent.name, agent);
+  }
+
+  async findAvailable(): Promise<AgentWrapper[]> {
+    const available: AgentWrapper[] = [];
+
+    for (const agent of this.agents.values()) {
+      if (await agent.isAvailable()) {
+        available.push(agent);
+      }
+    }
+
+    return available;
+  }
+}
+
+// Singleton instance
+let registryInstance: AgentRegistry | null = null;
+
+/**
+ * Get the global agent registry
+ */
+export function getAgentRegistry(): AgentRegistry {
+  if (!registryInstance) {
+    registryInstance = new DefaultAgentRegistry();
+  }
+  return registryInstance;
+}
+
+/**
+ * Get an agent by name, throwing if not found
+ */
+export function getAgent(name: string): AgentWrapper {
+  const registry = getAgentRegistry();
+  const agent = registry.get(name);
+
+  if (!agent) {
+    const available = registry.list().map((a) => a.name).join(', ');
+    throw new Error(`Unknown agent: ${name}. Available: ${available}`);
+  }
+
+  return agent;
+}
+
+/**
+ * Check if a specific agent is available
+ */
+export async function isAgentAvailable(name: string): Promise<boolean> {
+  const registry = getAgentRegistry();
+  const agent = registry.get(name);
+
+  if (!agent) {
+    return false;
+  }
+
+  return agent.isAvailable();
+}
diff --git a/src/agents/types.ts b/src/agents/types.ts
new file mode 100644
index 0000000..d326684
--- /dev/null
+++ b/src/agents/types.ts
@@ -0,0 +1,102 @@
+/**
+ * Agent wrapper types
+ *
+ * Agents are coding assistants that can be evaluated by sniffbench.
+ * Each agent wrapper provides a common interface for running prompts
+ * and capturing results.
+ */
+
+/**
+ * Options for running an agent
+ */
+export interface AgentRunOptions {
+  /** Working directory for the agent */
+  cwd: string;
+
+  /** Timeout in milliseconds */
+  timeoutMs?: number;
+
+  /** Environment variables to pass */
+  env?: Record<string, string>;
+
+  /** Callback for streaming output */
+  onOutput?: (chunk: string) => void;
+}
+
+/**
+ * Result from running an agent
+ */
+export interface AgentResult {
+  /** The agent's final answer/output */
+  answer: string;
+
+  /** Whether the run completed successfully */
+  success: boolean;
+
+  /** Error message if failed */
+  error?: string;
+
+  /** Whether the run timed out */
+  timedOut: boolean;
+
+  /** Duration in milliseconds */
+  durationMs: number;
+
+  /** Tools/commands the agent used (if trackable) */
+  toolsUsed?: string[];
+
+  /** Tokens used (if trackable) */
+  tokensUsed?: number;
+
+  /** Raw stdout */
+  stdout: string;
+
+  /** Raw stderr */
+  stderr: string;
+
+  /** Exit code */
+  exitCode: number | null;
+}
+
+/**
+ * Agent wrapper interface
+ */
+export interface AgentWrapper {
+  /** Agent identifier */
+  name: string;
+
+  /** Human-readable display name */
+  displayName: string;
+
+  /** Check if this agent is available on the system */
+  isAvailable(): Promise<boolean>;
+
+  /** Get version information */
+  getVersion(): Promise<string | null>;
+
+  /**
+   * Run a prompt through the agent
+   *
+   * @param prompt - The prompt/question to send to the agent
+   * @param options - Run options (cwd, timeout, etc.)
+   * @returns The agent's result
+   */
+  run(prompt: string, options: AgentRunOptions): Promise<AgentResult>;
+}
+
+/**
+ * Registry of available agents
+ */
+export interface AgentRegistry {
+  /** Get an agent by name */
+  get(name: string): AgentWrapper | undefined;
+
+  /** List all registered agents */
+  list(): AgentWrapper[];
+
+  /** Register a new agent */
+  register(agent: AgentWrapper): void;
+
+  /** Find available agents on the system */
+  findAvailable(): Promise<AgentWrapper[]>;
+}
diff --git a/src/cli/commands/interview.ts b/src/cli/commands/interview.ts
index 6c4552b..ccbc49a 100644
--- a/src/cli/commands/interview.ts
+++ b/src/cli/commands/interview.ts
@@ -15,6 +15,7 @@ import * as readline from 'readline';
 import { box } from '../../utils/ui';
 import { loadCases, getDefaultCasesDir } from '../../cases';
 import { Case } from '../../cases/types';
+import { getAgent, AgentWrapper, AgentResult } from '../../agents';
 
 interface InterviewOptions {
   cases?: string;
@@ -143,28 +144,31 @@ function formatAnswer(answer: string, maxLines: number = 30): string {
 }
 
 /**
- * Simulate agent response (placeholder - will integrate with real agent)
+ * Run agent on a comprehension question
  */
-async function getAgentResponse(caseData: Case, _agent: string): Promise<string> {
-  // TODO: Integrate with actual agent wrapper
-  // For now, return a placeholder that indicates the agent would explore
+async function getAgentResponse(
+  caseData: Case,
+  agent: AgentWrapper,
+  cwd: string,
+  onOutput?: (chunk: string) => void
+): Promise<AgentResult> {
+  // Build the prompt for the agent
+  // We frame it as a comprehension question about the codebase
+  const prompt = `You are being evaluated on your understanding of this codebase.
 
-  return `[Agent would explore the codebase and answer:]
+Please answer the following question by exploring the codebase:
 
 ${caseData.prompt}
 
----
-This is a placeholder response. In the full implementation, the agent
-(${_agent}) would:
+Take your time to explore and provide a thorough, accurate answer with specific file references where relevant.`;
 
-1. Analyze the codebase using allowed tools (read, grep, glob, search)
-2. Build understanding of the relevant areas
-3. Provide a detailed answer based on what it finds
+  const result = await agent.run(prompt, {
+    cwd,
+    timeoutMs: (caseData.expectations?.maxTimeSeconds || 300) * 1000,
+    onOutput,
+  });
 
-To implement:
-- Connect to Claude Code SDK or other agent wrappers
-- Capture the agent's exploration and final answer
-- Track tool usage for efficiency metrics`;
+  return result;
 }
 
 /**
@@ -172,11 +176,11 @@ To implement:
  */
 async function runInterviewQuestion(
   caseData: Case,
-  agent: string,
+  agent: AgentWrapper,
   rl: readline.Interface,
   store: BaselineStore,
   projectRoot: string
-): Promise<{ grade: number; skipped: boolean }> {
+): Promise<{ grade: number; skipped: boolean; durationMs?: number }> {
   const existingBaseline = store.baselines[caseData.id];
 
   // Show the question
@@ -192,18 +196,43 @@ async function runInterviewQuestion(
   }
 
   // Get agent's response
-  const spinner = ora('Agent is exploring the codebase...').start();
+  const spinner = ora(`${agent.displayName} is exploring the codebase...`).start();
 
   try {
-    const answer = await getAgentResponse(caseData, agent);
-    spinner.succeed('Agent completed');
+    const result = await getAgentResponse(caseData, agent, projectRoot, (chunk) => {
+      // Update spinner with progress indication
+      const lines = chunk.split('\n').filter(l => l.trim());
+      if (lines.length > 0) {
+        const lastLine = lines[lines.length - 1].slice(0, 50);
+        spinner.text = `${agent.displayName} is working... ${chalk.dim(lastLine)}`;
+      }
+    });
+
+    if (result.timedOut) {
+      spinner.fail(`${agent.displayName} timed out`);
+      console.log(chalk.yellow('\n  The agent took too long. Consider increasing the timeout.'));
+      return { grade: 0, skipped: true, durationMs: result.durationMs };
+    }
+
+    if (!result.success) {
+      spinner.fail(`${agent.displayName} failed: ${result.error}`);
+      return { grade: 0, skipped: true, durationMs: result.durationMs };
+    }
+
+    const durationSec = (result.durationMs / 1000).toFixed(1);
+    spinner.succeed(`${agent.displayName} completed in ${durationSec}s`);
 
     // Display the answer
     console.log(chalk.dim('\n  ─────────────────────────────────────────'));
     console.log(chalk.bold('  Agent\'s Answer:\n'));
-    console.log(formatAnswer(answer).split('\n').map(l => '  ' + l).join('\n'));
+    console.log(formatAnswer(result.answer).split('\n').map(l => '  ' + l).join('\n'));
     console.log(chalk.dim('\n  ─────────────────────────────────────────'));
 
+    // Show tools used if available
+    if (result.toolsUsed && result.toolsUsed.length > 0) {
+      console.log(chalk.dim(`\n  Tools used: ${result.toolsUsed.join(', ')}`));
+    }
+
     // Show grading scale and ask for grade
     showGradingScale();
     const grade = await askGrade(rl);
@@ -215,7 +244,7 @@ async function runInterviewQuestion(
     store.baselines[caseData.id] = {
       caseId: caseData.id,
       question: caseData.prompt,
-      answer,
+      answer: result.answer,
       grade,
       gradedAt: new Date().toISOString(),
       gradedBy: 'human',
@@ -226,7 +255,7 @@ async function runInterviewQuestion(
 
     console.log(chalk.green(`\n  ✓ Baseline saved (${grade}/10)`));
 
-    return { grade, skipped: false };
+    return { grade, skipped: false, durationMs: result.durationMs };
   } catch (err) {
     spinner.fail(`Failed: ${(err as Error).message}`);
     return { grade: 0, skipped: true };
@@ -247,8 +276,31 @@ export async function interviewCommand(options: InterviewOptions) {
     'sniff interview'
   ));
 
+  // Get the agent
+  let agent: AgentWrapper;
+  try {
+    agent = getAgent(options.agent);
+  } catch (err) {
+    console.log(chalk.red(`\n  Error: ${(err as Error).message}`));
+    return;
+  }
+
+  // Check agent availability
+  const spinner = ora(`Checking ${agent.displayName} availability...`).start();
+  const available = await agent.isAvailable();
+
+  if (!available) {
+    spinner.fail(`${agent.displayName} is not available`);
+    console.log(chalk.yellow(`\n  Make sure '${options.agent}' is installed and in your PATH.`));
+    console.log(chalk.dim(`  For Claude Code: https://claude.ai/code`));
+    return;
+  }
+
+  const version = await agent.getVersion();
+  spinner.succeed(`${agent.displayName} ${version ? `(${version})` : ''} is ready`);
+
   // Load comprehension cases
-  const spinner = ora('Loading comprehension cases...').start();
+  spinner.start('Loading comprehension cases...');
   const casesDir = getDefaultCasesDir();
 
   const cases = await loadCases(casesDir, {
@@ -307,7 +359,7 @@ export async function interviewCommand(options: InterviewOptions) {
       console.log(chalk.bold(`\n  [${i + 1}/${cases.length}] ${caseData.title}`));
       console.log(chalk.dim(`  Difficulty: ${caseData.difficulty}\n`));
 
-      const result = await runInterviewQuestion(caseData, options.agent, rl, store, projectRoot);
+      const result = await runInterviewQuestion(caseData, agent, rl, store, projectRoot);
       results.push({ caseId: caseData.id, ...result });
 
       if (i < cases.length - 1) {

From 83c5fe498ceb0a13f57a1d3c28c16435e38c9306 Mon Sep 17 00:00:00 2001
From: jharris1679 <j_harris@live.ca>
Date: Fri, 5 Dec 2025 17:35:00 -0500
Subject: [PATCH 2/5] feat: Add real-time streaming output for Claude Code
 agent
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Use --output-format stream-json for real-time streaming
- Add --include-partial-messages for text as it's generated
- Parse streaming JSON to extract text deltas and tool calls
- Format tool calls cleanly with name and key input
- Fix spawn stdio configuration for proper output capture
- Update interview prompt to request concise answers

ANS-441

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 src/agents/claude-code.ts     | 173 ++++++++++++++++++++++++++++------
 src/cli/commands/interview.ts |  39 ++++----
 2 files changed, 162 insertions(+), 50 deletions(-)

diff --git a/src/agents/claude-code.ts b/src/agents/claude-code.ts
index 75d3169..2fd6048 100644
--- a/src/agents/claude-code.ts
+++ b/src/agents/claude-code.ts
@@ -75,8 +75,10 @@ export class ClaudeCodeAgent implements AgentWrapper {
     return new Promise((resolve) => {
       // Build command args
       // -p: print mode (non-interactive, single prompt)
-      // --output-format: get structured output if available
-      const args = ['-p', prompt];
+      // --output-format stream-json: get real-time streaming JSON output
+      // --verbose: required for stream-json
+      // --include-partial-messages: stream text as it's generated
+      const args = ['-p', prompt, '--output-format', 'stream-json', '--verbose', '--include-partial-messages'];
 
       const proc = spawn(this.cliPath, args, {
         cwd: options.cwd,
@@ -86,21 +88,49 @@ export class ClaudeCodeAgent implements AgentWrapper {
           // Ensure non-interactive
           CI: 'true',
         },
-        timeout: timeoutMs,
+        stdio: ['pipe', 'pipe', 'pipe'],
       });
 
+      // Close stdin immediately - claude -p doesn't need interactive input
+      proc.stdin?.end();
+
       let stdout = '';
       let stderr = '';
       let timedOut = false;
+      let finalAnswer = '';
+      let lineBuffer = '';
 
       proc.stdout?.on('data', (data) => {
         const chunk = data.toString();
         stdout += chunk;
-        options.onOutput?.(chunk);
+
+        // Parse streaming JSON - each line is a JSON object
+        lineBuffer += chunk;
+        const lines = lineBuffer.split('\n');
+        lineBuffer = lines.pop() || ''; // Keep incomplete line in buffer
+
+        for (const line of lines) {
+          if (!line.trim()) continue;
+          try {
+            const msg = JSON.parse(line);
+            const displayText = this.extractDisplayText(msg);
+            if (displayText && options.onOutput) {
+              options.onOutput(displayText);
+            }
+            // Capture final answer from result
+            if (msg.type === 'result' && msg.result) {
+              finalAnswer = msg.result;
+            }
+          } catch {
+            // Not valid JSON, output raw
+            options.onOutput?.(line + '\n');
+          }
+        }
       });
 
       proc.stderr?.on('data', (data) => {
-        stderr += data.toString();
+        const chunk = data.toString();
+        stderr += chunk;
       });
 
       // Handle timeout
@@ -117,15 +147,11 @@ export class ClaudeCodeAgent implements AgentWrapper {
         clearTimeout(timer);
         const durationMs = Date.now() - startTime;
 
-        // Parse the output to extract the answer
-        // Claude Code's -p mode outputs the response directly
-        const answer = this.parseAnswer(stdout);
-
-        // Try to extract tool usage from output
-        const toolsUsed = this.parseToolsUsed(stdout);
+        // Try to extract tool usage from JSON output
+        const toolsUsed = this.parseToolsUsedFromJson(stdout);
 
         resolve({
-          answer,
+          answer: finalAnswer || this.parseAnswer(stdout),
           success: code === 0 && !timedOut,
           error: timedOut ? 'Timed out' : (code !== 0 ? `Exit code: ${code}` : undefined),
           timedOut,
@@ -168,27 +194,114 @@ export class ClaudeCodeAgent implements AgentWrapper {
   }
 
   /**
-   * Parse tools used from output
-   *
-   * Claude Code shows tool usage in its output. Try to extract them.
+   * Extract displayable text from a stream-json message
    */
-  private parseToolsUsed(stdout: string): string[] {
+  private extractDisplayText(msg: Record<string, unknown>): string | null {
+    // Handle streaming text deltas (with --include-partial-messages)
+    if (msg.type === 'stream_event') {
+      const event = msg.event as Record<string, unknown> | undefined;
+      if (event?.type === 'content_block_delta') {
+        const delta = event.delta as Record<string, unknown> | undefined;
+        if (delta?.type === 'text_delta' && typeof delta.text === 'string') {
+          return delta.text;
+        }
+      }
+      // Ignore content_block_start - we'll show details from assistant message
+    }
+
+    // Handle complete assistant messages with tool details
+    if (msg.type === 'assistant' && msg.message) {
+      const message = msg.message as Record<string, unknown>;
+      const content = message.content as Array<Record<string, unknown>> | undefined;
+      if (content && Array.isArray(content)) {
+        const textParts: string[] = [];
+        for (const part of content) {
+          if (part.type === 'tool_use') {
+            const input = part.input as Record<string, unknown> | undefined;
+            const formatted = this.formatToolCall(part.name as string, input);
+            if (formatted) {
+              textParts.push(formatted);
+            }
+          }
+        }
+        if (textParts.length > 0) {
+          return '\n' + textParts.join('\n') + '\n';
+        }
+      }
+    }
+
+    // Handle tool results - just show checkmark, skip content
+    if (msg.type === 'user') {
+      const toolResult = msg.tool_use_result as Record<string, unknown> | undefined;
+      if (toolResult) {
+        return null; // Don't show tool results - too noisy
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * Format a tool call for display
+   */
+  private formatToolCall(name: string, input: Record<string, unknown> | undefined): string | null {
+    if (!input) return `  › ${name}`;
+
+    const dim = '\x1b[2m'; // dim
+    const reset = '\x1b[0m';
+
+    switch (name) {
+      case 'Read': {
+        const path = (input.file_path as string || '').split('/').slice(-2).join('/');
+        return `  › Read ${dim}${path}${reset}`;
+      }
+      case 'Glob': {
+        return `  › Glob ${dim}${input.pattern || ''}${reset}`;
+      }
+      case 'Grep': {
+        return `  › Grep ${dim}"${input.pattern || ''}"${reset}`;
+      }
+      case 'Bash': {
+        const cmd = (input.command as string || '').substring(0, 50);
+        const truncated = (input.command as string || '').length > 50 ? '...' : '';
+        return `  › Bash ${dim}${cmd}${truncated}${reset}`;
+      }
+      case 'Edit':
+      case 'Write': {
+        const path = (input.file_path as string || '').split('/').slice(-2).join('/');
+        return `  › ${name} ${dim}${path}${reset}`;
+      }
+      case 'Task': {
+        return `  › Task ${dim}${input.description || ''}${reset}`;
+      }
+      default:
+        return `  › ${name}`;
+    }
+  }
+
+  /**
+   * Parse tools used from JSON output
+   */
+  private parseToolsUsedFromJson(stdout: string): string[] {
     const tools: Set<string> = new Set();
 
-    // Look for common tool patterns in Claude Code output
-    const toolPatterns = [
-      /Read\s+\S+/g,        // Read file
-      /Edit\s+\S+/g,        // Edit file
-      /Write\s+\S+/g,       // Write file
-      /Bash\s*\([^)]+\)/g,  // Bash command
-      /Grep\s+\S+/g,        // Grep search
-      /Glob\s+\S+/g,        // Glob search
-    ];
-
-    for (const pattern of toolPatterns) {
-      const matches = stdout.match(pattern);
-      if (matches) {
-        matches.forEach((m) => tools.add(m.split(/\s+/)[0]));
+    const lines = stdout.split('\n');
+    for (const line of lines) {
+      if (!line.trim()) continue;
+      try {
+        const msg = JSON.parse(line);
+        if (msg.type === 'assistant' && msg.message) {
+          const content = msg.message.content;
+          if (Array.isArray(content)) {
+            for (const part of content) {
+              if (part.type === 'tool_use' && part.name) {
+                tools.add(part.name);
+              }
+            }
+          }
+        }
+      } catch {
+        // Skip invalid JSON lines
       }
     }
 
diff --git a/src/cli/commands/interview.ts b/src/cli/commands/interview.ts
index ccbc49a..59e2fe4 100644
--- a/src/cli/commands/interview.ts
+++ b/src/cli/commands/interview.ts
@@ -160,7 +160,7 @@ Please answer the following question by exploring the codebase:
 
 ${caseData.prompt}
 
-Take your time to explore and provide a thorough, accurate answer with specific file references where relevant.`;
+Be concise but accurate. Focus on the key points with specific file references where relevant. Aim for a clear, well-organized answer that a developer could quickly scan.`;
 
   const result = await agent.run(prompt, {
     cwd,
@@ -195,38 +195,37 @@ async function runInterviewQuestion(
     }
   }
 
-  // Get agent's response
-  const spinner = ora(`${agent.displayName} is exploring the codebase...`).start();
+  // Get agent's response - stream output live
+  console.log(chalk.dim(`\n  ${agent.displayName} is exploring...\n`));
+  console.log(chalk.dim('  ─────────────────────────────────────────\n'));
+
+  let outputStarted = false;
+  const startTime = Date.now();
 
   try {
     const result = await getAgentResponse(caseData, agent, projectRoot, (chunk) => {
-      // Update spinner with progress indication
-      const lines = chunk.split('\n').filter(l => l.trim());
-      if (lines.length > 0) {
-        const lastLine = lines[lines.length - 1].slice(0, 50);
-        spinner.text = `${agent.displayName} is working... ${chalk.dim(lastLine)}`;
+      // Stream output directly to console
+      if (!outputStarted) {
+        outputStarted = true;
       }
+      process.stdout.write(chunk);
     });
 
+    const durationSec = ((Date.now() - startTime) / 1000).toFixed(1);
+    console.log(chalk.dim('\n\n  ─────────────────────────────────────────'));
+
     if (result.timedOut) {
-      spinner.fail(`${agent.displayName} timed out`);
-      console.log(chalk.yellow('\n  The agent took too long. Consider increasing the timeout.'));
+      console.log(chalk.yellow(`\n  ✗ ${agent.displayName} timed out after ${durationSec}s`));
+      console.log(chalk.yellow('  The agent took too long. Consider increasing the timeout.'));
       return { grade: 0, skipped: true, durationMs: result.durationMs };
     }
 
     if (!result.success) {
-      spinner.fail(`${agent.displayName} failed: ${result.error}`);
+      console.log(chalk.red(`\n  ✗ ${agent.displayName} failed: ${result.error}`));
       return { grade: 0, skipped: true, durationMs: result.durationMs };
     }
 
-    const durationSec = (result.durationMs / 1000).toFixed(1);
-    spinner.succeed(`${agent.displayName} completed in ${durationSec}s`);
-
-    // Display the answer
-    console.log(chalk.dim('\n  ─────────────────────────────────────────'));
-    console.log(chalk.bold('  Agent\'s Answer:\n'));
-    console.log(formatAnswer(result.answer).split('\n').map(l => '  ' + l).join('\n'));
-    console.log(chalk.dim('\n  ─────────────────────────────────────────'));
+    console.log(chalk.green(`\n  ✓ ${agent.displayName} completed in ${durationSec}s`));
 
     // Show tools used if available
     if (result.toolsUsed && result.toolsUsed.length > 0) {
@@ -257,7 +256,7 @@ async function runInterviewQuestion(
 
     return { grade, skipped: false, durationMs: result.durationMs };
   } catch (err) {
-    spinner.fail(`Failed: ${(err as Error).message}`);
+    console.log(chalk.red(`\n  ✗ Failed: ${(err as Error).message}`));
     return { grade: 0, skipped: true };
   }
 }

From 0feea213452fb2ebdda0a0f0f996dde867a05e57 Mon Sep 17 00:00:00 2001
From: jharris1679 <j_harris@live.ca>
Date: Sun, 7 Dec 2025 15:01:23 -0500
Subject: [PATCH 3/5] feat: Add colorful animated exploration spinner
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Cycling status messages (Reading files, Analyzing code, etc.)
- Agent name in orange, status in cycling colors
- Custom spinner frames (◐ ◓ ◑ ◒)
- Stops when output starts streaming

ANS-441

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 .sniffbench/baselines.json    | 31 +++++++++++++++++
 src/cli/commands/interview.ts | 64 +++++++++++++++++++++++++++++++----
 2 files changed, 89 insertions(+), 6 deletions(-)
 create mode 100644 .sniffbench/baselines.json

diff --git a/.sniffbench/baselines.json b/.sniffbench/baselines.json
new file mode 100644
index 0000000..a7dc0a6
--- /dev/null
+++ b/.sniffbench/baselines.json
@@ -0,0 +1,31 @@
+{
+  "version": "1.0",
+  "repoPath": "/Users/primaryuser/answerlayer/sniffbench",
+  "createdAt": "2025-12-05T22:14:10.828Z",
+  "baselines": {
+    "comp-001": {
+      "caseId": "comp-001",
+      "question": "Imagine you're writing the README for this project. Help me understand:\n\n- What does this project do? What problem does it solve?\n- Who is it for?\n- What are the main components or modules?\n- How is the codebase organized?\n\nGive me the \"elevator pitch\" version, then a slightly deeper dive\ninto the structure.\n",
+      "answer": "Based on my exploration, here's a comprehensive overview of the **sniffbench** project:\n\n---\n\n## 🎯 Elevator Pitch\n\n**Sniffbench is \"pytest for AI coding agents.\"** It's a benchmark suite that helps developers objectively measure how well AI assistants (Claude Code, Cursor, Aider, etc.) perform on real coding tasks.\n\n**The problem it solves:** When you change your AI coding setup—switch models, tweak prompts, try new tools—you're flying blind. Did it actually get better? Sniffbench provides data-driven evaluation so you can measure improvement objectively instead of relying on gut feeling.\n\n**Target audience:** Developers using AI coding assistants who want to establish baselines and compare performance across configurations, as well as tool developers building or improving AI agents.\n\n---\n\n## 📁 Codebase Structure\n\n```\nsniffbench/\n├── src/\n│   ├── cli/              # Command-line interface (Commander.js)\n│   │   ├── index.ts      # CLI entry point with 9 commands\n│   │   └── commands/     # Individual command implementations\n│   ├── agents/           # Agent wrapper infrastructure\n│   │   ├── types.ts      # AgentWrapper interface\n│   │   ├── claude-code.ts # Claude Code implementation  \n│   │   └── registry.ts   # Agent discovery/registration\n│   ├── cases/            # Test case management\n│   │   ├── types.ts      # Case & rubric schemas\n│   │   └── loader.ts     # YAML/JSON loader with validation\n│   ├── sandbox/          # Docker sandboxing for safe execution\n│   │   └── types.ts      # Sandbox interfaces\n│   ├── rubrics/          # Grading criteria system\n│   │   └── defaults.ts   # Default evaluation weights\n│   └── evaluation/       # Evaluation engine\n│       └── runner.ts     # Core runner: runCases()\n│\n├── cases/                # Test case definitions\n│   ├── comprehension/    # 12 interview-style evaluation cases\n│   └── bootstrap/        # 2 universal starter test cases\n│\n├── VALUES.md             # Philosophy: what good agents do\n├── ROADMAP.md            # 6-phase development plan\n└── package.json          # Node 18+, TypeScript, pnpm\n```\n\n---\n\n## 🔧 Main Components\n\n### 1. **Agent Wrappers** (`src/agents/`)\nStandard interface for any coding agent. Currently implements Claude Code (`claude-code.ts:1-80`), which spawns the `claude` CLI and tracks success, duration, tools used, and tokens.\n\n```typescript\n// src/agents/types.ts:23-35\ninterface AgentWrapper {\n  name: string;\n  run(prompt: string, options?: AgentRunOptions): Promise<AgentResult>;\n  isAvailable(): Promise<boolean>;\n}\n```\n\n### 2. **Case System** (`src/cases/`)\nYAML-defined test cases with prompts, files, difficulty levels, and grading rubrics. Categories include comprehension, bootstrap, and generated cases.\n\n### 3. **Rubric System** (`src/rubrics/`)\nDefault evaluation criteria with weighted metrics (`defaults.ts:1-80`):\n- Correctness: 40%\n- Code Quality: 25%\n- Safety: 20%\n- Performance: 10%\n- Maintainability: 5%\n\n### 4. **Sandbox** (`src/sandbox/`)\nDocker-based isolation to run evaluations safely without corrupting real codebases.\n\n### 5. **CLI** (`src/cli/index.ts:1-119`)\nCommands: `init`, `run`, `add`, `compare`, `report`, `cases`, `status`, `doctor`, `interview`\n\n---\n\n## 💡 Design Philosophy\n\nFrom `VALUES.md`, sniffbench measures what *actually matters* for coding agents:\n- **Style Adherence** – Follows existing project patterns\n- **Targeted Changes** – Minimal, focused changes (no over-engineering)\n- **Efficient Navigation** – Researches codebase before acting\n- **Non-Regression** – Existing tests still pass\n\nIt explicitly does **not** measure generic \"best practices\" divorced from project context.\n\n---\n\n## 📊 Current Status\n\n**Implemented:** CLI framework, case management, agent wrapper infrastructure, Claude Code integration, rubric system, 14 test cases\n\n**In Progress:** Docker sandboxing, interview command refinements\n\n**Planned:** LLM-powered case generation, multi-agent support (Cursor, Aider), reporting/comparison dashboards",
+      "grade": 9,
+      "gradedAt": "2025-12-05T22:16:04.914Z",
+      "gradedBy": "human"
+    },
+    "comp-007": {
+      "caseId": "comp-007",
+      "question": "What are the 3-5 most important external dependencies in this project,\nand how are they used?\n\nFor each one:\n- What problem does it solve?\n- Where in the code is it used?\n- Are there any wrappers or abstractions around it?\n- Could it be replaced? How hard would that be?\n",
+      "answer": "## Key External Dependencies in Sniffbench\n\nBased on my analysis, here are the **5 most important external dependencies**:\n\n---\n\n### 1. **Dockerode** (`dockerode@^4.0.2`) - CRITICAL\n\n**Problem:** Provides Node.js API for Docker daemon communication - creates isolated sandbox containers for agent evaluations.\n\n**Usage:**\n- **Primary:** `src/sandbox/docker.ts` (472 lines) - container lifecycle, execution, resource limits\n- **Consumers:** `src/cli/commands/run.ts`, `src/evaluation/runner.ts`, `src/cli/commands/doctor.ts`\n\n**Abstraction:** **Excellent** - Clean interface in `src/sandbox/types.ts` (`SandboxManager`, `Sandbox` interfaces). Factory in `src/sandbox/index.ts`. Rest of codebase uses interfaces only.\n\n**Replacement:** Medium difficulty - abstraction layer makes this feasible. Would implement new `SandboxManager` (e.g., Podman). ~2-4 days.\n\n---\n\n### 2. **Commander** (`commander@^12.0.0`) - CRITICAL\n\n**Problem:** CLI argument parsing, command routing, help text generation.\n\n**Usage:**\n- **Primary:** `src/cli/index.ts` (lines 3-118) - defines all commands (`init`, `run`, `add`, `cases`, `interview`, etc.)\n\n**Abstraction:** None - directly builds CLI tree.\n\n**Replacement:** High difficulty - entire CLI structure tied to Commander. ~3-5 days to port to yargs/minimist.\n\n---\n\n### 3. **YAML** (`yaml@^2.3.4`) - HIGH\n\n**Problem:** Parses YAML files for test case definitions and rubric configurations.\n\n**Usage:**\n- `src/cases/loader.ts:220` - parses case YAML files\n- `src/rubrics/loader.ts:57` - parses rubric definitions with inheritance\n\n**Abstraction:** None - direct `YAML.parse()` calls with error wrapping.\n\n**Replacement:** Medium - swap to js-yaml or similar. 2 files. ~1 day.\n\n---\n\n### 4. **Chalk** (`chalk@^5.3.0`) - HIGH\n\n**Problem:** Terminal coloring and styling for readable CLI output.\n\n**Usage:** 11 files across CLI commands - `src/utils/ui.ts` (box drawing), `src/cli/commands/run.ts` (status colors), all command files.\n\n**Abstraction:** Minimal - `src/utils/ui.ts` has a `box()` helper, but chalk used directly everywhere.\n\n**Replacement:** Medium - could use ANSI codes or alternatives. 11 files. ~1-2 days.\n\n---\n\n### 5. **Ora** (`ora@^8.0.0`) - MEDIUM\n\n**Problem:** Terminal spinners for progress indication.\n\n**Usage:** 5 CLI command files - `run.ts`, `status.ts`, `doctor.ts`, `cases.ts`, `interview.ts`\n\n**Abstraction:** None - direct instantiation.\n\n**Replacement:** Low difficulty - simple functionality. ~0.5-1 day.\n\n---\n\n### Summary\n\n| Dependency | Criticality | Abstraction | Replacement Cost |\n|------------|-------------|-------------|------------------|\n| Dockerode  | Critical    | Excellent   | Medium (~2-4d)   |\n| Commander  | Critical    | None        | High (~3-5d)     |\n| YAML       | High        | None        | Medium (~1d)     |\n| Chalk      | High        | Minimal     | Medium (~1-2d)   |\n| Ora        | Medium      | None        | Low (~0.5-1d)    |\n\n**Key insight:** Dockerode is the best-architected dependency with clean interface abstractions in `src/sandbox/`, while Commander is the most deeply embedded with no abstraction layer.",
+      "grade": 9,
+      "gradedAt": "2025-12-05T22:31:12.864Z",
+      "gradedBy": "human"
+    },
+    "comp-006": {
+      "caseId": "comp-006",
+      "question": "How is this application configured for different environments?\n\n- Where do configuration values come from (env vars, files, etc.)?\n- How does the code access configuration?\n- What's different between development, staging, and production?\n- Are there any secrets? How are they managed?\n",
+      "answer": "## Environment Configuration in Sniffbench\n\n### Configuration Sources\n\n1. **CLI Arguments** (highest priority)\n   - `--agent`, `--cases`, `--output`, `--timeout`, `--network` flags\n   - Defined in `src/cli/commands/*.ts`\n\n2. **Environment Variables**\n   - `CI='true'` - Set by `src/agents/claude-code.ts:89` for non-interactive mode\n   - `EDITOR` - Used by `src/cases/index.ts:118` for editing cases (defaults to 'vim')\n   - All env vars inherited by agent subprocesses (`src/agents/claude-code.ts:85-90`)\n\n3. **Project-Specific Directory** (`.sniffbench/`)\n   - `cases/` - Custom test cases\n   - `rubrics/` - Custom evaluation rubrics\n   - `baselines.json` - Interview baseline results\n\n4. **Installation Defaults**\n   - `cases/` and `rubrics/` directories in package root\n   - Hardcoded defaults in `src/sandbox/docker.ts:20-26` and `src/rubrics/defaults.ts`\n\n### How Code Accesses Configuration\n\n```typescript\n// Cases: src/cases/loader.ts:342-351\ngetDefaultCasesDir(projectRoot) // Checks .sniffbench/cases first, then falls back\n\n// Rubrics: src/rubrics/loader.ts:178-190  \nloadRubrics(projectRoot) // Loads from both project-specific and installation dirs\n\n// Sandbox: src/sandbox/docker.ts:189-197\nresolvedConfig = { ...config, image: config.image || DEFAULTS.image, ... }\n```\n\n### Environment Differentiation\n\n**Currently no dev/staging/production differentiation exists.** The architecture uses path-based resolution:\n1. Project-specific (`.sniffbench/`) - highest priority\n2. Installation defaults - fallback\n\nThis pattern could support future environment configs but isn't implemented.\n\n### Secrets Management\n\n**No explicit secret management implemented.** Security is handled via Docker isolation:\n- Network disabled by default (`src/sandbox/docker.ts:235`)\n- Read-only root filesystem (line 238)\n- Capability drops (lines 244-245)\n- No privileged mode (line 248)\n\nThe `SandboxConfig` interface supports `env?: Record<string, string>` for passing environment variables to sandboxed processes, which could be used for secrets in future.\n\n### Key Default Values\n\n| Setting | Default | Location |\n|---------|---------|----------|\n| Docker image | `node:20-slim` | `src/sandbox/docker.ts:20` |\n| Memory | 512MB | `src/sandbox/docker.ts:21` |\n| Timeout | 300s | `src/sandbox/docker.ts:23` |\n| Network | disabled | `src/sandbox/docker.ts:24` |\n| Pass threshold | 70% | `src/evaluation/runner.ts:366` |",
+      "grade": 9,
+      "gradedAt": "2025-12-05T22:33:15.420Z",
+      "gradedBy": "human"
+    }
+  }
+}
\ No newline at end of file
diff --git a/src/cli/commands/interview.ts b/src/cli/commands/interview.ts
index 59e2fe4..39953b7 100644
--- a/src/cli/commands/interview.ts
+++ b/src/cli/commands/interview.ts
@@ -8,7 +8,7 @@
  */
 
 import chalk from 'chalk';
-import ora from 'ora';
+import ora, { Ora } from 'ora';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as readline from 'readline';
@@ -17,6 +17,48 @@ import { loadCases, getDefaultCasesDir } from '../../cases';
 import { Case } from '../../cases/types';
 import { getAgent, AgentWrapper, AgentResult } from '../../agents';
 
+/**
+ * Exploration status messages - cycles through these while agent works
+ */
+const EXPLORATION_STATES = [
+  { text: 'Reading files', color: chalk.cyan },
+  { text: 'Scanning structure', color: chalk.blue },
+  { text: 'Analyzing code', color: chalk.magenta },
+  { text: 'Finding patterns', color: chalk.yellow },
+  { text: 'Building context', color: chalk.green },
+  { text: 'Connecting dots', color: chalk.cyan },
+];
+
+/**
+ * Create an animated exploration spinner
+ */
+function createExplorationSpinner(agentName: string): { spinner: Ora; stop: () => void } {
+  let stateIndex = 0;
+  const spinner = ora({
+    text: `${chalk.bold.hex('#D97706')(agentName)} ${EXPLORATION_STATES[0].color(EXPLORATION_STATES[0].text)}`,
+    spinner: {
+      interval: 80,
+      frames: ['◐', '◓', '◑', '◒'],
+    },
+    color: 'yellow',
+  }).start();
+
+  // Cycle through states
+  const interval = setInterval(() => {
+    stateIndex = (stateIndex + 1) % EXPLORATION_STATES.length;
+    const state = EXPLORATION_STATES[stateIndex];
+    spinner.text = `${chalk.bold.hex('#D97706')(agentName)} ${state.color(state.text)}`;
+  }, 2000);
+
+  return {
+    spinner,
+    stop: () => {
+      clearInterval(interval);
+      spinner.stop();
+    },
+  };
+}
+
 interface InterviewOptions {
   cases?: string;
   agent: string;
@@ -195,24 +237,33 @@ async function runInterviewQuestion(
     }
   }
 
-  // Get agent's response - stream output live
-  console.log(chalk.dim(`\n  ${agent.displayName} is exploring...\n`));
-  console.log(chalk.dim('  ─────────────────────────────────────────\n'));
+  // Get agent's response - stream output live with animated spinner
+  console.log('');
+  const exploration = createExplorationSpinner(agent.displayName);
 
   let outputStarted = false;
   const startTime = Date.now();
 
   try {
     const result = await getAgentResponse(caseData, agent, projectRoot, (chunk) => {
-      // Stream output directly to console
+      // Stop spinner and show separator when first output arrives
       if (!outputStarted) {
         outputStarted = true;
+        exploration.stop();
+        console.log(chalk.dim('\n  ─────────────────────────────────────────\n'));
       }
       process.stdout.write(chunk);
     });
 
+    // Ensure spinner is stopped
+    exploration.stop();
+
     const durationSec = ((Date.now() - startTime) / 1000).toFixed(1);
-    console.log(chalk.dim('\n\n  ─────────────────────────────────────────'));
+    if (!outputStarted) {
+      console.log(chalk.dim('\n  ─────────────────────────────────────────'));
+    } else {
+      console.log(chalk.dim('\n\n  ─────────────────────────────────────────'));
+    }
 
     if (result.timedOut) {
       console.log(chalk.yellow(`\n  ✗ ${agent.displayName} timed out after ${durationSec}s`));
@@ -256,6 +307,7 @@ async function runInterviewQuestion(
 
     return { grade, skipped: false, durationMs: result.durationMs };
   } catch (err) {
+    exploration.stop();
     console.log(chalk.red(`\n  ✗ Failed: ${(err as Error).message}`));
     return { grade: 0, skipped: true };
   }

From 1122337b96174262f2cee0c1d1b1375a7a98ac3a Mon Sep 17 00:00:00 2001
From: jharris1679 <j_harris@live.ca>
Date: Sun, 7 Dec 2025 17:19:46 -0500
Subject: [PATCH 4/5] chore: ignore .sniffbench directory
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Remove .sniffbench/baselines.json from tracking and add directory to .gitignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 .gitignore                 |  1 +
 .sniffbench/baselines.json | 31 -------------------------------
 2 files changed, 1 insertion(+), 31 deletions(-)
 delete mode 100644 .sniffbench/baselines.json

diff --git a/.gitignore b/.gitignore
index 1bb0d28..d526afb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,3 +46,4 @@ coverage/
 .env.local
 local_config.yaml
 results/
+.sniffbench/
diff --git a/.sniffbench/baselines.json b/.sniffbench/baselines.json
deleted file mode 100644
index a7dc0a6..0000000
--- a/.sniffbench/baselines.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
-  "version": "1.0",
-  "repoPath": "/Users/primaryuser/answerlayer/sniffbench",
-  "createdAt": "2025-12-05T22:14:10.828Z",
-  "baselines": {
-    "comp-001": {
-      "caseId": "comp-001",
-      "question": "Imagine you're writing the README for this project. Help me understand:\n\n- What does this project do? What problem does it solve?\n- Who is it for?\n- What are the main components or modules?\n- How is the codebase organized?\n\nGive me the \"elevator pitch\" version, then a slightly deeper dive\ninto the structure.\n",
-      "answer": "Based on my exploration, here's a comprehensive overview of the **sniffbench** project:\n\n---\n\n## 🎯 Elevator Pitch\n\n**Sniffbench is \"pytest for AI coding agents.\"** It's a benchmark suite that helps developers objectively measure how well AI assistants (Claude Code, Cursor, Aider, etc.) perform on real coding tasks.\n\n**The problem it solves:** When you change your AI coding setup—switch models, tweak prompts, try new tools—you're flying blind. Did it actually get better? Sniffbench provides data-driven evaluation so you can measure improvement objectively instead of relying on gut feeling.\n\n**Target audience:** Developers using AI coding assistants who want to establish baselines and compare performance across configurations, as well as tool developers building or improving AI agents.\n\n---\n\n## 📁 Codebase Structure\n\n```\nsniffbench/\n├── src/\n│   ├── cli/              # Command-line interface (Commander.js)\n│   │   ├── index.ts      # CLI entry point with 9 commands\n│   │   └── commands/     # Individual command implementations\n│   ├── agents/           # Agent wrapper infrastructure\n│   │   ├── types.ts      # AgentWrapper interface\n│   │   ├── claude-code.ts # Claude Code implementation  \n│   │   └── registry.ts   # Agent discovery/registration\n│   ├── cases/            # Test case management\n│   │   ├── types.ts      # Case & rubric schemas\n│   │   └── loader.ts     # YAML/JSON loader with validation\n│   ├── sandbox/          # Docker sandboxing for safe execution\n│   │   └── types.ts      # Sandbox interfaces\n│   ├── rubrics/          # Grading criteria system\n│   │   └── defaults.ts   # Default evaluation weights\n│   └── evaluation/       # Evaluation engine\n│       └── runner.ts     # Core runner: runCases()\n│\n├── cases/                # Test case definitions\n│   ├── comprehension/    # 12 interview-style evaluation cases\n│   └── bootstrap/        # 2 universal starter test cases\n│\n├── VALUES.md             # Philosophy: what good agents do\n├── ROADMAP.md            # 6-phase development plan\n└── package.json          # Node 18+, TypeScript, pnpm\n```\n\n---\n\n## 🔧 Main Components\n\n### 1. **Agent Wrappers** (`src/agents/`)\nStandard interface for any coding agent. Currently implements Claude Code (`claude-code.ts:1-80`), which spawns the `claude` CLI and tracks success, duration, tools used, and tokens.\n\n```typescript\n// src/agents/types.ts:23-35\ninterface AgentWrapper {\n  name: string;\n  run(prompt: string, options?: AgentRunOptions): Promise<AgentResult>;\n  isAvailable(): Promise<boolean>;\n}\n```\n\n### 2. **Case System** (`src/cases/`)\nYAML-defined test cases with prompts, files, difficulty levels, and grading rubrics. Categories include comprehension, bootstrap, and generated cases.\n\n### 3. **Rubric System** (`src/rubrics/`)\nDefault evaluation criteria with weighted metrics (`defaults.ts:1-80`):\n- Correctness: 40%\n- Code Quality: 25%\n- Safety: 20%\n- Performance: 10%\n- Maintainability: 5%\n\n### 4. **Sandbox** (`src/sandbox/`)\nDocker-based isolation to run evaluations safely without corrupting real codebases.\n\n### 5. **CLI** (`src/cli/index.ts:1-119`)\nCommands: `init`, `run`, `add`, `compare`, `report`, `cases`, `status`, `doctor`, `interview`\n\n---\n\n## 💡 Design Philosophy\n\nFrom `VALUES.md`, sniffbench measures what *actually matters* for coding agents:\n- **Style Adherence** – Follows existing project patterns\n- **Targeted Changes** – Minimal, focused changes (no over-engineering)\n- **Efficient Navigation** – Researches codebase before acting\n- **Non-Regression** – Existing tests still pass\n\nIt explicitly does **not** measure generic \"best practices\" divorced from project context.\n\n---\n\n## 📊 Current Status\n\n**Implemented:** CLI framework, case management, agent wrapper infrastructure, Claude Code integration, rubric system, 14 test cases\n\n**In Progress:** Docker sandboxing, interview command refinements\n\n**Planned:** LLM-powered case generation, multi-agent support (Cursor, Aider), reporting/comparison dashboards",
-      "grade": 9,
-      "gradedAt": "2025-12-05T22:16:04.914Z",
-      "gradedBy": "human"
-    },
-    "comp-007": {
-      "caseId": "comp-007",
-      "question": "What are the 3-5 most important external dependencies in this project,\nand how are they used?\n\nFor each one:\n- What problem does it solve?\n- Where in the code is it used?\n- Are there any wrappers or abstractions around it?\n- Could it be replaced? How hard would that be?\n",
-      "answer": "## Key External Dependencies in Sniffbench\n\nBased on my analysis, here are the **5 most important external dependencies**:\n\n---\n\n### 1. **Dockerode** (`dockerode@^4.0.2`) - CRITICAL\n\n**Problem:** Provides Node.js API for Docker daemon communication - creates isolated sandbox containers for agent evaluations.\n\n**Usage:**\n- **Primary:** `src/sandbox/docker.ts` (472 lines) - container lifecycle, execution, resource limits\n- **Consumers:** `src/cli/commands/run.ts`, `src/evaluation/runner.ts`, `src/cli/commands/doctor.ts`\n\n**Abstraction:** **Excellent** - Clean interface in `src/sandbox/types.ts` (`SandboxManager`, `Sandbox` interfaces). Factory in `src/sandbox/index.ts`. Rest of codebase uses interfaces only.\n\n**Replacement:** Medium difficulty - abstraction layer makes this feasible. Would implement new `SandboxManager` (e.g., Podman). ~2-4 days.\n\n---\n\n### 2. **Commander** (`commander@^12.0.0`) - CRITICAL\n\n**Problem:** CLI argument parsing, command routing, help text generation.\n\n**Usage:**\n- **Primary:** `src/cli/index.ts` (lines 3-118) - defines all commands (`init`, `run`, `add`, `cases`, `interview`, etc.)\n\n**Abstraction:** None - directly builds CLI tree.\n\n**Replacement:** High difficulty - entire CLI structure tied to Commander. ~3-5 days to port to yargs/minimist.\n\n---\n\n### 3. **YAML** (`yaml@^2.3.4`) - HIGH\n\n**Problem:** Parses YAML files for test case definitions and rubric configurations.\n\n**Usage:**\n- `src/cases/loader.ts:220` - parses case YAML files\n- `src/rubrics/loader.ts:57` - parses rubric definitions with inheritance\n\n**Abstraction:** None - direct `YAML.parse()` calls with error wrapping.\n\n**Replacement:** Medium - swap to js-yaml or similar. 2 files. ~1 day.\n\n---\n\n### 4. **Chalk** (`chalk@^5.3.0`) - HIGH\n\n**Problem:** Terminal coloring and styling for readable CLI output.\n\n**Usage:** 11 files across CLI commands - `src/utils/ui.ts` (box drawing), `src/cli/commands/run.ts` (status colors), all command files.\n\n**Abstraction:** Minimal - `src/utils/ui.ts` has a `box()` helper, but chalk used directly everywhere.\n\n**Replacement:** Medium - could use ANSI codes or alternatives. 11 files. ~1-2 days.\n\n---\n\n### 5. **Ora** (`ora@^8.0.0`) - MEDIUM\n\n**Problem:** Terminal spinners for progress indication.\n\n**Usage:** 5 CLI command files - `run.ts`, `status.ts`, `doctor.ts`, `cases.ts`, `interview.ts`\n\n**Abstraction:** None - direct instantiation.\n\n**Replacement:** Low difficulty - simple functionality. ~0.5-1 day.\n\n---\n\n### Summary\n\n| Dependency | Criticality | Abstraction | Replacement Cost |\n|------------|-------------|-------------|------------------|\n| Dockerode  | Critical    | Excellent   | Medium (~2-4d)   |\n| Commander  | Critical    | None        | High (~3-5d)     |\n| YAML       | High        | None        | Medium (~1d)     |\n| Chalk      | High        | Minimal     | Medium (~1-2d)   |\n| Ora        | Medium      | None        | Low (~0.5-1d)    |\n\n**Key insight:** Dockerode is the best-architected dependency with clean interface abstractions in `src/sandbox/`, while Commander is the most deeply embedded with no abstraction layer.",
-      "grade": 9,
-      "gradedAt": "2025-12-05T22:31:12.864Z",
-      "gradedBy": "human"
-    },
-    "comp-006": {
-      "caseId": "comp-006",
-      "question": "How is this application configured for different environments?\n\n- Where do configuration values come from (env vars, files, etc.)?\n- How does the code access configuration?\n- What's different between development, staging, and production?\n- Are there any secrets? How are they managed?\n",
-      "answer": "## Environment Configuration in Sniffbench\n\n### Configuration Sources\n\n1. **CLI Arguments** (highest priority)\n   - `--agent`, `--cases`, `--output`, `--timeout`, `--network` flags\n   - Defined in `src/cli/commands/*.ts`\n\n2. **Environment Variables**\n   - `CI='true'` - Set by `src/agents/claude-code.ts:89` for non-interactive mode\n   - `EDITOR` - Used by `src/cases/index.ts:118` for editing cases (defaults to 'vim')\n   - All env vars inherited by agent subprocesses (`src/agents/claude-code.ts:85-90`)\n\n3. **Project-Specific Directory** (`.sniffbench/`)\n   - `cases/` - Custom test cases\n   - `rubrics/` - Custom evaluation rubrics\n   - `baselines.json` - Interview baseline results\n\n4. **Installation Defaults**\n   - `cases/` and `rubrics/` directories in package root\n   - Hardcoded defaults in `src/sandbox/docker.ts:20-26` and `src/rubrics/defaults.ts`\n\n### How Code Accesses Configuration\n\n```typescript\n// Cases: src/cases/loader.ts:342-351\ngetDefaultCasesDir(projectRoot) // Checks .sniffbench/cases first, then falls back\n\n// Rubrics: src/rubrics/loader.ts:178-190  \nloadRubrics(projectRoot) // Loads from both project-specific and installation dirs\n\n// Sandbox: src/sandbox/docker.ts:189-197\nresolvedConfig = { ...config, image: config.image || DEFAULTS.image, ... }\n```\n\n### Environment Differentiation\n\n**Currently no dev/staging/production differentiation exists.** The architecture uses path-based resolution:\n1. Project-specific (`.sniffbench/`) - highest priority\n2. Installation defaults - fallback\n\nThis pattern could support future environment configs but isn't implemented.\n\n### Secrets Management\n\n**No explicit secret management implemented.** Security is handled via Docker isolation:\n- Network disabled by default (`src/sandbox/docker.ts:235`)\n- Read-only root filesystem (line 238)\n- Capability drops (lines 244-245)\n- No privileged mode (line 248)\n\nThe `SandboxConfig` interface supports `env?: Record<string, string>` for passing environment variables to sandboxed processes, which could be used for secrets in future.\n\n### Key Default Values\n\n| Setting | Default | Location |\n|---------|---------|----------|\n| Docker image | `node:20-slim` | `src/sandbox/docker.ts:20` |\n| Memory | 512MB | `src/sandbox/docker.ts:21` |\n| Timeout | 300s | `src/sandbox/docker.ts:23` |\n| Network | disabled | `src/sandbox/docker.ts:24` |\n| Pass threshold | 70% | `src/evaluation/runner.ts:366` |",
-      "grade": 9,
-      "gradedAt": "2025-12-05T22:33:15.420Z",
-      "gradedBy": "human"
-    }
-  }
-}
\ No newline at end of file

From e1370a7955e623841ffeaf64d40aa731b5da42d3 Mon Sep 17 00:00:00 2001
From: jharris1679 <j_harris@live.ca>
Date: Sun, 7 Dec 2025 19:25:49 -0500
Subject: [PATCH 5/5] feat: improve interview UX with live streaming and input
 timeout
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Show tool call count in spinner while agent works
- Stream agent's text response live when it starts
- Filter tool calls (›) from text output to prevent garbled display
- Add 5-minute timeout for user input prompts
- Recreate readline interface after agent run to handle stdin disruption
- Preserve proper markdown formatting in streamed output

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 src/cli/commands/interview.ts | 167 ++++++++++++++++++++++++++++------
 1 file changed, 138 insertions(+), 29 deletions(-)

diff --git a/src/cli/commands/interview.ts b/src/cli/commands/interview.ts
index 39953b7..bfd7970 100644
--- a/src/cli/commands/interview.ts
+++ b/src/cli/commands/interview.ts
@@ -30,12 +30,24 @@ const EXPLORATION_STATES = [
 ];
 
 /**
- * Create an animated exploration spinner
+ * Create an animated exploration spinner that shows tool activity
  */
-function createExplorationSpinner(agentName: string): { spinner: Ora; stop: () => void } {
+function createExplorationSpinner(agentName: string): {
+  spinner: Ora;
+  stop: () => void;
+  updateWithToolCall: (toolInfo: string) => void;
+  toolCalls: string[];
+} {
   let stateIndex = 0;
+  const toolCalls: string[] = [];
+
+  const getBaseText = () => {
+    const state = EXPLORATION_STATES[stateIndex];
+    return `${chalk.bold.hex('#D97706')(agentName)} ${state.color(state.text)}`;
+  };
+
   const spinner = ora({
-    text: `${chalk.bold.hex('#D97706')(agentName)} ${EXPLORATION_STATES[0].color(EXPLORATION_STATES[0].text)}`,
+    text: getBaseText(),
     spinner: {
       interval: 80,
       frames: ['◐', '◓', '◑', '◒'],
@@ -46,12 +58,37 @@ function createExplorationSpinner(agentName: string): { spinner: Ora; stop: () =
   // Cycle through states
   const interval = setInterval(() => {
     stateIndex = (stateIndex + 1) % EXPLORATION_STATES.length;
-    const state = EXPLORATION_STATES[stateIndex];
-    spinner.text = `${chalk.bold.hex('#D97706')(agentName)} ${state.color(state.text)}`;
+    const toolCount = toolCalls.length;
+    const lastTool = toolCalls[toolCalls.length - 1];
+    if (toolCount > 0 && lastTool) {
+      spinner.text = `${getBaseText()} ${chalk.dim(`(${toolCount} tools) ${lastTool}`)}`;
+    } else {
+      spinner.text = getBaseText();
+    }
   }, 2000);
 
+  // Update spinner to show tool call info (only actual tool calls, not text output)
+  const updateWithToolCall = (toolInfo: string) => {
+    if (!toolInfo.trim()) return;
+
+    // Only capture lines that are actual tool calls (formatted by claude-code.ts)
+    // Tool calls look like: "  › Read src/file.ts" or "  › Glob **/*.ts"
+    const lines = toolInfo.split('\n');
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (trimmed.startsWith('›')) {
+        // Strip ANSI codes for clean storage
+        const clean = trimmed.replace(/\x1b\[[0-9;]*m/g, '').substring(0, 80);
+        toolCalls.push(clean);
+        spinner.text = `${getBaseText()} ${chalk.dim(`(${toolCalls.length} tools) ${clean}`)}`;
+      }
+    }
+  };
+
   return {
     spinner,
+    toolCalls,
+    updateWithToolCall,
     stop: () => {
       clearInterval(interval);
       spinner.stop();
@@ -127,6 +164,10 @@ function saveBaselines(projectRoot: string, store: BaselineStore): void {
  * Create readline interface for user input
  */
 function createPrompt(): readline.Interface {
+  // Ensure stdin is flowing
+  if (process.stdin.isPaused()) {
+    process.stdin.resume();
+  }
   return readline.createInterface({
     input: process.stdin,
     output: process.stdout,
@@ -134,12 +175,47 @@ function createPrompt(): readline.Interface {
 }
 
 /**
- * Ask user a question and get response
+ * Check if readline is still usable
+ */
+function isReadlineOpen(rl: readline.Interface): boolean {
+  // @ts-ignore - accessing internal property to check state
+  return rl.terminal !== undefined && !rl.closed;
+}
+
+/** Default timeout for user input (5 minutes) */
+const USER_INPUT_TIMEOUT_MS = 5 * 60 * 1000;
+
+/**
+ * Ask user a question and get response with timeout
  */
-async function ask(rl: readline.Interface, question: string): Promise<string> {
-  return new Promise((resolve) => {
+async function ask(rl: readline.Interface, question: string, timeoutMs: number = USER_INPUT_TIMEOUT_MS): Promise<string> {
+  return new Promise((resolve, reject) => {
+    let answered = false;
+
+    const timeout = setTimeout(() => {
+      if (!answered) {
+        answered = true;
+        resolve(''); // Return empty on timeout
+      }
+    }, timeoutMs);
+
+    // Handle readline close (e.g., stdin EOF)
+    const onClose = () => {
+      if (!answered) {
+        answered = true;
+        clearTimeout(timeout);
+        resolve('');
+      }
+    };
+    rl.once('close', onClose);
+
     rl.question(question, (answer) => {
-      resolve(answer.trim());
+      if (!answered) {
+        answered = true;
+        clearTimeout(timeout);
+        rl.removeListener('close', onClose);
+        resolve(answer.trim());
+      }
     });
   });
 }
@@ -215,6 +291,7 @@ Be concise but accurate. Focus on the key points with specific file references w
 
 /**
  * Run a single interview question
+ * Returns the readline interface (may be recreated if stdin was disrupted)
  */
 async function runInterviewQuestion(
   caseData: Case,
@@ -222,7 +299,7 @@ async function runInterviewQuestion(
   rl: readline.Interface,
   store: BaselineStore,
   projectRoot: string
-): Promise<{ grade: number; skipped: boolean; durationMs?: number }> {
+): Promise<{ grade: number; skipped: boolean; durationMs?: number; rl: readline.Interface }> {
   const existingBaseline = store.baselines[caseData.id];
 
   // Show the question
@@ -233,47 +310,66 @@ async function runInterviewQuestion(
     const regrade = await ask(rl, chalk.cyan('  Re-run and re-grade? (y/N): '));
 
     if (regrade.toLowerCase() !== 'y') {
-      return { grade: existingBaseline.grade, skipped: true };
+      return { grade: existingBaseline.grade, skipped: true, rl };
     }
   }
 
-  // Get agent's response - stream output live with animated spinner
+  // Get agent's response - stream output live with animated spinner at bottom
   console.log('');
   const exploration = createExplorationSpinner(agent.displayName);
 
   let outputStarted = false;
   const startTime = Date.now();
 
+  let textOutputStarted = false;
+
   try {
     const result = await getAgentResponse(caseData, agent, projectRoot, (chunk) => {
-      // Stop spinner and show separator when first output arrives
-      if (!outputStarted) {
-        outputStarted = true;
-        exploration.stop();
-        console.log(chalk.dim('\n  ─────────────────────────────────────────\n'));
+      outputStarted = true;
+
+      // Check if this chunk contains tool calls (lines starting with ›)
+      // Tool calls are formatted like "  › Read src/file.ts"
+      const isToolCall = chunk.trim().startsWith('›');
+
+      if (isToolCall) {
+        // Extract tool call info and update spinner
+        const clean = chunk.trim().replace(/\x1b\[[0-9;]*m/g, '').substring(0, 80);
+        exploration.toolCalls.push(clean);
+        const state = EXPLORATION_STATES[0];
+        const baseText = `${chalk.bold.hex('#D97706')(agent.displayName)} ${state.color(state.text)}`;
+        exploration.spinner.text = `${baseText} ${chalk.dim(`(${exploration.toolCalls.length} tools)`)}`;
+      } else if (chunk.trim()) {
+        // This is text content - stop spinner and stream it
+        if (!textOutputStarted) {
+          textOutputStarted = true;
+          exploration.stop();
+          console.log(chalk.dim('\n  ─────────────────────────────────────────\n'));
+        }
+        process.stdout.write(chunk);
       }
-      process.stdout.write(chunk);
     });
 
     // Ensure spinner is stopped
-    exploration.stop();
+    if (!textOutputStarted) {
+      exploration.stop();
+    }
 
     const durationSec = ((Date.now() - startTime) / 1000).toFixed(1);
-    if (!outputStarted) {
-      console.log(chalk.dim('\n  ─────────────────────────────────────────'));
-    } else {
+    if (textOutputStarted) {
       console.log(chalk.dim('\n\n  ─────────────────────────────────────────'));
+    } else {
+      console.log(chalk.dim('\n  ─────────────────────────────────────────'));
     }
 
     if (result.timedOut) {
       console.log(chalk.yellow(`\n  ✗ ${agent.displayName} timed out after ${durationSec}s`));
       console.log(chalk.yellow('  The agent took too long. Consider increasing the timeout.'));
-      return { grade: 0, skipped: true, durationMs: result.durationMs };
+      return { grade: 0, skipped: true, durationMs: result.durationMs, rl };
     }
 
     if (!result.success) {
       console.log(chalk.red(`\n  ✗ ${agent.displayName} failed: ${result.error}`));
-      return { grade: 0, skipped: true, durationMs: result.durationMs };
+      return { grade: 0, skipped: true, durationMs: result.durationMs, rl };
     }
 
     console.log(chalk.green(`\n  ✓ ${agent.displayName} completed in ${durationSec}s`));
@@ -283,6 +379,17 @@ async function runInterviewQuestion(
       console.log(chalk.dim(`\n  Tools used: ${result.toolsUsed.join(', ')}`));
     }
 
+    // Recreate readline after agent run - stdin may have been disrupted
+    // by the spawned claude process
+    if (!isReadlineOpen(rl)) {
+      rl.close();
+      rl = createPrompt();
+    }
+    // Ensure stdin is flowing
+    if (process.stdin.isPaused()) {
+      process.stdin.resume();
+    }
+
     // Show grading scale and ask for grade
     showGradingScale();
     const grade = await askGrade(rl);
@@ -305,11 +412,11 @@ async function runInterviewQuestion(
 
     console.log(chalk.green(`\n  ✓ Baseline saved (${grade}/10)`));
 
-    return { grade, skipped: false, durationMs: result.durationMs };
+    return { grade, skipped: false, durationMs: result.durationMs, rl };
   } catch (err) {
     exploration.stop();
     console.log(chalk.red(`\n  ✗ Failed: ${(err as Error).message}`));
-    return { grade: 0, skipped: true };
+    return { grade: 0, skipped: true, rl };
   }
 }
 
@@ -386,8 +493,8 @@ export async function interviewCommand(options: InterviewOptions) {
     console.log(`  ${status}  ${chalk.bold(c.id)}: ${c.title}`);
   }
 
-  // Create prompt
-  const rl = createPrompt();
+  // Create prompt - may be recreated if stdin is disrupted during agent run
+  let rl = createPrompt();
 
   try {
     // Ask if user wants to continue
@@ -411,7 +518,9 @@ export async function interviewCommand(options: InterviewOptions) {
       console.log(chalk.dim(`  Difficulty: ${caseData.difficulty}\n`));
 
       const result = await runInterviewQuestion(caseData, agent, rl, store, projectRoot);
-      results.push({ caseId: caseData.id, ...result });
+      // Update rl in case it was recreated after agent run
+      rl = result.rl;
+      results.push({ caseId: caseData.id, grade: result.grade, skipped: result.skipped });
 
       if (i < cases.length - 1) {
         const next = await ask(rl, chalk.cyan('\n  Continue to next question? (Y/n/q to quit): '));