feat(cockpit): aimock E2E harness — Phase 1 (streaming pilot, replaces legacy cockpit e2e)

.github/workflows/ci.yml

@@ -161,9 +161,22 @@ jobs:
         with:
           node-version: 22
           cache: npm
+      - name: Install uv
+        uses: astral-sh/setup-uv@v8.0.0
+        with:
+          python-version: '3.12'
       - run: npm ci
+      - working-directory: cockpit/langgraph/streaming/python
+        run: uv sync
       - run: npx playwright install --with-deps chromium
       - run: npx nx e2e cockpit --skip-nx-cache
+      - name: Upload Playwright trace on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cockpit-e2e-trace
+          path: apps/cockpit/e2e/test-results/
+          retention-days: 7
 
   website-e2e:
     name: Website — e2e

apps/cockpit/e2e/.gitignore

@@ -0,0 +1,3 @@
+test-results/
+playwright-report/
+*.tmp

apps/cockpit/e2e/README.md

@@ -0,0 +1,33 @@
+# cockpit e2e
+
+Cross-stack E2E harness for cockpit example apps. Uses [`@copilotkit/aimock`](https://github.com/CopilotKit/aimock) as a deterministic mock for LLM API calls; the per-product Python LangGraph dev server is launched with `OPENAI_BASE_URL` pointed at it; Playwright drives the example Angular app in real Chromium.
+
+Phase 1 covers `c-messages` only. Future phases each add one example (one fixture + one spec file per PR).
+
+## Run the suite
+
+```
+npx nx e2e cockpit
+```
+
+Replay-only. No `OPENAI_API_KEY` needed. Reads committed fixtures from `fixtures/`.
+
+## Refresh a fixture
+
+Each captured fixture has a recipe script under `scripts/`. Example for the c-messages fixture:
+
+```
+OPENAI_API_KEY=sk-... uv run --project cockpit/langgraph/streaming/python \
+  python apps/cockpit/e2e/scripts/record-c-messages.py
+```
+
+Commit the updated `fixtures/c-messages.json`. Scripts are dev-only; CI never runs them.
+
+## Layout
+
+- `aimock-runner.ts` — programmatic boot of the mock server (mirrors `examples/chat/aimock-e2e/aimock-runner.ts`).
+- `test-helpers.ts` — `sendPromptAndWait` helper that waits on `chat-message[data-streaming="false"]`.
+- `fixtures/` — committed JSON fixtures keyed by example.
+- `scripts/` — fixture-capture recipes (one per fixture).
+- `playwright.config.ts` — Playwright config with globalSetup that boots aimock + LangGraph + Angular dev server.
+- `c-messages.spec.ts` — Phase 1 pilot.

apps/cockpit/e2e/aimock-runner.spec.ts

@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: MIT
+import { describe, it, expect, afterEach } from 'vitest';
+import { writeFileSync, mkdtempSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { startAimock, type AimockHandle } from './aimock-runner';
+
+describe('startAimock', () => {
+  let handle: AimockHandle | null = null;
+  let workDir = '';
+
+  afterEach(async () => {
+    if (handle) await handle.stop();
+    handle = null;
+    if (workDir) rmSync(workDir, { recursive: true, force: true });
+    workDir = '';
+  });
+
+  it('boots a replay server backed by a fixture file', async () => {
+    workDir = mkdtempSync(join(tmpdir(), 'aimock-test-'));
+    const fixturePath = join(workDir, 'hi.json');
+    writeFileSync(
+      fixturePath,
+      JSON.stringify({
+        fixtures: [
+          { match: { userMessage: 'say hi briefly' }, response: { content: 'Hi!' } },
+        ],
+      }),
+    );
+
+    handle = await startAimock({ mode: 'replay', fixturePath });
+    expect(handle.port).toBeGreaterThan(0);
+    expect(handle.baseUrl).toMatch(/^http:\/\/.+\/v1$/);
+
+    // The OpenAI SDK call path is exercised in Task 0's de-risk; this
+    // unit test stops at "the harness started cleanly and exposes the
+    // documented shape."
+  });
+
+  it('stop() is idempotent', async () => {
+    workDir = mkdtempSync(join(tmpdir(), 'aimock-test-'));
+    const fixturePath = join(workDir, 'hi.json');
+    writeFileSync(fixturePath, JSON.stringify({ fixtures: [] }));
+    handle = await startAimock({ mode: 'replay', fixturePath });
+    await handle.stop();
+    await handle.stop();
+    expect(true).toBe(true);
+  });
+
+  it('loads and merges all .json files in a directory', async () => {
+    workDir = mkdtempSync(join(tmpdir(), 'aimock-test-'));
+    writeFileSync(
+      join(workDir, 'a.json'),
+      JSON.stringify({
+        fixtures: [{ match: { userMessage: 'one' }, response: { content: 'A' } }],
+      }),
+    );
+    writeFileSync(
+      join(workDir, 'b.json'),
+      JSON.stringify({
+        fixtures: [{ match: { userMessage: 'two' }, response: { content: 'B' } }],
+      }),
+    );
+    // Non-JSON file in the dir should be ignored.
+    writeFileSync(join(workDir, 'README.md'), '# not a fixture');
+
+    handle = await startAimock({ mode: 'replay', fixturePath: workDir });
+    expect(handle.port).toBeGreaterThan(0);
+    expect(handle.baseUrl).toMatch(/^http:\/\/.+\/v1$/);
+  });
+});

apps/cockpit/e2e/aimock-runner.ts

@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: MIT
+import { LLMock } from '@copilotkit/aimock';
+import { readFileSync, readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+
+export interface AimockHandle {
+  /** Port the mock server is listening on. */
+  readonly port: number;
+  /** Full base URL the OpenAI SDK should target (includes /v1 suffix). */
+  readonly baseUrl: string;
+  /** Tear down the server. Safe to call multiple times. */
+  stop(): Promise<void>;
+}
+
+export interface AimockStartOptions {
+  mode: 'replay';
+  /** Path to a single fixture file OR a directory of fixture files. */
+  fixturePath: string;
+}
+
+// Raw JSON entry shape passes through to aimock's FixtureFileEntry — the
+// `match` block can carry richer discriminators (toolName, hasToolResult,
+// turnIndex, etc.) that are needed to distinguish a parent LLM's first call
+// from its continuation after a tool round. We don't narrow the shape here;
+// aimock's `addFixturesFromJSON` validates structure at load time.
+type FixtureFileEntry = Record<string, unknown>;
+
+function loadFixtureEntries(fixturePath: string): FixtureFileEntry[] {
+  const stats = statSync(fixturePath);
+  const out: FixtureFileEntry[] = [];
+  const readFile = (full: string): void => {
+    const raw = readFileSync(full, 'utf-8');
+    const parsed = JSON.parse(raw) as { fixtures: FixtureFileEntry[] };
+    for (const fx of parsed.fixtures) out.push(fx);
+  };
+  if (stats.isDirectory()) {
+    const files = readdirSync(fixturePath)
+      .filter((f) => f.endsWith('.json'))
+      .sort();
+    for (const file of files) readFile(join(fixturePath, file));
+    return out;
+  }
+  readFile(fixturePath);
+  return out;
+}
+
+export async function startAimock(opts: AimockStartOptions): Promise<AimockHandle> {
+  const entries = loadFixtureEntries(opts.fixturePath);
+
+  // Use a large chunkSize so each response arrives in 1-2 SSE deltas. This
+  // intentionally turns off the partial-markdown streaming path for harness
+  // tests: structural assertions (code fence, list) measure the FINAL rendered
+  // DOM, not the progressive render. With aggressive default chunking, the
+  // partial-markdown parser sometimes can't recover a triple-backtick fence
+  // that gets split mid-token, and the final state ends up as inline <code>
+  // instead of <pre><code>. Streaming-progressive behavior is covered by the
+  // Phase 1 unit-variance tables; the e2e harness is for final-state
+  // invariants and cross-stack integration.
+  const mock = new LLMock({ port: 0, chunkSize: 4096 });
+  if (entries.length > 0) {
+    mock.addFixturesFromJSON(entries as never);
+  }
+  await mock.start();
+
+  const port = mock.port;
+  const baseUrl = `${mock.url}/v1`;
+  let stopped = false;
+
+  return {
+    port,
+    baseUrl,
+    async stop() {
+      if (stopped) return;
+      stopped = true;
+      await mock.stop();
+    },
+  };
+}

apps/cockpit/e2e/fixtures/streaming.json

@@ -0,0 +1,12 @@
+{
+  "fixtures": [
+    {
+      "match": {
+        "userMessage": "Tell me one quick fact about Angular signals in two sentences."
+      },
+      "response": {
+        "content": "Angular signals are a reactive primitive (signal, computed, effect) that track dependencies to provide fine-grained reactivity and more efficient change detection. They let you update state synchronously via set()/update() and ensure only consumers that read an affected signal are re\u2011evaluated."
+      }
+    }
+  ]
+}