Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions packages/gittensory-engine/src/miner/attempt-log.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ export const ATTEMPT_LOG_EVENT_TYPES = Object.freeze([
"attempt_succeeded",
"attempt_failed",
"attempt_aborted",
// #5185: one summary row per completed attempt, written by the miner CLI (attempt-cli.js) once `runIterateLoop`
// returns -- distinct from the five iteration-level types above (all written from inside iterate-loop.ts's own
// per-iteration decision trail). Carries provider/costUsd, the two real (never-fabricated) signals a
// per-provider usage dashboard needs that no iteration-level event captures today.
"attempt_outcome_summary",
] as const);

export type AttemptLogEventType = (typeof ATTEMPT_LOG_EVENT_TYPES)[number];
Expand All @@ -22,6 +27,15 @@ export type AttemptLogEvent = {
mode: CodingAgentExecutionMode;
reason: string;
payload?: Record<string, unknown> | undefined;
/** Coding-agent provider name (claude-cli/codex-cli/agent-sdk/noop) this attempt used, when known (#5185).
* Optional: every event type that predates this field omits it; only `attempt_outcome_summary` sets it. */
provider?: string | undefined;
/** Real dollar cost, mirroring `CodingAgentDriverResult.costUsd`'s own convention: absent (not zero) when the
* provider never reports a cost signal, never fabricated (#5185). */
costUsd?: number | undefined;
/** Real token count, when some future driver reports one. Always absent today -- no driver reports real token
* usage yet (#5395) -- an honest gap, not a fabricated zero. */
tokensUsed?: number | undefined;
};

export type NormalizedAttemptLogEvent = {
Expand All @@ -31,6 +45,9 @@ export type NormalizedAttemptLogEvent = {
mode: CodingAgentExecutionMode;
reason: string;
payloadJson: string;
provider: string | null;
costUsd: number | null;
tokensUsed: number | null;
};

const attemptEventTypeSet = new Set<string>(ATTEMPT_LOG_EVENT_TYPES);
Expand Down Expand Up @@ -84,6 +101,21 @@ function normalizeMode(value: unknown): CodingAgentExecutionMode {
return mode as CodingAgentExecutionMode;
}

/** `undefined` -> `null` ("not set"); any other non-empty-string value must be a valid string, or this fails
* closed rather than silently coercing (#5185). */
function normalizeOptionalString(value: unknown, code: string): string | null {
if (value === undefined) return null;
return normalizeRequiredString(value, code);
}

/** `undefined` -> `null` ("no signal, never fabricated as 0"); any other value must be a finite number >= 0
* (#5185) -- mirrors `CodingAgentDriverResult.costUsd`'s own absent-vs-zero distinction. */
function normalizeOptionalNonNegativeNumber(value: unknown, code: string): number | null {
if (value === undefined) return null;
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new Error(code);
return value;
}

/** Validate and normalize an attempt-log row before append. Fail-closed on unknown types/modes. */
export function normalizeAttemptLogEvent(input: unknown): NormalizedAttemptLogEvent {
if (!input || typeof input !== "object") throw new Error("invalid_event");
Expand All @@ -97,6 +129,9 @@ export function normalizeAttemptLogEvent(input: unknown): NormalizedAttemptLogEv
mode: normalizeMode(event.mode),
reason: normalizeRequiredString(event.reason, "invalid_reason"),
payloadJson: serializePayload(event.payload),
provider: normalizeOptionalString(event.provider, "invalid_provider"),
costUsd: normalizeOptionalNonNegativeNumber(event.costUsd, "invalid_cost_usd"),
tokensUsed: normalizeOptionalNonNegativeNumber(event.tokensUsed, "invalid_tokens_used"),
};
}

Expand Down
43 changes: 43 additions & 0 deletions packages/gittensory-engine/test/attempt-log.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,52 @@ test("ATTEMPT_LOG_EVENT_TYPES is a fixed vocabulary", () => {
"attempt_succeeded",
"attempt_failed",
"attempt_aborted",
"attempt_outcome_summary",
]);
});

test("normalizeAttemptLogEvent leaves provider/costUsd/tokensUsed null when omitted, and passes through real values", () => {
const omitted = normalizeAttemptLogEvent({
eventType: "attempt_started",
attemptId: "a-1",
actionClass: "codegen",
mode: "live",
reason: "live run",
});
assert.equal(omitted.provider, null);
assert.equal(omitted.costUsd, null);
assert.equal(omitted.tokensUsed, null);

const withValues = normalizeAttemptLogEvent({
eventType: "attempt_outcome_summary",
attemptId: "a-1",
actionClass: "attempt_submitted",
mode: "live",
reason: "attempt finished",
provider: "claude-cli",
costUsd: 0.42,
tokensUsed: 1000,
});
assert.equal(withValues.provider, "claude-cli");
assert.equal(withValues.costUsd, 0.42);
assert.equal(withValues.tokensUsed, 1000);
});

test("normalizeAttemptLogEvent rejects a negative/non-finite costUsd or tokensUsed, never coercing to 0", () => {
const base = {
eventType: "attempt_outcome_summary",
attemptId: "a-1",
actionClass: "attempt_submitted",
mode: "live",
reason: "attempt finished",
};
assert.throws(() => normalizeAttemptLogEvent({ ...base, costUsd: -1 }), /invalid_cost_usd/);
assert.throws(() => normalizeAttemptLogEvent({ ...base, costUsd: Number.NaN }), /invalid_cost_usd/);
assert.throws(() => normalizeAttemptLogEvent({ ...base, costUsd: "0.5" }), /invalid_cost_usd/);
assert.throws(() => normalizeAttemptLogEvent({ ...base, tokensUsed: -1 }), /invalid_tokens_used/);
assert.throws(() => normalizeAttemptLogEvent({ ...base, provider: "" }), /invalid_provider/);
});

test("normalizeAttemptLogEvent validates mode and payload round-trip", () => {
const normalized = normalizeAttemptLogEvent({
eventType: "attempt_shadow",
Expand Down
4 changes: 3 additions & 1 deletion packages/gittensory-miner/docs/observability.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ The miner writes append-only SQLite ledgers under `GITTENSORY_MINER_CONFIG_DIR`
[`DEPLOYMENT.md`](../DEPLOYMENT.md)):

- **`attempt-log.sqlite3`** — the driver-level attempt event trace (event type, action class, mode, reason,
timestamps), table `attempt_log_events`.
timestamps), table `attempt_log_events`. One `attempt_outcome_summary` row per completed attempt also carries
the real configured `provider` and the real accumulated `cost_usd` (#5185) — `tokens_used` is always `NULL`
today, an honest gap rather than a fabricated `0`: no coding-agent driver reports real token usage yet (#5395).
- **`prediction-ledger.sqlite3`** — recorded predicted-gate verdicts for later scoring.

Those live files can contain free-form payloads, repo/target identifiers, readiness scores, and blocker/warning
Expand Down
24 changes: 23 additions & 1 deletion packages/gittensory-miner/lib/attempt-cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// governor.convergenceInput is an honest first-attempt-shaped literal, not a real per-issue attempt-history
// query (attempt-log.js's schema has no repo+issue index, and reenqueue counts aren't tracked anywhere yet).

import { resolveCodingAgentModeFromConfig } from "@loopover/engine";
import { resolveCodingAgentModeFromConfig, resolveFirstConfiguredCodingAgentDriverName } from "@loopover/engine";
import { argsWantJson, describeCliError, reportCliFailure } from "./cli-error.js";
import { constructProductionCodingAgentDriver } from "./coding-agent-construction.js";
import { runSlopAssessment } from "./slop-assessment.js";
Expand Down Expand Up @@ -477,6 +477,28 @@ export async function runAttempt(args, options = {}) {
...(claimConflict !== undefined ? { claimConflict } : {}),
};

// One summary row per completed attempt (#5185), for the Grafana per-provider usage dashboard the redacted
// AMS reporting export exposes -- distinct from the per-iteration attempt_started/attempt_tool_edit/... trail
// iterate-loop.ts already writes. No fallback for an unconfigured provider: buildAttemptDeps already fails
// closed (throws) on the same env before a worktree is even allocated, so reaching this point guarantees
// resolveFirstConfiguredCodingAgentDriverName(env) resolves a real name. tokensUsed is deliberately omitted
// (normalizes to null): no driver reports real token usage today (#5395), and null-for-"no signal" is more
// honest here than a fabricated 0. A logging failure must never fail an otherwise-successful attempt --
// mirrors iterate-loop.ts's own safeAppendAttemptLogEvent non-fatal handling.
try {
attemptLog.appendAttemptLogEvent({
eventType: "attempt_outcome_summary",
attemptId,
actionClass: finalResult.outcome,
mode,
reason: `attempt finished with outcome: ${result.outcome}`,
provider: resolveFirstConfiguredCodingAgentDriverName(env),
costUsd: finalResult.totalCostUsd,
});
} catch {
// Deliberately swallowed -- see comment above.
}

if (parsed.json) {
console.log(JSON.stringify(finalResult, null, 2));
} else {
Expand Down
8 changes: 8 additions & 0 deletions packages/gittensory-miner/lib/attempt-log.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,14 @@ export type AttemptLogEntry = {
mode: string;
reason: string;
payload: Record<string, unknown>;
/** Coding-agent provider name, when the event set one (#5185). Null for every event type that predates this
* field. */
provider: string | null;
/** Real dollar cost, when the event set one (#5185). Null (not 0) when absent -- never fabricated. */
costUsd: number | null;
/** Real token count, when some future driver reports one (#5185). Always null today -- no driver reports real
* token usage yet (#5395). */
tokensUsed: number | null;
createdAt: string;
};

Expand Down
34 changes: 32 additions & 2 deletions packages/gittensory-miner/lib/attempt-log.js
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ function rowToEntry(row) {
mode: row.mode,
reason: row.reason,
payload,
provider: row.provider,
costUsd: row.cost_usd,
tokensUsed: row.tokens_used,
createdAt: row.created_at,
};
}
Expand All @@ -69,9 +72,31 @@ function rowToNormalized(row) {
mode: row.mode,
reason: row.reason,
payloadJson: row.payload_json,
provider: row.provider,
costUsd: row.cost_usd,
tokensUsed: row.tokens_used,
};
}

// Add the provider/cost_usd/tokens_used columns (#5185) to an on-disk file created before they existed. `CREATE
// TABLE IF NOT EXISTS` above is a no-op against an already-existing table, so a pre-#5185 file needs this
// explicit ALTER -- guarded by a per-column presence check (same technique as governor-state.js's own
// ensurePauseColumns) so a file missing only one of the three still gets exactly what it's missing.
function ensureOutcomeColumns(db) {
const existingColumns = new Set(
db.prepare("PRAGMA table_info(attempt_log_events)").all().map((column) => column.name),
);
if (!existingColumns.has("provider")) {
db.exec("ALTER TABLE attempt_log_events ADD COLUMN provider TEXT");
}
if (!existingColumns.has("cost_usd")) {
db.exec("ALTER TABLE attempt_log_events ADD COLUMN cost_usd REAL");
}
if (!existingColumns.has("tokens_used")) {
db.exec("ALTER TABLE attempt_log_events ADD COLUMN tokens_used INTEGER");
}
}

/**
* Opens the append-only attempt log, creating the table on first use. `seq` is a monotonically increasing counter
* maintained by this module (next = current MAX(seq) + 1) with a UNIQUE(seq) constraint. Rows read back in seq ASC
Expand All @@ -93,16 +118,18 @@ export function initAttemptLog(dbPath = resolveAttemptLogDbPath()) {
created_at TEXT NOT NULL
)
`);
ensureOutcomeColumns(db);
db.exec(
"CREATE INDEX IF NOT EXISTS idx_attempt_log_attempt ON attempt_log_events (attempt_id, seq)",
);

const nextSeqStatement = db.prepare("SELECT COALESCE(MAX(seq), 0) + 1 AS nextSeq FROM attempt_log_events");
const appendStatement = db.prepare(`
INSERT INTO attempt_log_events (
seq, attempt_id, event_type, action_class, mode, reason, payload_json, created_at
seq, attempt_id, event_type, action_class, mode, reason, payload_json, provider, cost_usd, tokens_used,
created_at
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
const getByIdStatement = db.prepare("SELECT * FROM attempt_log_events WHERE id = ?");
const readAllStatement = db.prepare("SELECT * FROM attempt_log_events ORDER BY seq ASC");
Expand All @@ -126,6 +153,9 @@ export function initAttemptLog(dbPath = resolveAttemptLogDbPath()) {
normalized.mode,
normalized.reason,
normalized.payloadJson,
normalized.provider,
normalized.costUsd,
normalized.tokensUsed,
createdAt,
);
const entry = rowToEntry(getByIdStatement.get(Number(result.lastInsertRowid)));
Expand Down
9 changes: 7 additions & 2 deletions scripts/export-ams-reporting-db.sh
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,9 @@ DETACH report;

# attempt_log_events: DROP `reason` and `payload_json` -- both free-form (payload_json in particular can nest
# arbitrary per-event-type detail, up to and including file paths/diffs/prompt fragments). Every other column is
# a bounded-vocabulary identifier/enum/timestamp, safe for a shared reporting export.
# a bounded-vocabulary identifier/enum/timestamp, safe for a shared reporting export -- including provider/
# cost_usd/tokens_used (#5185, added by attempt-cli.js's own attempt_outcome_summary event), each a real
# structured value (provider name, a dollar figure, a token count), never free text.
export_ledger \
"attempt-log" \
"$ATTEMPT_LOG_SOURCE_DB" \
Expand All @@ -152,11 +154,14 @@ export_ledger \
event_type TEXT NOT NULL,
action_class TEXT NOT NULL,
mode TEXT NOT NULL,
provider TEXT,
cost_usd REAL,
tokens_used INTEGER,
created_at TEXT NOT NULL
);
CREATE INDEX attempt_log_events_attempt_idx ON attempt_log_events(attempt_id, seq);
CREATE INDEX attempt_log_events_created_idx ON attempt_log_events(created_at);" \
"id, seq, attempt_id, event_type, action_class, mode, created_at"
"id, seq, attempt_id, event_type, action_class, mode, provider, cost_usd, tokens_used, created_at"

# predictions: kept as-is. Unlike attempt_log_events, every column here is already a bounded identifier, enum,
# score, or a fixed-vocabulary code array (blocker_codes_json/warning_codes_json -- engine-defined codes, never
Expand Down
43 changes: 43 additions & 0 deletions test/unit/coding-agent-miner.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -116,10 +116,53 @@ describe("attempt log normalization (#4294)", () => {
"attempt_succeeded",
"attempt_failed",
"attempt_aborted",
"attempt_outcome_summary",
]);
expect(Object.isFrozen(ATTEMPT_LOG_EVENT_TYPES)).toBe(true);
});

it("leaves provider/costUsd/tokensUsed null when omitted, real values when set (#5185)", () => {
const omitted = normalizeAttemptLogEvent({
eventType: "attempt_started",
attemptId: "a-1",
actionClass: "codegen",
mode: "live",
reason: "live run",
});
expect(omitted.provider).toBeNull();
expect(omitted.costUsd).toBeNull();
expect(omitted.tokensUsed).toBeNull();

const withValues = normalizeAttemptLogEvent({
eventType: "attempt_outcome_summary",
attemptId: "a-1",
actionClass: "attempt_submitted",
mode: "live",
reason: "attempt finished",
provider: "codex-cli",
costUsd: 0,
tokensUsed: 0,
});
expect(withValues.provider).toBe("codex-cli");
expect(withValues.costUsd).toBe(0);
expect(withValues.tokensUsed).toBe(0);
});

it("rejects a negative/non-finite/non-numeric costUsd or tokensUsed, and a blank provider (#5185)", () => {
const base = {
eventType: "attempt_outcome_summary",
attemptId: "a-1",
actionClass: "attempt_submitted",
mode: "live",
reason: "attempt finished",
};
expect(() => normalizeAttemptLogEvent({ ...base, costUsd: -1 })).toThrow(/invalid_cost_usd/);
expect(() => normalizeAttemptLogEvent({ ...base, costUsd: Number.NaN })).toThrow(/invalid_cost_usd/);
expect(() => normalizeAttemptLogEvent({ ...base, costUsd: "0.5" } as unknown)).toThrow(/invalid_cost_usd/);
expect(() => normalizeAttemptLogEvent({ ...base, tokensUsed: -1 })).toThrow(/invalid_tokens_used/);
expect(() => normalizeAttemptLogEvent({ ...base, provider: " " })).toThrow(/invalid_provider/);
});

it("normalizes a valid event with payload round-trip", () => {
const normalized = normalizeAttemptLogEvent({
eventType: "attempt_shadow",
Expand Down
Loading
Loading