Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions packages/gittensory-miner/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,30 @@ Additive only: the existing `rows` JSON key and PR table are unchanged; `runPort
after the existing table. A real GUI dashboard surface is out of scope here — `apps/gittensory-miner-ui/` is
Phase 6 of the same roadmap tracker and hasn't been scaffolded yet. (#4279)

## Local storage

Four independent local SQLite stores back the commands above. Each keeps its own file, its own table, and its own
env-var override — this is a DRY pass over their shared path-resolution/open boilerplate (`local-store.js`), not a
merge into one database. (#4272)

| Store | File | Table | Module | Env var override |
| --- | --- | --- | --- | --- |
| Run state | `run-state.sqlite3` | `miner_run_state` | `run-state.js` | `GITTENSORY_MINER_RUN_STATE_DB` |
| Claim ledger | `claim-ledger.sqlite3` | `miner_claims` | `claim-ledger.js` | `GITTENSORY_MINER_CLAIM_LEDGER_DB` |
| Portfolio queue | `portfolio-queue.sqlite3` | `miner_portfolio_queue` | `portfolio-queue.js` | `GITTENSORY_MINER_PORTFOLIO_QUEUE_DB` |
| Event ledger | `event-ledger.sqlite3` | `miner_event_ledger` | `event-ledger.js` | `GITTENSORY_MINER_EVENT_LEDGER_DB` |

Every store resolves its file the same way: the store-specific env var above, else `GITTENSORY_MINER_CONFIG_DIR`,
else `XDG_CONFIG_HOME` (falling back to `~/.config`), joined with `gittensory-miner/<file>`. Every store also opens
its file with `0700`/`0600` permissions and a shared `PRAGMA busy_timeout` so two instances on the same file
serialize writes instead of racing.

The "PR portfolio" `manage status` renders is currently a **read-time join**, not a dedicated table:
`collectManageStatus` reads `portfolio-queue.js` rows (via the `pr:{number}` identifier convention) and joins them
against `event-ledger.js`'s free-form `manage_pr_update` JSON events at query time, on every read. Decision: keep
this as a read-time join for now; revisit a dedicated indexed table only if/when PR-portfolio reads become frequent
enough (e.g. a live-polling dashboard) that the per-read linear event-ledger scan becomes a measured bottleneck.

## Install

See [`docs/miner-goal-spec.md`](docs/miner-goal-spec.md) for the `.gittensory-miner.yml` field reference and [`.gittensory-miner.yml.example`](../../.gittensory-miner.yml.example) at the repo root.
Expand Down
29 changes: 4 additions & 25 deletions packages/gittensory-miner/lib/claim-ledger.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
import { chmodSync, mkdirSync } from "node:fs";
import { homedir } from "node:os";
import { dirname, join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } from "./local-store.js";

// The miner's local soft-claim ledger (#2314): a 100% client-side record of "I'm working on issue #N in repo X",
// so Phase 2's soft-claim adjudication (sibling issues) has somewhere to persist claims. Schema + CRUD only — no
Expand All @@ -15,26 +12,11 @@ const defaultDbFileName = "claim-ledger.sqlite3";
let defaultClaimLedger = null;

export function resolveClaimLedgerDbPath(env = process.env) {
const explicitPath = typeof env.GITTENSORY_MINER_CLAIM_LEDGER_DB === "string"
? env.GITTENSORY_MINER_CLAIM_LEDGER_DB.trim()
: "";
if (explicitPath) return explicitPath;

const explicitConfigDir = typeof env.GITTENSORY_MINER_CONFIG_DIR === "string"
? env.GITTENSORY_MINER_CONFIG_DIR.trim()
: "";
if (explicitConfigDir) return join(explicitConfigDir, defaultDbFileName);

const configHome = typeof env.XDG_CONFIG_HOME === "string" && env.XDG_CONFIG_HOME.trim()
? env.XDG_CONFIG_HOME.trim()
: join(homedir(), ".config");
return join(configHome, "gittensory-miner", defaultDbFileName);
return resolveLocalStoreDbPath(defaultDbFileName, "GITTENSORY_MINER_CLAIM_LEDGER_DB", env);
}

function normalizeDbPath(dbPath) {
const path = (dbPath ?? resolveClaimLedgerDbPath()).trim();
if (!path) throw new Error("invalid_claim_ledger_db_path");
return path;
return normalizeLocalStoreDbPath(dbPath, resolveClaimLedgerDbPath(), "invalid_claim_ledger_db_path");
}

function normalizeRepoFullName(repoFullName) {
Expand Down Expand Up @@ -74,10 +56,7 @@ function rowToClaim(row) {
*/
export function openClaimLedger(dbPath = resolveClaimLedgerDbPath()) {
const resolvedPath = normalizeDbPath(dbPath);
mkdirSync(dirname(resolvedPath), { recursive: true, mode: 0o700 });
const db = new DatabaseSync(resolvedPath);
chmodSync(resolvedPath, 0o600);
db.exec("PRAGMA busy_timeout = 5000");
const db = openLocalStoreDb(resolvedPath);
// LOCAL bookkeeping only: this table records which issues this miner instance has soft-claimed on this
// machine. It does NOT adjudicate contested duplicates — sibling miners claiming the same issue are
// resolved elsewhere via `isDuplicateClusterWinnerByClaim` from `@jsonbored/gittensory-engine` (#3355).
Expand Down
30 changes: 4 additions & 26 deletions packages/gittensory-miner/lib/event-ledger.js
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
import { chmodSync, mkdirSync } from "node:fs";
import { homedir } from "node:os";
import { dirname, join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { isDeepStrictEqual } from "node:util";
import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } from "./local-store.js";

// The miner's local, append-only event ledger (#2290): an immutable audit trail of every significant miner-loop
// event (discovered_issue, plan_built, plan_step_completed, pr_prepared, … — a small fixed vocabulary for this
Expand All @@ -16,26 +13,11 @@ const defaultDbFileName = "event-ledger.sqlite3";
let defaultEventLedger = null;

export function resolveEventLedgerDbPath(env = process.env) {
const explicitPath = typeof env.GITTENSORY_MINER_EVENT_LEDGER_DB === "string"
? env.GITTENSORY_MINER_EVENT_LEDGER_DB.trim()
: "";
if (explicitPath) return explicitPath;

const explicitConfigDir = typeof env.GITTENSORY_MINER_CONFIG_DIR === "string"
? env.GITTENSORY_MINER_CONFIG_DIR.trim()
: "";
if (explicitConfigDir) return join(explicitConfigDir, defaultDbFileName);

const configHome = typeof env.XDG_CONFIG_HOME === "string" && env.XDG_CONFIG_HOME.trim()
? env.XDG_CONFIG_HOME.trim()
: join(homedir(), ".config");
return join(configHome, "gittensory-miner", defaultDbFileName);
return resolveLocalStoreDbPath(defaultDbFileName, "GITTENSORY_MINER_EVENT_LEDGER_DB", env);
}

function normalizeDbPath(dbPath) {
const path = (dbPath ?? resolveEventLedgerDbPath()).trim();
if (!path) throw new Error("invalid_event_ledger_db_path");
return path;
return normalizeLocalStoreDbPath(dbPath, resolveEventLedgerDbPath(), "invalid_event_ledger_db_path");
}

function normalizeEventType(type) {
Expand Down Expand Up @@ -108,11 +90,7 @@ function rowToEntry(row) {
*/
export function initEventLedger(dbPath = resolveEventLedgerDbPath()) {
const resolvedPath = normalizeDbPath(dbPath);
mkdirSync(dirname(resolvedPath), { recursive: true, mode: 0o700 });
const db = new DatabaseSync(resolvedPath);
chmodSync(resolvedPath, 0o600);
// Wait (rather than fail) for a concurrent writer's lock so two ledger instances on the same file serialize.
db.exec("PRAGMA busy_timeout = 5000");
const db = openLocalStoreDb(resolvedPath);
// `UNIQUE(seq)` makes the monotonic-ordering guarantee an enforced invariant: a duplicate seq can never persist,
// even if the append path were ever changed.
db.exec(`
Expand Down
18 changes: 18 additions & 0 deletions packages/gittensory-miner/lib/local-store.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import type { DatabaseSync } from "node:sqlite";

export function resolveLocalStoreDbPath(
defaultDbFileName: string,
explicitEnvVarName: string,
env?: Record<string, string | undefined>,
): string;

export function normalizeLocalStoreDbPath(
dbPath: string | null | undefined,
resolvedDefault: string,
invalidPathError: string,
): string;

export function openLocalStoreDb(
resolvedPath: string,
options?: { busyTimeoutMs?: number },
): DatabaseSync;
52 changes: 52 additions & 0 deletions packages/gittensory-miner/lib/local-store.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
import { chmodSync, mkdirSync } from "node:fs";
import { homedir } from "node:os";
import { dirname, join } from "node:path";
import { DatabaseSync } from "node:sqlite";

// Shared path-resolution + DB-open boilerplate for the package's local SQLite stores (#4272). This is a DRY pass
// only, not a merge: run-state.js, claim-ledger.js, portfolio-queue.js, and event-ledger.js each keep their own
// `.sqlite3` file, table, and env var — this module just extracts the ~15 lines each hand-duplicated
// (env-var/config-dir/XDG path resolution, mkdirSync(0o700) + chmodSync(0o600), and `PRAGMA busy_timeout`).

/**
* Resolve a local store's DB path from, in order: an explicit env var, `GITTENSORY_MINER_CONFIG_DIR`,
* `XDG_CONFIG_HOME` (falling back to `~/.config`) — mirroring every store's prior hand-written resolver.
*/
export function resolveLocalStoreDbPath(defaultDbFileName, explicitEnvVarName, env = process.env) {
const explicitPath = typeof env[explicitEnvVarName] === "string" ? env[explicitEnvVarName].trim() : "";
if (explicitPath) return explicitPath;

const explicitConfigDir = typeof env.GITTENSORY_MINER_CONFIG_DIR === "string"
? env.GITTENSORY_MINER_CONFIG_DIR.trim()
: "";
if (explicitConfigDir) return join(explicitConfigDir, defaultDbFileName);

const configHome = typeof env.XDG_CONFIG_HOME === "string" && env.XDG_CONFIG_HOME.trim()
? env.XDG_CONFIG_HOME.trim()
: join(homedir(), ".config");
return join(configHome, "gittensory-miner", defaultDbFileName);
}

/** Trim and validate a caller-supplied (or resolved-default) DB path, throwing `invalidPathError` if it is empty. */
export function normalizeLocalStoreDbPath(dbPath, resolvedDefault, invalidPathError) {
const raw = dbPath ?? resolvedDefault;
if (typeof raw !== "string" || !raw.trim()) throw new Error(invalidPathError);
return raw.trim();
}

/**
* Open (creating parent dirs on first use) a local store's SQLite file with 0700/0600 permissions and a shared
* busy-timeout, so two instances of the same store on one file serialize writes instead of racing. Skips the
* mkdir/chmod steps for the special `:memory:` path, which has no on-disk file. `run-state.js` previously opened
* its DB with no busy-timeout at all (the one inconsistency among the four stores this issue found); folding it
* through this shared helper gives it the same wait-don't-fail behavior the other three already had.
*/
export function openLocalStoreDb(resolvedPath, options = {}) {
const busyTimeoutMs = options.busyTimeoutMs ?? 5000;
const isMemory = resolvedPath === ":memory:";
if (!isMemory) mkdirSync(dirname(resolvedPath), { recursive: true, mode: 0o700 });
const db = new DatabaseSync(resolvedPath);
if (!isMemory) chmodSync(resolvedPath, 0o600);
db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}`);
return db;
}
34 changes: 5 additions & 29 deletions packages/gittensory-miner/lib/portfolio-queue.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
import { chmodSync, mkdirSync } from "node:fs";
import { homedir } from "node:os";
import { dirname, join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } from "./local-store.js";

// The miner's local portfolio/queue store (#2292): a 100% client-side, prioritized backlog of candidate work
// items across every repo the miner has been pointed at ("what should I look at next, across everything I'm
Expand All @@ -15,26 +12,11 @@ const defaultDbFileName = "portfolio-queue.sqlite3";
let defaultPortfolioQueueStore = null;

export function resolvePortfolioQueueDbPath(env = process.env) {
const explicitPath = typeof env.GITTENSORY_MINER_PORTFOLIO_QUEUE_DB === "string"
? env.GITTENSORY_MINER_PORTFOLIO_QUEUE_DB.trim()
: "";
if (explicitPath) return explicitPath;

const explicitConfigDir = typeof env.GITTENSORY_MINER_CONFIG_DIR === "string"
? env.GITTENSORY_MINER_CONFIG_DIR.trim()
: "";
if (explicitConfigDir) return join(explicitConfigDir, defaultDbFileName);

const configHome = typeof env.XDG_CONFIG_HOME === "string" && env.XDG_CONFIG_HOME.trim()
? env.XDG_CONFIG_HOME.trim()
: join(homedir(), ".config");
return join(configHome, "gittensory-miner", defaultDbFileName);
return resolveLocalStoreDbPath(defaultDbFileName, "GITTENSORY_MINER_PORTFOLIO_QUEUE_DB", env);
}

function normalizeDbPath(dbPath) {
const raw = dbPath ?? resolvePortfolioQueueDbPath();
if (typeof raw !== "string" || !raw.trim()) throw new Error("invalid_portfolio_queue_db_path");
return raw.trim();
return normalizeLocalStoreDbPath(dbPath, resolvePortfolioQueueDbPath(), "invalid_portfolio_queue_db_path");
}

function normalizeRepoFullName(repoFullName) {
Expand Down Expand Up @@ -78,14 +60,8 @@ function rowToEntry(row) {
*/
export function initPortfolioQueueStore(dbPath = resolvePortfolioQueueDbPath()) {
const resolvedPath = normalizeDbPath(dbPath);
// The store is a persistent local file; the special in-memory path (':memory:') has no file to create or chmod.
if (resolvedPath !== ":memory:") {
mkdirSync(dirname(resolvedPath), { recursive: true, mode: 0o700 });
}
const db = new DatabaseSync(resolvedPath);
if (resolvedPath !== ":memory:") chmodSync(resolvedPath, 0o600);
// Wait (rather than fail) for a concurrent writer's lock so two queue instances on the same file serialize.
db.exec("PRAGMA busy_timeout = 5000");
// openLocalStoreDb skips mkdir/chmod for the special in-memory path (':memory:'), which has no file on disk.
const db = openLocalStoreDb(resolvedPath);
db.exec(`
CREATE TABLE IF NOT EXISTS miner_portfolio_queue (
repo_full_name TEXT NOT NULL,
Expand Down
28 changes: 4 additions & 24 deletions packages/gittensory-miner/lib/run-state.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
import { chmodSync, mkdirSync } from "node:fs";
import { homedir } from "node:os";
import { dirname, join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } from "./local-store.js";

export const RUN_STATES = Object.freeze(["idle", "discovering", "planning", "preparing"]);

Expand All @@ -10,26 +7,11 @@ const defaultDbFileName = "run-state.sqlite3";
let defaultRunStateStore = null;

export function resolveRunStateDbPath(env = process.env) {
const explicitPath = typeof env.GITTENSORY_MINER_RUN_STATE_DB === "string"
? env.GITTENSORY_MINER_RUN_STATE_DB.trim()
: "";
if (explicitPath) return explicitPath;

const explicitConfigDir = typeof env.GITTENSORY_MINER_CONFIG_DIR === "string"
? env.GITTENSORY_MINER_CONFIG_DIR.trim()
: "";
if (explicitConfigDir) return join(explicitConfigDir, defaultDbFileName);

const configHome = typeof env.XDG_CONFIG_HOME === "string" && env.XDG_CONFIG_HOME.trim()
? env.XDG_CONFIG_HOME.trim()
: join(homedir(), ".config");
return join(configHome, "gittensory-miner", defaultDbFileName);
return resolveLocalStoreDbPath(defaultDbFileName, "GITTENSORY_MINER_RUN_STATE_DB", env);
}

function normalizeDbPath(dbPath) {
const path = (dbPath ?? resolveRunStateDbPath()).trim();
if (!path) throw new Error("invalid_run_state_db_path");
return path;
return normalizeLocalStoreDbPath(dbPath, resolveRunStateDbPath(), "invalid_run_state_db_path");
}

function normalizeRepoFullName(repoFullName) {
Expand All @@ -51,9 +33,7 @@ function normalizeRunState(state) {
*/
export function initRunStateStore(dbPath = resolveRunStateDbPath()) {
const resolvedPath = normalizeDbPath(dbPath);
mkdirSync(dirname(resolvedPath), { recursive: true, mode: 0o700 });
const db = new DatabaseSync(resolvedPath);
chmodSync(resolvedPath, 0o600);
const db = openLocalStoreDb(resolvedPath);
db.exec(`
CREATE TABLE IF NOT EXISTS miner_run_state (
repo_full_name TEXT PRIMARY KEY,
Expand Down
2 changes: 1 addition & 1 deletion packages/gittensory-miner/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
"expected-engine.version"
],
"scripts": {
"build": "node --check bin/gittensory-miner.js && node --check lib/version.js && node --check lib/cli.js && node --check lib/deny-check.js && node --check lib/run-state-cli.js && node --check lib/update-check.js && node --check lib/worktree-allocator.js && node --check lib/opportunity-fanout.js && node --check lib/ci-poller.js && node --check lib/run-state.js && node --check lib/deny-hooks.js && node --check lib/event-ledger.js && node --check lib/event-ledger-cli.js && node --check lib/claim-ledger.js && node --check lib/claim-ledger-expiry.js && node --check lib/claim-ledger-cli.js && node --check lib/claim-adjudication.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-queue-manager.js && node --check lib/portfolio-discovery.js && node --check lib/opportunity-ranker.js && node --check lib/plan-store.js && node --check lib/plan-store-cli.js && node --check lib/rejection-templates.js && node --check lib/governor-ledger.js && node --check lib/governor-ledger-cli.js && node --check lib/manage-status.js && node --check lib/manage-poll.js && node --check lib/status.js && node --check lib/laptop-init.js && node --check lib/replay-objective-anchor.js && node --check lib/replay-task-generation.js && node --check lib/calibration-types.js && node --check lib/calibration.js"
"build": "node --check bin/gittensory-miner.js && node --check lib/version.js && node --check lib/local-store.js && node --check lib/cli.js && node --check lib/deny-check.js && node --check lib/run-state-cli.js && node --check lib/update-check.js && node --check lib/worktree-allocator.js && node --check lib/opportunity-fanout.js && node --check lib/ci-poller.js && node --check lib/run-state.js && node --check lib/deny-hooks.js && node --check lib/event-ledger.js && node --check lib/event-ledger-cli.js && node --check lib/claim-ledger.js && node --check lib/claim-ledger-expiry.js && node --check lib/claim-ledger-cli.js && node --check lib/claim-adjudication.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-queue-manager.js && node --check lib/portfolio-discovery.js && node --check lib/opportunity-ranker.js && node --check lib/plan-store.js && node --check lib/plan-store-cli.js && node --check lib/rejection-templates.js && node --check lib/governor-ledger.js && node --check lib/governor-ledger-cli.js && node --check lib/manage-status.js && node --check lib/manage-poll.js && node --check lib/status.js && node --check lib/laptop-init.js && node --check lib/replay-objective-anchor.js && node --check lib/replay-task-generation.js && node --check lib/calibration-types.js && node --check lib/calibration.js"
},
"dependencies": {
"@jsonbored/gittensory-engine": ">=0.1.0 <1.0.0"
Expand Down
30 changes: 30 additions & 0 deletions test/unit/miner-local-store-readme.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "vitest";

const readmePath = join(process.cwd(), "packages/gittensory-miner/README.md");

describe("gittensory-miner local storage README (#4272)", () => {
it("documents all four local stores together with their file/table/module/env-var", () => {
const readme = readFileSync(readmePath, "utf8");
expect(readme).toContain("## Local storage");
expect(readme).toContain("run-state.sqlite3");
expect(readme).toContain("miner_run_state");
expect(readme).toContain("claim-ledger.sqlite3");
expect(readme).toContain("miner_claims");
expect(readme).toContain("portfolio-queue.sqlite3");
expect(readme).toContain("miner_portfolio_queue");
expect(readme).toContain("event-ledger.sqlite3");
expect(readme).toContain("miner_event_ledger");
expect(readme).toContain("GITTENSORY_MINER_RUN_STATE_DB");
expect(readme).toContain("GITTENSORY_MINER_CLAIM_LEDGER_DB");
expect(readme).toContain("GITTENSORY_MINER_PORTFOLIO_QUEUE_DB");
expect(readme).toContain("GITTENSORY_MINER_EVENT_LEDGER_DB");
});

it("documents the PR-portfolio read-time-join decision", () => {
const readme = readFileSync(readmePath, "utf8");
expect(readme).toContain("read-time join");
expect(readme).toContain("manage_pr_update");
});
});
Loading