Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/gittensory-miner/lib/discover-cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ export async function runDiscover(args, options = {}) {
goalSpecsByRepo: options.goalSpecsByRepo,
goalSpecContentByRepo: options.goalSpecContentByRepo,
});
const enqueueSummary = enqueue(rankedSummary.issues, { queueStore: portfolioQueue });
const enqueueSummary = enqueue(rankedSummary.issues, { queueStore: portfolioQueue, apiBaseUrl });

const result = {
fanOutCount: fanOut.issues.length,
Expand Down
12 changes: 7 additions & 5 deletions packages/gittensory-miner/lib/loop-cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ export async function runLoop(args, options = {}) {
if (issueNumber === null) {
// Never produced by enqueueRankedDiscovery in practice (always "issue:N") -- fail soft rather than
// crash the whole run: this exact item can never be attempted, so it will never resolve on retry.
portfolioQueue.markDone(claimed.repoFullName, claimed.identifier);
portfolioQueue.markDone(claimed.repoFullName, claimed.identifier, claimed.apiBaseUrl);
cycles.push({ cycle: cycleIndex, outcome: "skipped_malformed_identifier", identifier: claimed.identifier });
claimed = portfolioQueue.dequeueNext();
continue;
Expand All @@ -364,7 +364,9 @@ export async function runLoop(args, options = {}) {
convergence: convergenceInput,
convergenceThresholds: amsPolicy.spec.convergenceThresholds ?? DEFAULT_AMS_POLICY_SPEC.convergenceThresholds,
inFlightItem: { repoFullName: claimed.repoFullName, identifier: claimed.identifier },
markFailed: (repoFullName, identifier) => portfolioQueue.markFailed(repoFullName, identifier),
// Echoes claimed.apiBaseUrl (#5563), NOT the callback's own repoFullName/identifier alone -- two forge
// hosts can share an in-flight item with the same repo name+identifier.
markFailed: (repoFullName, identifier) => portfolioQueue.markFailed(repoFullName, identifier, claimed.apiBaseUrl),
},
{ append: (event) => governorLedger.appendGovernorEvent(event) },
);
Expand Down Expand Up @@ -419,14 +421,14 @@ export async function runLoop(args, options = {}) {
const permanentBlock = attemptOutcome === "blocked_rejection_signaled";

if (submitted) {
portfolioQueue.markDone(claimed.repoFullName, claimed.identifier);
portfolioQueue.markDone(claimed.repoFullName, claimed.identifier, claimed.apiBaseUrl);
convergenceInput.reachedDone = true;
convergenceInput.consecutiveFailures = 0;
} else if (permanentBlock) {
portfolioQueue.markDone(claimed.repoFullName, claimed.identifier);
portfolioQueue.markDone(claimed.repoFullName, claimed.identifier, claimed.apiBaseUrl);
convergenceInput.consecutiveFailures += 1;
} else {
portfolioQueue.markFailed(claimed.repoFullName, claimed.identifier);
portfolioQueue.markFailed(claimed.repoFullName, claimed.identifier, claimed.apiBaseUrl);
convergenceInput.consecutiveFailures += 1;
convergenceInput.reenqueues += 1;
}
Expand Down
1 change: 1 addition & 0 deletions packages/gittensory-miner/lib/portfolio-discovery.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ export type EnqueueRankedDiscoveryOptions = {
queueStore: PortfolioQueueStore;
eventLedger?: EventLedger;
minRankScore?: number | null;
apiBaseUrl?: string;
};

export type EnqueueRankedDiscoverySummary = {
Expand Down
5 changes: 5 additions & 0 deletions packages/gittensory-miner/lib/portfolio-discovery.js
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ export function enqueueRankedDiscovery(rankedIssues, options = {}) {
}

const minRankScore = normalizeMinRankScore(options.minRankScore);
// #5563: threaded through from the caller's already-resolved forge host, so a non-default (GitHub Enterprise)
// tenant's ranked issues land in the queue scoped to their own host instead of colliding with a same-named
// owner/repo on github.com. Omitted/nullish falls through to the queue store's own github.com default.
const apiBaseUrl = options.apiBaseUrl;

const summary = {
enqueued: 0,
Expand All @@ -73,6 +77,7 @@ export function enqueueRankedDiscovery(rankedIssues, options = {}) {
repoFullName: normalized.repoFullName,
identifier: `issue:${normalized.issueNumber}`,
priority: normalized.rankScore,
apiBaseUrl,
});
summary.enqueued += 1;

Expand Down
1 change: 1 addition & 0 deletions packages/gittensory-miner/lib/portfolio-queue-cli.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export type ParsedQueueDoneArgs =
identifier: string;
dryRun: boolean;
json: boolean;
apiBaseUrl: string | undefined;
}
| { error: string };

Expand Down
63 changes: 48 additions & 15 deletions packages/gittensory-miner/lib/portfolio-queue-cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@ import { argsWantJson, describeCliError, reportCliFailure } from "./cli-error.js

const QUEUE_LIST_USAGE = "Usage: gittensory-miner queue list [--repo <owner/repo>] [--json]";
const QUEUE_NEXT_USAGE = "Usage: gittensory-miner queue next [--dry-run] [--json]";
const QUEUE_DONE_USAGE = "Usage: gittensory-miner queue done <owner/repo> <identifier> [--dry-run] [--json]";
const QUEUE_RELEASE_USAGE = "Usage: gittensory-miner queue release <owner/repo> <identifier> [--dry-run] [--json]";
const QUEUE_REQUEUE_USAGE = "Usage: gittensory-miner queue requeue <owner/repo> <identifier> [--dry-run] [--json]";
const QUEUE_DONE_USAGE =
"Usage: gittensory-miner queue done <owner/repo> <identifier> [--api-base-url <url>] [--dry-run] [--json]";
const QUEUE_RELEASE_USAGE =
"Usage: gittensory-miner queue release <owner/repo> <identifier> [--api-base-url <url>] [--dry-run] [--json]";
const QUEUE_REQUEUE_USAGE =
"Usage: gittensory-miner queue requeue <owner/repo> <identifier> [--api-base-url <url>] [--dry-run] [--json]";
const QUEUE_CLAIM_BATCH_USAGE =
"Usage: gittensory-miner queue claim-batch [--global-wip <n>] [--per-repo-wip <n>] [--dry-run] [--json]";

Expand Down Expand Up @@ -87,28 +90,58 @@ export function parseQueueNextArgs(args) {
return { json: parsed.json, dryRun: parsed.dryRun };
}

/** Shared `<owner/repo> <identifier> [--json]` parse for the item-targeting subcommands (done/release/requeue).
* `usage` is the command-specific message surfaced on a malformed argv. */
/** Shared `<owner/repo> <identifier> [--api-base-url <url>] [--json]` parse for the item-targeting subcommands
* (done/release/requeue). `usage` is the command-specific message surfaced on a malformed argv. */
function parseRepoIdentifierArgs(args, usage) {
const parsed = parseJsonFlag(args);
if ("error" in parsed) return parsed;
if (parsed.positional.length !== 2) {
const options = { json: false, dryRun: false, apiBaseUrl: undefined };
const positional = [];

for (let index = 0; index < args.length; index += 1) {
const token = args[index];
if (token === "--json") {
options.json = true;
continue;
}
// #4847: reports what a real mutation would do and returns before opening the portfolio queue at all.
if (token === "--dry-run") {
options.dryRun = true;
continue;
}
// #5563: scope the target to a non-default forge host, so it doesn't collide with (or get confused for) a
// same-named repo on the default github.com host.
if (token === "--api-base-url") {
const value = args[index + 1];
if (!value || value.startsWith("-")) {
return { error: usage };
}
options.apiBaseUrl = value;
index += 1;
continue;
}
if (token.startsWith("-")) {
return { error: `Unknown option: ${token}` };
}
positional.push(token);
}

if (positional.length !== 2) {
return { error: usage };
}

const repo = parseRepoArg(parsed.positional[0], usage);
const repo = parseRepoArg(positional[0], usage);
if ("error" in repo) return repo;

const identifier = parsed.positional[1]?.trim();
const identifier = positional[1]?.trim();
if (!identifier) {
return { error: usage };
}

return {
repoFullName: repo.repoFullName,
identifier,
dryRun: parsed.dryRun,
json: parsed.json,
dryRun: options.dryRun,
json: options.json,
apiBaseUrl: options.apiBaseUrl,
};
}

Expand Down Expand Up @@ -230,7 +263,7 @@ export function runQueueDone(args, options = {}) {

try {
return withPortfolioQueue(options, (portfolioQueue) => {
const entry = portfolioQueue.markDone(parsed.repoFullName, parsed.identifier);
const entry = portfolioQueue.markDone(parsed.repoFullName, parsed.identifier, parsed.apiBaseUrl);
if (!entry) {
return reportCliFailure(parsed.json, "queue_entry_not_found");
}
Expand Down Expand Up @@ -266,7 +299,7 @@ export function runQueueRelease(args, options = {}) {

try {
return withPortfolioQueue(options, (portfolioQueue) => {
const entry = portfolioQueue.reclaimStuckItem(parsed.repoFullName, parsed.identifier);
const entry = portfolioQueue.reclaimStuckItem(parsed.repoFullName, parsed.identifier, parsed.apiBaseUrl);
if (!entry) {
return reportCliFailure(parsed.json, "queue_entry_not_in_progress");
}
Expand Down Expand Up @@ -303,7 +336,7 @@ export function runQueueRequeue(args, options = {}) {

try {
return withPortfolioQueue(options, (portfolioQueue) => {
const entry = portfolioQueue.requeueItem(parsed.repoFullName, parsed.identifier);
const entry = portfolioQueue.requeueItem(parsed.repoFullName, parsed.identifier, parsed.apiBaseUrl);
if (!entry) {
return reportCliFailure(parsed.json, "queue_entry_not_requeuable");
}
Expand Down
4 changes: 3 additions & 1 deletion packages/gittensory-miner/lib/portfolio-queue-expiry.js
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,9 @@ export function sweepStuckItems(store, nowMs, maxLeaseMs = DEFAULT_MAX_LEASE_MS)
const stuck = findStuckItems(inProgress, nowMs, maxLeaseMs);
const reclaimed = [];
for (const item of stuck) {
const updated = store.reclaimStuckItem(item.repoFullName, item.identifier);
// Echo the item's OWN apiBaseUrl back (#5563) rather than defaulting: two forge hosts can each have an
// in-flight item with the same owner/repo+identifier, and defaulting here would reclaim the wrong host's row.
const updated = store.reclaimStuckItem(item.repoFullName, item.identifier, item.apiBaseUrl);
if (updated) reclaimed.push(updated);
}
return reclaimed;
Expand Down
4 changes: 2 additions & 2 deletions packages/gittensory-miner/lib/portfolio-queue-manager.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ export type PortfolioQueueManager = {
dbPath: string;
enqueue(item: EnqueueItem): QueueEntry;
listQueue(repoFullName?: string | null): QueueEntry[];
markDone(repoFullName: string, identifier: string): QueueEntry | null;
markFailed(repoFullName: string, identifier: string): QueueEntry | null;
markDone(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;
markFailed(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;
reclaimStuckItems(maxLeaseMs?: number): QueueEntry[];
claimNextBatch(): QueueEntry[];
close(): void;
Expand Down
14 changes: 10 additions & 4 deletions packages/gittensory-miner/lib/portfolio-queue-manager.js
Original file line number Diff line number Diff line change
Expand Up @@ -89,16 +89,22 @@ export function initPortfolioQueueManager(options = {}) {
listQueue(repoFullName) {
return store.listQueue(repoFullName);
},
markDone(repoFullName, identifier) {
return store.markDone(repoFullName, identifier);
markDone(repoFullName, identifier, apiBaseUrl) {
return store.markDone(repoFullName, identifier, apiBaseUrl);
},
markFailed(repoFullName, identifier) {
return store.markFailed(repoFullName, identifier);
markFailed(repoFullName, identifier, apiBaseUrl) {
return store.markFailed(repoFullName, identifier, apiBaseUrl);
},
/** Sweep leases orphaned by a crashed/killed process back to 'queued', returning the reclaimed items (#4827). */
reclaimStuckItems(maxLeaseMs = staleLeaseMs) {
return sweepStuckItems(store, Date.now(), maxLeaseMs);
},
// NOTE (#5563): claimNextBatch's engine-driven selection (queueItemId/parseQueueItemId, entriesToPortfolioQueue)
// has no apiBaseUrl dimension -- @jsonbored/gittensory-engine's PortfolioQueueItem shape predates multi-forge
// support. selectFn below therefore never supplies target.apiBaseUrl, so batchClaim falls back to the
// github.com default for every claim; a non-default-host item enqueued under a different apiBaseUrl safely
// fails to match (no row, no claim, no corruption) rather than being claimed under the wrong host. Retrofitting
// the engine primitive itself with a forge dimension is out of this store-level fix's scope.
claimNextBatch() {
// Reclaim orphaned leases first, so an item stranded 'in_progress' by a dead process becomes eligible again
// instead of permanently consuming a WIP slot and starving the queue.
Expand Down
19 changes: 12 additions & 7 deletions packages/gittensory-miner/lib/portfolio-queue.d.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
export type QueueStatus = "queued" | "in_progress" | "done";

export type QueueEntry = {
apiBaseUrl: string;
repoFullName: string;
identifier: string;
priority: number;
Expand All @@ -12,10 +13,12 @@ export type EnqueueItem = {
repoFullName: string;
identifier: string;
priority?: number | null;
apiBaseUrl?: string;
};

/** Lease-annotated view of an in-flight row: when it was claimed, for the expiry sweep (#4827). */
export type QueueLeaseEntry = {
apiBaseUrl: string;
repoFullName: string;
identifier: string;
status: QueueStatus;
Expand All @@ -28,12 +31,14 @@ export type PortfolioQueueStore = {
dequeueNext(): QueueEntry | null;
listQueue(repoFullName?: string | null): QueueEntry[];
listInProgress(): QueueLeaseEntry[];
markDone(repoFullName: string, identifier: string): QueueEntry | null;
markFailed(repoFullName: string, identifier: string): QueueEntry | null;
reclaimStuckItem(repoFullName: string, identifier: string): QueueEntry | null;
requeueItem(repoFullName: string, identifier: string): QueueEntry | null;
markDone(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;
markFailed(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;
reclaimStuckItem(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;
requeueItem(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;
batchClaim(
selectFn: (entries: QueueEntry[]) => Array<{ repoFullName: string; identifier: string }>,
selectFn: (
entries: QueueEntry[],
) => Array<{ repoFullName: string; identifier: string; apiBaseUrl?: string }>,
): QueueEntry[];
close(): void;
};
Expand All @@ -50,8 +55,8 @@ export function dequeueNext(): QueueEntry | null;

export function listQueue(repoFullName?: string | null): QueueEntry[];

export function markDone(repoFullName: string, identifier: string): QueueEntry | null;
export function markDone(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;

export function markFailed(repoFullName: string, identifier: string): QueueEntry | null;
export function markFailed(repoFullName: string, identifier: string, apiBaseUrl?: string): QueueEntry | null;

export function closeDefaultPortfolioQueueStore(): void;
Loading