Skip to content

fix(scoring): require solved-by-PR evidence for linked issues #506

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JSONbored merged 1 commit into from
Jun 10, 2026
Merged

fix(scoring): require solved-by-PR evidence for linked issues #506

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 24 additions & 12 deletions src/scoring/preview.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@ export type LinkedIssueMultiplierContext = {
warnings?: string[] | undefined;
};

const PROJECTED_SOLVED_BY_PULL_REQUEST_VALIDATION = Symbol("projectedSolvedByPullRequestValidation");

type ProjectedLinkedIssueMultiplierContext = LinkedIssueMultiplierContext & {
[PROJECTED_SOLVED_BY_PULL_REQUEST_VALIDATION]?: true;
};

export type LinkedIssueMultiplierDecision = {
mode: "none" | "standard" | "maintainer";
status: LinkedIssueMultiplierStatus;
Expand Down Expand Up @@ -713,13 +719,18 @@ function decideLinkedIssueMultiplier(
};
}

const status = context?.status ?? (solvedByPullRequests.length > 0 ? "validated" : issueNumbers.length > 0 ? "raw" : "unavailable");
const projectedSolvedByPullRequestValidation = (context as ProjectedLinkedIssueMultiplierContext | undefined)?.[PROJECTED_SOLVED_BY_PULL_REQUEST_VALIDATION] === true;
const requestedStatus = context?.status ?? (solvedByPullRequests.length > 0 ? "validated" : issueNumbers.length > 0 ? "raw" : "unavailable");
const hasSolvedByPullRequestEvidence = solvedByPullRequests.length > 0 || projectedSolvedByPullRequestValidation;
const status = requestedStatus === "validated" && !hasSolvedByPullRequestEvidence ? (issueNumbers.length > 0 ? "raw" : "unavailable") : requestedStatus;
const source = context?.source ?? (status === "unavailable" ? "missing" : "user_supplied");
const branchEligible = !(branchEligibility.required && branchEligibility.status === "ineligible");
const eligible = status === "validated" && branchEligible;
const eligible = status === "validated" && hasSolvedByPullRequestEvidence && branchEligible;
const reason =
branchEligible || status !== "validated"
? context?.reason ?? linkedIssueReason(status, source, issueNumbers, solvedByPullRequests)
? status === requestedStatus
? context?.reason ?? linkedIssueReason(status, source, issueNumbers, solvedByPullRequests)
: linkedIssueReason(status, source, issueNumbers, solvedByPullRequests)
: "Branch eligibility is confirmed ineligible; standard issue multiplier is not applied.";
return {
mode,
Expand All @@ -740,18 +751,19 @@ function withValidatedLinkedIssueScenario(input: ScorePreviewInput): ScorePrevie
if (mode === "maintainer") return input;
const issueNumbers = uniquePositiveInts(input.linkedIssueContext?.issueNumbers ?? []);
const solvedByPullRequests = uniquePositiveInts(input.linkedIssueContext?.solvedByPullRequests ?? []);
const linkedIssueContext: ProjectedLinkedIssueMultiplierContext = {
...input.linkedIssueContext,
status: "validated",
source: input.linkedIssueContext?.source ?? "user_supplied",
issueNumbers,
solvedByPullRequests,
warnings: [],
[PROJECTED_SOLVED_BY_PULL_REQUEST_VALIDATION]: true,
};
return {
...input,
linkedIssueMode: "standard",
linkedIssueContext: {
...input.linkedIssueContext,
status: "validated",
source: input.linkedIssueContext?.source ?? "user_supplied",
issueNumbers,
solvedByPullRequests,
reason: "Projection assumes linked issue context is solved-by-PR validated.",
warnings: [],
},
linkedIssueContext,
};
}

Expand Down
16 changes: 15 additions & 1 deletion test/unit/eligibility-scenarios.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ describe("eligible branch with validated linked issue", () => {
status: "validated",
source: "official_mirror",
issueNumbers: [42],
solvedByPullRequests: [],
solvedByPullRequests: [101],
},
branchEligibility: { status: "eligible", source: "github_metadata" },
});
Expand Down Expand Up @@ -309,6 +309,20 @@ describe("branch-eligibility-missing — metadata not provided", () => {
const codes = result.blockedBy.map((b) => b.code);
expect(codes).toContain("branch_eligibility_missing");
});

it("keeps eligibility unconfirmed when the link is validated but branch metadata is missing", () => {
const validatedLinkMissingBranch = preview({
linkedIssueMode: "standard",
linkedIssueContext: { status: "validated", source: "official_mirror", issueNumbers: [10], solvedByPullRequests: [11] },
});
const plan = deriveEligibilityPlan(validatedLinkMissingBranch);
expect(plan).toMatchObject({
eligible: false,
linkedIssueStatus: "validated",
branchEligibilityStatus: "unknown",
});
expect(plan.publicSummary).toMatch(/not yet validated|validation is needed/i);
});
});

// ── Public/private sanitizer tests ────────────────────────────────────────
Expand Down
23 changes: 21 additions & 2 deletions test/unit/scoring.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import { getLatestScoringModelSnapshot } from "../../src/db/repositories";
import { detectActiveModel, parsePythonNumberConstants, refreshScoringModelSnapshot } from "../../src/scoring/model";
import { buildScorePreview, makeScorePreviewRecord } from "../../src/scoring/preview";
import type { ScorePreviewInput } from "../../src/scoring/preview";
import type { RepositoryRecord, ScoringModelSnapshotRecord } from "../../src/types";
import { createTestEnv } from "../helpers/d1";

Expand Down Expand Up @@ -354,6 +355,19 @@ MAX_CODE_DENSITY_MULTIPLIER = 1.15
snapshot,
input: { ...baseInput, linkedIssueContext: { status: "validated", source: "github_cache", issueNumbers: [11] } },
});
const validatedWithoutIssueOrSolver = buildScorePreview({
repo,
snapshot,
input: { ...baseInput, linkedIssueContext: { status: "validated", source: "github_cache" } },
});
const forgedProjectedValidatedWithoutSolverNumber = buildScorePreview({
repo,
snapshot,
input: {
...baseInput,
linkedIssueContext: { status: "validated", source: "user_supplied", issueNumbers: [14], projectedSolvedByPullRequestValidation: true } as unknown as ScorePreviewInput["linkedIssueContext"],
},
});
const rawByDefault = buildScorePreview({
repo,
snapshot,
Expand All @@ -379,7 +393,9 @@ MAX_CODE_DENSITY_MULTIPLIER = 1.15
expect(raw.linkedIssueMultiplier).toMatchObject({ status: "raw", eligible: false, appliedMultiplier: 1 });
expect(raw.scoreEstimate.issueMultiplier).toBe(1);
expect(raw.blockedBy).toEqual(expect.arrayContaining([expect.objectContaining({ code: "linked_issue_unvalidated", severity: "context" })]));
expect(raw.scenarioPreviews.find((scenario) => scenario.name === "linkedIssueFixed")?.linkedIssueMultiplier).toMatchObject({ status: "validated", appliedMultiplier: 1.33 });
const rawFixedScenario = raw.scenarioPreviews.find((scenario) => scenario.name === "linkedIssueFixed");
expect(rawFixedScenario?.linkedIssueMultiplier).toMatchObject({ status: "validated", appliedMultiplier: 1.33 });
expect(rawFixedScenario?.linkedIssueMultiplier.reason).toBe("Linked issue context is solved-by-PR validated for issue(s) #7.");
expect(validated.linkedIssueMultiplier).toMatchObject({ status: "validated", eligible: true, solvedByPullRequests: [101], appliedMultiplier: 1.33 });
expect(validated.scoreEstimate.issueMultiplier).toBe(1.33);
expect(invalid.linkedIssueMultiplier).toMatchObject({ status: "invalid", eligible: false, appliedMultiplier: 1 });
Expand All @@ -388,7 +404,10 @@ MAX_CODE_DENSITY_MULTIPLIER = 1.15
expect(plausible.linkedIssueMultiplier).toMatchObject({ status: "plausible", eligible: false, appliedMultiplier: 1 });
expect(plausible.warnings.join(" ")).toMatch(/plausible.*not solved-by-PR/i);
expect(defaultValidated.linkedIssueMultiplier).toMatchObject({ status: "validated", source: "user_supplied", solvedByPullRequests: [110], appliedMultiplier: 1.33 });
expect(validatedWithoutSolverNumber.linkedIssueMultiplier.reason).toMatch(/validated for issue\(s\) #11\./i);
expect(validatedWithoutSolverNumber.linkedIssueMultiplier).toMatchObject({ status: "raw", eligible: false, appliedMultiplier: 1 });
expect(validatedWithoutSolverNumber.linkedIssueMultiplier.reason).toMatch(/no solved-by-PR validation/i);
expect(validatedWithoutIssueOrSolver.linkedIssueMultiplier).toMatchObject({ status: "unavailable", eligible: false, issueNumbers: [], appliedMultiplier: 1 });
expect(forgedProjectedValidatedWithoutSolverNumber.linkedIssueMultiplier).toMatchObject({ status: "raw", eligible: false, issueNumbers: [14], appliedMultiplier: 1 });
expect(rawByDefault.linkedIssueMultiplier).toMatchObject({ status: "raw", source: "user_supplied", issueNumbers: [12], appliedMultiplier: 1 });
expect(unavailableByDefault.linkedIssueMultiplier).toMatchObject({ status: "unavailable", source: "missing", issueNumbers: [], appliedMultiplier: 1 });
expect(malformedNumbers.linkedIssueMultiplier).toMatchObject({ issueNumbers: [13], solvedByPullRequests: [120] });
Expand Down