Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions web/backend/app/api/routes/review_fact_decomp.py
Original file line number Diff line number Diff line change
Expand Up @@ -285,13 +285,22 @@ def submit_model_eval_review(
raise HTTPException(
status_code=400, detail="Each claim needs an explicit review decision"
)
valid_positions = {c.position for c in claims}
for claim in body.human_claims:
try:
claim.response_spans = _validate_submitted_spans(
item.prompt_text, claim.response_spans
)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
if (
claim.split_from_position is not None
and claim.split_from_position not in valid_positions
):
raise HTTPException(
status_code=400,
detail=f"split_from_position {claim.split_from_position} does not reference a model claim",
)
assert task.id is not None
try:
ratings = ModelCorrectionRating(
Expand Down
1 change: 1 addition & 0 deletions web/backend/app/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,7 @@ class HumanClaim(SQLModel):
min_length=1, max_length=MAX_SPANS_PER_CLAIM
)
label: ImportanceLabel
split_from_position: int | None = Field(default=None, ge=0)

@field_validator("claim_text")
@classmethod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ def _valid_submission() -> dict:
"claim_text": "The response mentions alpha.",
"label": "semi-important",
"response_spans": [{"start": 0, "end": 5, "text": "Alpha"}],
"split_from_position": None,
}
],
"coverage_checked": True,
Expand Down
2 changes: 1 addition & 1 deletion web/frontend/openapi.json

Large diffs are not rendered by default.

12 changes: 12 additions & 0 deletions web/frontend/src/client/schemas.gen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2147,6 +2147,18 @@ export const HumanClaimSchema = {
minItems: 1,
title: 'Response Spans',
type: 'array'
},
split_from_position: {
anyOf: [
{
minimum: 0,
type: 'integer'
},
{
type: 'null'
}
],
title: 'Split From Position'
}
},
required: ['claim_text', 'response_spans', 'label'],
Expand Down
1 change: 1 addition & 0 deletions web/frontend/src/client/types.gen.ts
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,7 @@ export type HumanClaim = {
claim_text: string;
label: 'vital' | 'semi-important' | 'unimportant';
response_spans: Array<ResponseClaimSpan>;
split_from_position?: (number | null);
};

export type label = 'vital' | 'semi-important' | 'unimportant';
Expand Down
219 changes: 214 additions & 5 deletions web/frontend/src/components/annotation/ClaimCorrectionList.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@ export type ClaimGroup = {
looksGood?: boolean
}

export type SplitAtomDraft = {
id: number
claim_text: string
response_spans: ClaimResponseSpan[]
label?: ImportanceLabel
}

export function initialModelClaim(claim: ReviewModelClaim): DraftHumanClaim {
return { claim_text: claim.claim_text, response_spans: claim.response_spans }
}
Expand Down Expand Up @@ -66,13 +73,147 @@ export function Labels({
)
}

function SplitEditor({
position,
atoms,
locked,
stagedSpans,
labels,
onChange,
onConfirm,
onCancel,
}: {
position: number
atoms: SplitAtomDraft[]
locked: boolean
stagedSpans: ClaimResponseSpan[]
labels: ImportanceGuideLabel[]
onChange: (atoms: SplitAtomDraft[]) => void
onConfirm: (atoms: SplitAtomDraft[]) => void
onCancel: () => void
}) {
const updateAtom = (id: number, patch: Partial<SplitAtomDraft>) =>
onChange(
atoms.map((atom) => (atom.id === id ? { ...atom, ...patch } : atom)),
)

const canConfirm =
atoms.length >= 2 &&
atoms.every(
(a) => a.claim_text.trim() && a.response_spans.length > 0 && a.label,
)

return (
<fieldset
disabled={locked}
className="mt-4 space-y-3 rounded-lg border border-dashed p-3"
>
<p className="text-xs font-semibold uppercase tracking-wide text-muted-foreground">
Split into atoms — Claim {position + 1}
</p>
{atoms.map((atom, i) => (
<div
key={atom.id}
className="space-y-2 rounded-md border bg-muted/10 p-3"
>
<div className="flex items-center justify-between">
<span className="text-xs font-medium text-muted-foreground">
Atom {i + 1}
</span>
{atoms.length > 2 && (
<Button
size="sm"
variant="ghost"
type="button"
onClick={() =>
onChange(
atoms.filter((candidate) => candidate.id !== atom.id),
)
}
>
Remove
</Button>
)}
</div>
<textarea
aria-label={`Atom ${i + 1} text`}
className="w-full rounded border bg-background p-2 text-sm"
maxLength={20000}
placeholder="First select span text."
value={atom.claim_text}
onChange={(e) =>
updateAtom(atom.id, { claim_text: e.target.value })
}
/>
<div className="flex flex-wrap items-center gap-2">
<span className="text-xs text-muted-foreground">
{atom.response_spans.length
? atom.response_spans.map((s) => s.text).join(" … ")
: "No spans selected"}
</span>
<Button
size="sm"
variant="outline"
type="button"
disabled={!stagedSpans.length}
onClick={() =>
updateAtom(atom.id, {
response_spans: [...stagedSpans],
claim_text: stagedSpans.map((s) => s.text).join(" "),
})
}
>
Use staged spans
</Button>
</div>
<Labels
value={atom.label}
name={`Atom ${i + 1}`}
labels={labels}
onChange={(label) => updateAtom(atom.id, { label })}
/>
</div>
))}
<div className="flex flex-wrap gap-2">
<Button
size="sm"
variant="outline"
type="button"
onClick={() => {
const id = Math.max(-1, ...atoms.map((atom) => atom.id)) + 1
onChange([
...atoms,
{ id, claim_text: "", response_spans: [], label: undefined },
])
}}
>
+ Add atom
</Button>
<Button
size="sm"
type="button"
disabled={!canConfirm}
onClick={() => onConfirm(atoms)}
>
Confirm split
</Button>
<Button size="sm" variant="ghost" type="button" onClick={onCancel}>
Cancel
</Button>
</div>
</fieldset>
)
}

function ClaimRow({
group,
name,
active,
locked,
labels,
stagedSpans,
splitSection,
onSplitStart,
onChange,
onRemove,
onFocus,
Expand All @@ -83,6 +224,8 @@ function ClaimRow({
locked: boolean
labels: ImportanceGuideLabel[]
stagedSpans: ClaimResponseSpan[]
splitSection?: React.ReactNode
onSplitStart?: () => void
onChange: (
claim: DraftHumanClaim,
issue?: string | null,
Expand Down Expand Up @@ -258,6 +401,16 @@ function ClaimRow({
>
Multiple Facts
</Button>
{onSplitStart ? (
<Button
type="button"
size="sm"
variant="outline"
onClick={onSplitStart}
>
Split into atoms
</Button>
) : null}
</fieldset>
{flagged ? (
<textarea
Expand Down Expand Up @@ -305,6 +458,7 @@ function ClaimRow({
</>
)}
</fieldset>
{splitSection}
</div>
</details>
)
Expand All @@ -317,16 +471,24 @@ export function ClaimCorrectionList({
hiddenModels,
locked,
stagedSpans,
splitDrafts,
splitTargetId,
onChange,
onRemove,
onFocusClaim,
onSplitConfirm,
onSplitStart,
onSplitChange,
onSplitCancel,
}: {
activePosition: number | null
groups: ClaimGroup[]
guide: ImportanceGuide
hiddenModels: boolean
locked: boolean
stagedSpans: ClaimResponseSpan[]
splitDrafts: Record<number, SplitAtomDraft[]>
splitTargetId: number | null
onChange: (
id: number,
claim: DraftHumanClaim,
Expand All @@ -337,10 +499,34 @@ export function ClaimCorrectionList({
) => void
onRemove: (id: number) => void
onFocusClaim: (position: number) => void
onSplitConfirm: (groupId: number, atoms: SplitAtomDraft[]) => void
onSplitStart: (groupId: number) => void
onSplitChange: (groupId: number, atoms: SplitAtomDraft[]) => void
onSplitCancel: (groupId: number) => void
}) {
const models = groups.filter((group) => group.original)
const humans = groups.filter((group) => !group.original)
const row = (group: ClaimGroup, name: string) => (

const humanName = (group: ClaimGroup) => {
const pos = group.claim.split_from_position
if (pos != null) {
const splitIndex = humans
.filter((g) => g.claim.split_from_position === pos)
.indexOf(group)
return `Split ${splitIndex + 1} (from Claim ${pos + 1})`
}
const nonSplitIndex = humans
.filter((g) => g.claim.split_from_position == null)
.indexOf(group)
return `Human claim ${nonSplitIndex + 1}`
}

const row = (
group: ClaimGroup,
name: string,
splitSection?: React.ReactNode,
onSplitStart?: () => void,
) => (
<ClaimRow
key={group.id}
group={group}
Expand All @@ -349,13 +535,16 @@ export function ClaimCorrectionList({
locked={locked}
labels={guide.labels}
stagedSpans={stagedSpans}
splitSection={splitSection}
onSplitStart={onSplitStart}
onChange={(claim, issue, duplicate, looksGood, multipleFacts) =>
onChange(group.id, claim, issue, duplicate, looksGood, multipleFacts)
}
onRemove={() => onRemove(group.id)}
onFocus={() => onFocusClaim(group.id)}
/>
)

return (
<section
className="space-y-4"
Expand All @@ -376,9 +565,29 @@ export function ClaimCorrectionList({
className="space-y-3"
data-testid="model-claims"
>
{models.map((group) =>
row(group, `Claim ${group.original!.position + 1}`),
)}
{models.map((group) => {
const isSplitting = group.id === splitTargetId
const splitSection = isSplitting ? (
<SplitEditor
position={group.original!.position}
atoms={splitDrafts[group.id]}
locked={locked}
stagedSpans={stagedSpans}
labels={guide.labels}
onChange={(atoms) => onSplitChange(group.id, atoms)}
onConfirm={(atoms) => onSplitConfirm(group.id, atoms)}
onCancel={() => onSplitCancel(group.id)}
/>
) : undefined
const startSplit =
!locked && !isSplitting ? () => onSplitStart(group.id) : undefined
return row(
group,
`Claim ${group.original!.position + 1}`,
splitSection,
startSplit,
)
})}
{!models.length ? (
<p className="text-sm text-muted-foreground">
No model claims proposed.
Expand All @@ -395,7 +604,7 @@ export function ClaimCorrectionList({
Add only worthwhile assertions that are present in the source text.
</p>
<div className="space-y-3" data-testid="human-claims">
{humans.map((group, i) => row(group, `Human claim ${i + 1}`))}
{humans.map((group) => row(group, humanName(group)))}
{!humans.length ? (
<p className="text-sm text-muted-foreground">
Select source text to add a missing claim.
Expand Down
Loading
Loading