diff --git a/packages/web/src/components/checklist-ui/compare/MultiPartQuestionPage.jsx b/packages/web/src/components/checklist-ui/compare/MultiPartQuestionPage.jsx index 14a8655fe..34871b6c9 100644 --- a/packages/web/src/components/checklist-ui/compare/MultiPartQuestionPage.jsx +++ b/packages/web/src/components/checklist-ui/compare/MultiPartQuestionPage.jsx @@ -17,8 +17,14 @@ export default function MultiPartQuestionPage(props) { // Local state for each part const [localFinal, setLocalFinal] = createSignal({}); const [selectedSource, setSelectedSource] = createSignal(null); + const [hasAutoFilled, setHasAutoFilled] = createSignal(false); + + // Reset auto-fill tracking when question changes + createEffect(() => { + props.questionKey; + setHasAutoFilled(false); + }); - // Check if both reviewers have the same answers const reviewersAgree = () => multiPartAnswersEqual(props.reviewer1Answers, props.reviewer2Answers); @@ -51,6 +57,40 @@ export default function MultiPartQuestionPage(props) { } }); + // Check if the final answer last column has at least one part as true + function hasValidFinalAnswer(finalAnswers, partKeys) { + if (!finalAnswers || !Array.isArray(partKeys) || partKeys.length === 0) return false; + return partKeys.some(dk => { + const part = finalAnswers[dk]; + if (!part?.answers || !Array.isArray(part.answers) || part.answers.length === 0) return false; + const lastCol = part.answers[part.answers.length - 1]; + return Array.isArray(lastCol) && lastCol.some(v => v === true); + }); + } + + // Auto-fill when reviewers agree and no final answer exists + createEffect(() => { + const keys = dataKeys(); + if (!keys || keys.length === 0) return; + + // Check if we have a valid final answer (has at least one part) + let hasFinalAnswer = hasValidFinalAnswer(props.finalAnswers, keys); + + // Only auto-fill if: reviewers agree, no final answer exists, we have reviewer1's answer, and we haven't auto-filled yet + if ( + props.isAgreement && + !hasFinalAnswer && + props.reviewer1Answers && + keys.some(dk => props.reviewer1Answers[dk]) && + !hasAutoFilled() && + props.onFinalChange + ) { + const newFinal = JSON.parse(JSON.stringify(props.reviewer1Answers)); + props.onFinalChange(newFinal); + setHasAutoFilled(true); + } + }); + function multiPartAnswersEqual(a, b) { if (!a || !b) return false; for (const dk of dataKeys()) { diff --git a/packages/web/src/components/checklist-ui/compare/ReconciliationQuestionPage.jsx b/packages/web/src/components/checklist-ui/compare/ReconciliationQuestionPage.jsx index 42f8e7657..cf391d68f 100644 --- a/packages/web/src/components/checklist-ui/compare/ReconciliationQuestionPage.jsx +++ b/packages/web/src/components/checklist-ui/compare/ReconciliationQuestionPage.jsx @@ -51,8 +51,14 @@ function SingleQuestionPage(props) { // Local state for the final/merged answer that user can edit const [localFinal, setLocalFinal] = createSignal(null); const [selectedSource, setSelectedSource] = createSignal(null); // 'reviewer1' | 'reviewer2' | 'custom' + const [hasAutoFilled, setHasAutoFilled] = createSignal(false); + + // Reset auto-fill tracking when question changes + createEffect(() => { + props.questionKey; + setHasAutoFilled(false); + }); - // Check if both reviewers have the same answers const reviewersAgree = () => answersEqual(props.reviewer1Answers, props.reviewer2Answers); // Initialize local final from props or default to reviewer1 @@ -68,12 +74,41 @@ function SingleQuestionPage(props) { setSelectedSource('custom'); } } else if (props.reviewer1Answers) { - // Default to reviewer1 + // Default to reviewer1 for local state setLocalFinal(JSON.parse(JSON.stringify(props.reviewer1Answers))); setSelectedSource('reviewer1'); } }); + // Check if the final answer last column has at least one part as true + function hasValidFinalAnswer(finalAnswers) { + if ( + !finalAnswers?.answers || + !Array.isArray(finalAnswers.answers) || + finalAnswers.answers.length === 0 + ) + return false; + const lastCol = finalAnswers.answers[finalAnswers.answers.length - 1]; + return Array.isArray(lastCol) && lastCol.some(v => v === true); + } + + // Auto-fill when reviewers agree and no final answer exists + createEffect(() => { + let hasFinalAnswer = hasValidFinalAnswer(props.finalAnswers); + // Only auto-fill if: reviewers agree, no final answer exists, we have reviewer1's answer, and we haven't auto-filled yet + if ( + props.isAgreement && + !hasFinalAnswer && + props.reviewer1Answers && + !hasAutoFilled() && + props.onFinalChange + ) { + const newFinal = JSON.parse(JSON.stringify(props.reviewer1Answers)); + props.onFinalChange(newFinal); + setHasAutoFilled(true); + } + }); + // Check if two answer objects are equal function answersEqual(a, b) { if (!a || !b) return false; diff --git a/packages/web/src/components/checklist-ui/compare/ReconciliationWrapper.jsx b/packages/web/src/components/checklist-ui/compare/ReconciliationWrapper.jsx index e161317f7..7e3a7ff42 100644 --- a/packages/web/src/components/checklist-ui/compare/ReconciliationWrapper.jsx +++ b/packages/web/src/components/checklist-ui/compare/ReconciliationWrapper.jsx @@ -35,7 +35,6 @@ export default function ReconciliationWrapper() { getReconciliationProgress, getQuestionNote, saveReconciliationProgress, - clearReconciliationProgress, connect, } = useProject(params.projectId); @@ -365,8 +364,16 @@ export default function ReconciliationWrapper() { title: reconciledName || 'Reconciled Checklist', }); - // Clear the reconciliation progress since we've completed it - clearReconciliationProgress(params.studyId); + // Mark the individual reviewer checklists as completed + updateChecklist(params.studyId, params.checklist1Id, { + status: CHECKLIST_STATUS.COMPLETED, + }); + updateChecklist(params.studyId, params.checklist2Id, { + status: CHECKLIST_STATUS.COMPLETED, + }); + + // Keep reconciliation progress (checklist1Id and checklist2Id) so users can view previous reviewers + // The progress data is needed for the "View Previous" button in the completed tab // Navigate back to the project view (completed tab) navigate(`/projects/${params.projectId}?tab=completed`); diff --git a/packages/web/src/components/project-ui/overview-tab/OverviewTab.jsx b/packages/web/src/components/project-ui/overview-tab/OverviewTab.jsx index c0b7764ed..51bf81df7 100644 --- a/packages/web/src/components/project-ui/overview-tab/OverviewTab.jsx +++ b/packages/web/src/components/project-ui/overview-tab/OverviewTab.jsx @@ -15,6 +15,10 @@ import { Avatar, useConfirmDialog, showToast, Progress, Collapsible } from '@cor import { API_BASE } from '@config/api.js'; import { CHECKLIST_STATUS } from '@/constants/checklist-status.js'; import { shouldShowInTab } from '@/lib/checklist-domain.js'; +import { + calculateInterRaterReliability, + getKappaInterpretation, +} from '@/lib/inter-rater-reliability.js'; import CircularProgress from './CircularProgress.jsx'; /** @@ -149,6 +153,11 @@ export default function OverviewTab() { return projectActionsStore.checklist.getData(studyId, checklistId); }; + // Calculate inter-rater reliability metrics + const interRaterMetrics = createMemo(() => { + return calculateInterRaterReliability(studies(), getChecklistData); + }); + // Calculate unassigned studies for Reviewer Assignment visibility const unassignedStudies = createMemo(() => studies().filter(s => !s.reviewer1 && !s.reviewer2)); @@ -206,6 +215,37 @@ export default function OverviewTab() { + + {/* Inter-rater Reliability Section */} + 0}> +
+

Inter-rater Reliability

+
+
+

{interRaterMetrics().studyCount}

+

Studies Included

+
+
+

+ {interRaterMetrics().percentAgreement != null ? + `${interRaterMetrics().percentAgreement.toFixed(1)}%` + : 'N/A'} +

+

Percent Agreement

+
+
+

+ {interRaterMetrics().cohensKappa != null ? + interRaterMetrics().cohensKappa.toFixed(3) + : 'N/A'} +

+

+ Cohen's Kappa ({getKappaInterpretation(interRaterMetrics().cohensKappa)}) +

+
+
+
+ {/* Section 2: Team & Collaboration */} diff --git a/packages/web/src/lib/inter-rater-reliability.js b/packages/web/src/lib/inter-rater-reliability.js new file mode 100644 index 000000000..471378679 --- /dev/null +++ b/packages/web/src/lib/inter-rater-reliability.js @@ -0,0 +1,207 @@ +/** + * Inter-rater Reliability Calculation Utilities + * + * Calculates simple percent agreement and Cohen's Kappa for AMSTAR2 checklists + * across dual-reviewer studies. + */ + +import { CHECKLIST_STATUS } from '@/constants/checklist-status.js'; +import { getAnswers } from '@/AMSTAR2/checklist.js'; +import { getQuestionKeys } from '@/AMSTAR2/checklist-compare.js'; + +/** + * Calculate inter-rater reliability metrics for all eligible studies + * @param {Array} studies - Array of study objects + * @param {Function} getChecklistData - Function to retrieve checklist data (studyId, checklistId) => { answers: {...} } + * @returns {Object} Metrics object with percentAgreement, cohensKappa, studyCount, totalComparisons + */ +export function calculateInterRaterReliability(studies, getChecklistData) { + if (!studies || !Array.isArray(studies) || studies.length === 0) { + return { + percentAgreement: null, + cohensKappa: null, + studyCount: 0, + totalComparisons: 0, + }; + } + + // Filter studies with dual reviewers + const dualReviewerStudies = studies.filter(s => s.reviewer1 && s.reviewer2); + + if (dualReviewerStudies.length === 0) { + return { + percentAgreement: null, + cohensKappa: null, + studyCount: 0, + totalComparisons: 0, + }; + } + + // Collect all question comparisons across all studies + const allComparisons = []; + let eligibleStudyCount = 0; + + for (const study of dualReviewerStudies) { + const checklists = study.checklists || []; + + // Find 2 completed AMSTAR2 checklists (one per reviewer) + const completedChecklists = checklists.filter( + c => c.status === CHECKLIST_STATUS.COMPLETED && c.type === 'AMSTAR2', + ); + + // Must have exactly 2 completed checklists + if (completedChecklists.length !== 2) continue; + + // Verify one is from reviewer1 and one is from reviewer2 + const reviewer1Checklist = completedChecklists.find(c => c.assignedTo === study.reviewer1); + const reviewer2Checklist = completedChecklists.find(c => c.assignedTo === study.reviewer2); + + if (!reviewer1Checklist || !reviewer2Checklist) continue; + + // Get checklist data + const checklist1Data = getChecklistData?.(study.id, reviewer1Checklist.id); + const checklist2Data = getChecklistData?.(study.id, reviewer2Checklist.id); + + if (!checklist1Data?.answers || !checklist2Data?.answers) continue; + + // Extract answers using getAnswers function + const answers1 = getAnswers(checklist1Data.answers); + const answers2 = getAnswers(checklist2Data.answers); + + if (!answers1 || !answers2) continue; + + // Get question keys (q1-q16, with q9 and q11 consolidated) + const questionKeys = getQuestionKeys(); + + // Compare each question + for (const questionKey of questionKeys) { + const answer1 = answers1[questionKey]; + const answer2 = answers2[questionKey]; + + // Skip if either answer is missing/null + if (answer1 == null || answer2 == null) continue; + + allComparisons.push({ + questionKey, + reviewer1: answer1, + reviewer2: answer2, + agree: answer1 === answer2, + }); + } + + eligibleStudyCount++; + } + + if (allComparisons.length === 0) { + return { + percentAgreement: null, + cohensKappa: null, + studyCount: 0, + totalComparisons: 0, + }; + } + + // Calculate percent agreement + const agreements = allComparisons.filter(c => c.agree).length; + const percentAgreement = (agreements / allComparisons.length) * 100; + + // Calculate Cohen's Kappa + const cohensKappa = calculateCohensKappa(allComparisons); + + return { + percentAgreement, + cohensKappa, + studyCount: eligibleStudyCount, + totalComparisons: allComparisons.length, + }; +} + +/** + * Calculate Cohen's Kappa from comparison data + * @param {Array} comparisons - Array of {questionKey, reviewer1, reviewer2, agree} + * @returns {number|null} Cohen's Kappa value or null if calculation not possible + */ +function calculateCohensKappa(comparisons) { + if (!comparisons || comparisons.length === 0) return null; + + // Get all unique answer values + const allAnswers = new Set(); + comparisons.forEach(c => { + allAnswers.add(c.reviewer1); + allAnswers.add(c.reviewer2); + }); + + const answerCategories = Array.from(allAnswers).sort(); + + if (answerCategories.length === 0) return null; + + // Build confusion matrix: [reviewer1][reviewer2] = count + const matrix = {}; + answerCategories.forEach(a1 => { + matrix[a1] = {}; + answerCategories.forEach(a2 => { + matrix[a1][a2] = 0; + }); + }); + + comparisons.forEach(c => { + matrix[c.reviewer1][c.reviewer2]++; + }); + + // Calculate observed agreement (P_o) + let observedAgreement = 0; + answerCategories.forEach(category => { + observedAgreement += matrix[category][category] || 0; + }); + const P_o = observedAgreement / comparisons.length; + + // Calculate expected agreement (P_e) from marginal distributions + const n = comparisons.length; + const reviewer1Marginals = {}; + const reviewer2Marginals = {}; + + answerCategories.forEach(category => { + reviewer1Marginals[category] = 0; + reviewer2Marginals[category] = 0; + }); + + comparisons.forEach(c => { + reviewer1Marginals[c.reviewer1]++; + reviewer2Marginals[c.reviewer2]++; + }); + + let expectedAgreement = 0; + answerCategories.forEach(category => { + const p1 = reviewer1Marginals[category] / n; + const p2 = reviewer2Marginals[category] / n; + expectedAgreement += p1 * p2; + }); + + const P_e = expectedAgreement; + + // Calculate Cohen's Kappa: κ = (P_o - P_e) / (1 - P_e) + // Handle edge case where P_e is very close to 1 (numerical precision) + const denominator = 1 - P_e; + if (Math.abs(denominator) < 1e-10) { + // Perfect expected agreement means no variance, return 1 if observed is also perfect + return Math.abs(P_o - 1) < 1e-10 ? 1 : null; + } + + const kappa = (P_o - P_e) / denominator; + return kappa; +} + +/** + * Get interpretation text for Cohen's Kappa value + * @param {number|null} kappa - Cohen's Kappa value + * @returns {string} Interpretation text + */ +export function getKappaInterpretation(kappa) { + if (kappa == null) return 'N/A'; + if (kappa < 0) return 'Poor'; + if (kappa <= 0.2) return 'Slight'; + if (kappa <= 0.4) return 'Fair'; + if (kappa <= 0.6) return 'Moderate'; + if (kappa <= 0.8) return 'Substantial'; + return 'Almost Perfect'; +}