diff --git a/actions/setup/js/trace_graders.cjs b/actions/setup/js/trace_graders.cjs index 4e9c0462249..0a2b64f3242 100644 --- a/actions/setup/js/trace_graders.cjs +++ b/actions/setup/js/trace_graders.cjs @@ -460,10 +460,40 @@ function evaluateThreshold(value, direction, threshold) { function sanitizeSummaryText(value) { return String(value ?? "") .replace(/\r?\n/g, " ") + .replace(/\\/g, "\\\\") + .replace(/\|/g, "\\|") .replace(/[<>&]/g, ch => (ch === "<" ? "<" : ch === ">" ? ">" : "&")) .trim(); } +/** + * @param {GraderResult} result + * @returns {result is GraderResult & {value: number}} + */ +function hasComputedValue(result) { + return typeof result.value === "number"; +} + +/** + * Build the Graders section body for the GitHub Actions step summary. + * @param {GraderResult[]} results + * @returns {string} + */ +function buildGradersSummaryBody(results) { + const computedResults = results.filter(hasComputedValue); + if (computedResults.length === 0) { + return "\n\nNo grader values available.\n\n"; + } + + const statusLabels = { pass: "Pass", fail: "Fail", error: "Error", unavailable: "Unavailable" }; + const rows = computedResults.map(result => { + const value = String(Number(result.value.toFixed(4))); + return `| ${statusLabels[result.status] || "Unknown"} | ${sanitizeSummaryText(result.name)} | ${sanitizeSummaryText(result.source)} | ${value} | ${sanitizeSummaryText(result.unit || "—")} |`; + }); + + return ["", "", "| Status | Grader | Source | Value | Unit |", "| --- | --- | --- | --- | --- |", ...rows, "", ""].join("\n"); +} + /** * Normalize a grader result from either built-in number or custom object return. * @param {string} id @@ -806,25 +836,7 @@ async function main(manifestB64, execSpecB64) { } // Step summary - core.summary.addHeading("Graders", 3); - const tableResults = results.filter(r => r.status !== "unavailable"); - if (tableResults.length > 0) { - const rows = tableResults.map(r => { - const statusIcon = r.status === "pass" ? "✅" : r.status === "fail" ? "❌" : "⚠️"; - const val = r.value !== null ? String(Number(r.value.toFixed(4))) : "—"; - return [statusIcon, sanitizeSummaryText(r.name), r.source, val, sanitizeSummaryText(r.unit || "—")]; - }); - core.summary.addTable([ - [ - { data: "", header: true }, - { data: "Grader", header: true }, - { data: "Source", header: true }, - { data: "Value", header: true }, - { data: "Unit", header: true }, - ], - ...rows, - ]); - } + core.summary.addDetails("Graders", buildGradersSummaryBody(results)); const errResults = results.filter(r => r.error); if (errResults.length > 0) { const errLines = errResults.map(r => `- **${sanitizeSummaryText(r.id)}**: runtime error (see step logs)`).join("\n"); @@ -850,6 +862,7 @@ module.exports = { runCustomGrader, runOperationalValueGrader, normalizeResult, + buildGradersSummaryBody, evaluateThreshold, BUILTIN_GRADERS, BUILTIN_META, diff --git a/actions/setup/js/trace_graders.test.cjs b/actions/setup/js/trace_graders.test.cjs index 7b4aa60e201..3c837464f11 100644 --- a/actions/setup/js/trace_graders.test.cjs +++ b/actions/setup/js/trace_graders.test.cjs @@ -20,6 +20,7 @@ const { runBuiltinGrader, runCustomGrader, normalizeResult, + buildGradersSummaryBody, evaluateThreshold, BUILTIN_GRADERS, BUILTIN_META, @@ -68,6 +69,42 @@ function makeTrace(overrides = {}) { } describe("trace_graders", () => { + describe("buildGradersSummaryBody", () => { + it("renders all computed grader values without emojis", () => { + const summary = buildGradersSummaryBody([ + { id: "tool-success-rate", name: "Tool success rate", value: 0.98765, unit: "ratio", status: "pass", source: "builtin" }, + { id: "custom", name: "Custom", value: 2, unit: "count", status: "fail", source: "inline" }, + { id: "unavailable", name: "Unavailable", value: null, unit: "", status: "unavailable", source: "builtin" }, + ]); + + expect(summary).toContain("| Pass | Tool success rate | builtin | 0.9877 | ratio |"); + expect(summary).toContain("| Fail | Custom | inline | 2 | count |"); + expect(summary).not.toContain("Unavailable"); + expect(summary).not.toMatch(/[\u{1F300}-\u{1FAFF}]/u); + }); + + it("escapes untrusted table cells", () => { + const summary = buildGradersSummaryBody([{ id: "custom", name: "Custom | ", value: 1, unit: "unit|&", status: "pass", source: "inline|source" }]); + + expect(summary).toContain("Custom \\| <grader>"); + expect(summary).toContain("inline\\|source"); + expect(summary).toContain("unit\\|&"); + }); + + it("escapes backslashes before pipes", () => { + const summary = buildGradersSummaryBody([{ id: "custom", name: "A\\|B", value: 1, unit: "count", status: "pass", source: "inline" }]); + + expect(summary).toContain("| Pass | A\\\\\\|B | inline | 1 | count |"); + }); + + it("surrounds the table with blank lines for details rendering", () => { + const summary = buildGradersSummaryBody([{ id: "custom", name: "Custom", value: 1, unit: "count", status: "pass", source: "inline" }]); + + expect(summary.startsWith("\n\n")).toBe(true); + expect(summary.endsWith("\n\n")).toBe(true); + }); + }); + describe("archiveOperationalValueEvaluator", () => { it("writes only evaluator bytes matching the frozen digest", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "operational-value-evaluator-archive-"));