diff --git a/.gitignore b/.gitignore index 018649b11..3923746a5 100644 --- a/.gitignore +++ b/.gitignore @@ -168,6 +168,7 @@ examples/api_test/ .claude .kiro .pr-review/ +.scratch_wl/ # Frontend (evalscope/web) node_modules/ diff --git a/AGENTS.md b/AGENTS.md index b404baa75..714778f43 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -126,6 +126,21 @@ Don't try to learn the architecture from this file — read these and grep: 6. Auto-discovered by globbing `evalscope/benchmarks/*/**/*_adapter.py`. 7. Add a smoke test. +## Adding a judge-scored benchmark + +An adapter must **never** call `self.llm_judge.judge()` or parse a judge reply itself — that debt is fenced off by `tests/api/judge/test_gates.py`, which scans every file under `evalscope/benchmarks/` (helpers included, so moving a parser into `utils.py` does not evade it). Score through the JSON output contract in `evalscope/api/judge/` instead: + +1. Pick a `scoring_policy` (`JUDGE_ONLY` or `JUDGE_DEFAULT`). Judge scoring always goes through the contract; there is no opt-in flag and no legacy path. +2. **Single verdict per sample:** implement `judge_definition(context)` and return `JudgeDefinition.labels(...)` for a label mapping or `JudgeDefinition.numeric(...)` for a 0-1 rating. A generic `prompt_template` must state grading criteria only: `OutputContract.instruction()` appends the reply format. An adapter that preserves an official fixed output template may keep that format instruction instead, provided its `OutputContract` schema matches the official template. +3. **Custom shape (multiple cases, ratings, rubrics):** `judge_definition(context)` declares a Pydantic `schema_model`, wraps it in `OutputContract`, and returns `JudgeDefinition.workflow(...)`: + - `cases` contains one `JudgeCase(case_id, output_contract, metadata)` per thing to judge. + - `request(case, placement, completed, context)` renders messages and appends `case.output_contract.instruction()` so the prompt and parser cannot drift. An official fixed output template may be used instead when its required fields and constraints match the case's `OutputContract` schema. + - `reduce(verdicts, context)` folds parsed verdicts into `{metric: value}`. Read a verdict's context from `CaseVerdict.metadata`, never by parsing `case_id`. + - Optional `expand`, `fallback`, and `finalize` callbacks handle staged cases, rule fallbacks, and score finalization. They may be nested functions or private adapter helpers, but are passed only through the returned definition. +4. **Rule short-circuit:** if deterministic scoring settles the sample before judge I/O, return `JudgeDefinition.skip(score, reason='...')`. The non-empty reason is persisted in `Score.metadata` as `judge_skipped=True` and `judge_skip_reason`; the web review panel displays it as rule-based scoring. +5. The executor owns request execution, position swap, repeats, multi-judge aggregation and fail-closed exclusion. Transport retries belong to the model implementation; a reply that fails the contract is not automatically retried and excludes the sample from the metric — never scored 0 or full credit — so a metric's `num` can be below the sample count. +6. Add a scripted-judge test in `tests/api/judge/test_migrated_adapters.py` covering: a valid verdict, a parse failure (prose / malformed), and a transport `[ERROR]` — each must exclude, not silently score. A judge double must carry the surface the definition reads (`score_type`, `score_mapping`, `build_prompt`), and be injected through the `llm_judge` setter rather than a private attribute. + ## Conventions & gotchas - `eval_type`: `openai_api`, `llm_ckpt`, `mock_llm`, `text2image`, `image_editing`. Deprecated aliases: `server` → `openai_api`, `checkpoint` → `llm_ckpt`. diff --git a/docs/en/advanced_guides/add_benchmark.md b/docs/en/advanced_guides/add_benchmark.md index 29215c8e7..bbc9f2ab1 100644 --- a/docs/en/advanced_guides/add_benchmark.md +++ b/docs/en/advanced_guides/add_benchmark.md @@ -47,7 +47,7 @@ DataAdapter adopts a Pipeline architecture, supporting custom behavior through h calculate_metrics() ├── filter_prediction() │ └── extract_answer() [Optional User Implementation] - ├── match_score() / llm_match_score() + ├── match_score() / score_with_judge_contracts() └── Returns SampleScore 4. Result Aggregation Phase @@ -61,6 +61,31 @@ DataAdapter adopts a Pipeline architecture, supporting custom behavior through h └── Returns Report ``` +### Adding an LLM-Judged Benchmark + +Users enable judging with the typed `judge` configuration described in [Judge Parameters](../get_started/parameters.md#judge-parameters): + +```python +TaskConfig( + model='MODEL_UNDER_TEST', + datasets=['your_benchmark'], + judge={ + 'strategy': 'llm', + 'models': {'model_id': 'JUDGE_MODEL', 'api_url': 'OPENAI_COMPATIBLE_URL', 'api_key': 'API_KEY'}, + }, +) +``` + +For an adapter author, judge I/O belongs to `evalscope.api.judge`; do not call `self.llm_judge.judge()` or parse a model reply in the adapter. + +1. Declare `scoring_policy`: use `JUDGE_ONLY` when rule scoring is not meaningful, `JUDGE_DEFAULT` when rules remain available but `auto` should judge, and `RULE_DEFAULT` when `auto` should keep rule scoring. +2. Implement the single adapter entry point, `judge_definition(context)`. For an ordinary one-verdict task, return `JudgeDefinition.labels(...)` or `JudgeDefinition.numeric(...)` with a Pydantic verdict schema. These helpers append the JSON output instruction and keep the prompt, schema, and metric mapping together. +3. For a rubric, multiple claims, or staged task, define a Pydantic verdict schema and `OutputContract`, then return `JudgeDefinition.workflow(cases=..., request=..., reduce=...)`. Pass `expand=...`, `fallback=...`, or `finalize=...` only when the workflow needs them. The callbacks can be nested in `judge_definition()` or private adapter helpers, but they must be owned by the returned definition rather than exposed as adapter hooks. Append `case.output_contract.instruction()` in a custom request unless the official fixed JSON instruction exactly matches that schema. Use `CaseVerdict.metadata` rather than encoding state into `case_id`. +4. When a deterministic rule settles a sample without a model call, return `JudgeDefinition.skip(score, reason='...')`. `reason` is required and is persisted as `Score.metadata['judge_skip_reason']` with `Score.metadata['judge_skipped'] = True`; the web review panel labels this as rule-based scoring rather than an LLM verdict. +5. Add a scripted-judge test covering a valid JSON verdict, malformed/prose output, and a transport error. Invalid judge replies are excluded from the metric; they are not converted to a zero score and are not automatically retried by the executor. + +The executor owns request dispatch, position swaps, repeats, multi-judge quorum, aggregation, and review diagnostics. The model transport owns its own retry policy through `generation_config`. + ### Core Data Structures #### 1. Sample Object diff --git a/docs/en/advanced_guides/custom_dataset/llm.md b/docs/en/advanced_guides/custom_dataset/llm.md index fbc7ba4bd..5f887aa0c 100644 --- a/docs/en/advanced_guides/custom_dataset/llm.md +++ b/docs/en/advanced_guides/custom_dataset/llm.md @@ -240,14 +240,20 @@ run_task(task_cfg=task_cfg) **Method 2: Evaluation based on LLM** -LLM-based evaluation can conveniently assess the correctness of model outputs (or other dimensions of metrics, requiring custom prompt settings). Below is an example configuring `judge_model_args` parameters, using the preset `pattern` mode to determine the correctness of model outputs. +LLM-based evaluation can conveniently assess the correctness of model outputs (or other dimensions of metrics, requiring custom prompt settings). Below is an example using the preset `pattern` JSON contract to determine correctness. For a complete explanation of judge parameters, please refer to [documentation](../../get_started/parameters.md#judge-parameters). +```{note} +The judge replies with a single JSON object. A reply that cannot be read as one is **excluded** from +the metric rather than scored 0, so `Num` may be lower than the sample count. A +custom `prompt_template` should state the grading criteria only; the reply format is appended +automatically. +``` + ```python import os from evalscope import TaskConfig, run_task -from evalscope.constants import JudgeStrategy task_cfg = TaskConfig( model='Qwen/Qwen2.5-0.5B-Instruct', @@ -262,22 +268,18 @@ task_cfg = TaskConfig( ], } }, - # judge related parameters - judge_model_args={ - 'model_id': 'qwen2.5-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': os.getenv('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096 + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'qwen2.5-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': os.getenv('DASHSCOPE_API_KEY'), + 'generation_config': {'temperature': 0.0, 'max_tokens': 4096}, }, - # Determine if the model output is correct based on reference answers and model output - 'score_type': 'pattern', + 'contract': {'score_type': 'pattern'}, }, # eval concurrency number eval_batch_size=5, - # Use LLM for evaluation - judge_strategy=JudgeStrategy.LLM, ) run_task(task_cfg=task_cfg) @@ -298,13 +300,12 @@ run_task(task_cfg=task_cfg) If the dataset lacks reference answers, an LLM judge can be used to evaluate the model's output answers. Without configuring an LLM, no scoring results will be available. -Below is an example configuring `judge_model_args` parameters, using the preset `numeric` mode to automatically assess model output scores from dimensions such as accuracy, relevance, and usefulness. Higher scores indicate better model output. +Below is an example using the preset `numeric` JSON contract to automatically assess model output scores from dimensions such as accuracy, relevance, and usefulness. Higher scores indicate better model output. For a complete explanation of judge parameters, please refer to [documentation](../../get_started/parameters.md#judge-parameters). ```python import os from evalscope import TaskConfig, run_task -from evalscope.constants import JudgeStrategy task_cfg = TaskConfig( model='Qwen/Qwen2.5-0.5B-Instruct', @@ -319,22 +320,18 @@ task_cfg = TaskConfig( ], } }, - # judge related parameters - judge_model_args={ - 'model_id': 'qwen2.5-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': os.getenv('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096 + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'qwen2.5-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': os.getenv('DASHSCOPE_API_KEY'), + 'generation_config': {'temperature': 0.0, 'max_tokens': 4096}, }, - # Direct scoring - 'score_type': 'numeric', + 'contract': {'score_type': 'numeric'}, }, # eval concurrency number eval_batch_size=5, - # Use LLM for evaluation - judge_strategy=JudgeStrategy.LLM, ) run_task(task_cfg=task_cfg) diff --git a/docs/en/advanced_guides/custom_dataset/vlm.md b/docs/en/advanced_guides/custom_dataset/vlm.md index 8f32b5728..313b525a6 100644 --- a/docs/en/advanced_guides/custom_dataset/vlm.md +++ b/docs/en/advanced_guides/custom_dataset/vlm.md @@ -266,7 +266,7 @@ You can specify a judge model through the `judge_model` parameter to generate re ```python from evalscope.run import run_task -from evalscope.constants import EvalType, JudgeStrategy +from evalscope.constants import EvalType from os import environ as env task_cfg = TaskConfig( @@ -282,17 +282,16 @@ task_cfg = TaskConfig( } }, limit=5, - judge_model_args={ - 'model_id': 'qwen-plus', # Does not need to be a multimodal model - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': env.get('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096 + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'qwen-plus', # Does not need to be a multimodal model + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': env.get('DASHSCOPE_API_KEY'), + 'generation_config': {'temperature': 0.0, 'max_tokens': 4096}, }, }, eval_batch_size=5, - judge_strategy=JudgeStrategy.LLM, ) result = run_task(task_cfg=task_cfg) ``` @@ -307,9 +306,8 @@ evalscope eval \ --datasets general_vqa \ --dataset-args '{"general_vqa": {"local_path": "custom_eval/multimodal/vqa", "subset_list": ["example_openai"]}}' \ --limit 5 \ - --judge-model-args '{"model_id": "qwen-plus", "api_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", "api_key": "$DASHSCOPE_API_KEY", "generation_config": {"temperature": 0.0, "max_tokens": 4096}}' \ - --judge-worker-num 5 \ - --judge-strategy llm + --eval-batch-size 5 \ + --judge '{"strategy": "llm", "models": {"model_id": "qwen-plus", "api_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", "api_key": "$DASHSCOPE_API_KEY", "generation_config": {"temperature": 0.0, "max_tokens": 4096}}}' ``` Evaluation will output accuracy metrics: diff --git a/docs/en/benchmarks/air_bench_chat.md b/docs/en/benchmarks/air_bench_chat.md index 62b3a7d4c..1a5668391 100644 --- a/docs/en/benchmarks/air_bench_chat.md +++ b/docs/en/benchmarks/air_bench_chat.md @@ -10,39 +10,22 @@ AIR-Bench Chat is the generative half of [AIR-Bench](https://arxiv.org/abs/2402. - **Task Type**: Open-ended audio question answering. - **Input**: An audio clip plus a free-form question. - **Output**: A textual answer evaluated against the reference response. +- **Modalities**: Audio (human speech, natural sounds, music) + text. -## Categories (8 tasks → 5 reported categories) +## Key Features -The 8 Chat tasks are aggregated by the official `cal_score.py` into five categories: +- ~2k open-ended audio QA pairs across speech, sound, music and mixed-audio scenes; the generative half of AIR-Bench (ACL 2024). +- 8 Chat tasks aggregated by the official `cal_score.py` into 5 reported categories: `speech` (`speech_QA`, `speech_dialogue_QA`), `sound` (`sound_QA`, `sound_generation_QA`), `music` (`music_QA`, `music_generation_analysis_QA`), `speech_and_sound` (`speech_and_sound_QA`), `speech_and_music` (`speech_and_music_QA`). The paper's Mixed-audio = mean(speech_and_sound, speech_and_music). +- Position bias is removed by judging every sample twice with reference/prediction order swapped, then averaging (disable via `extra_params={'do_swap': False}` to halve judge cost). +- Hosted on ModelScope ([`evalscope/AIR-Bench-Dataset`](https://modelscope.cn/datasets/evalscope/AIR-Bench-Dataset)) in an audiofolder + JSON layout; the full release is ~49 GB, so limit tasks via `extra_params={'tasks': [...]}` for partial runs. -- `speech`: `speech_QA`, `speech_dialogue_QA` -- `sound`: `sound_QA`, `sound_generation_QA` -- `music`: `music_QA`, `music_generation_analysis_QA` -- `speech_and_sound`: `speech_and_sound_QA` -- `speech_and_music`: `speech_and_music_QA` +## Evaluation Notes -The paper's **Mixed-audio = mean(speech_and_sound, speech_and_music)**. - -## Dataset Access - -- The dataset is hosted on ModelScope: [`evalscope/AIR-Bench-Dataset`](https://modelscope.cn/datasets/evalscope/AIR-Bench-Dataset). It uses an *audiofolder + JSON metadata* layout. evalscope downloads it lazily via `modelscope.dataset_snapshot_download` on first run; the full release is ~49 GB, so it is recommended to limit which tasks are pulled via `extra_params`. +- Metrics: `judge_score` is the model's mean judge score; `win_rate` records how often the model strictly beats the reference. +- The judge LLM receives the question, the textual audio description (`meta_info`), the reference answer (`answer_gt`), and the model's response, and outputs two integer scores in `[1, 10]`. Use a judge that supports long contexts, since `meta_info` may exceed 4k tokens for dialogue tasks. +- The official leaderboard uses `gpt-4-0125-preview`. If that exact snapshot is unavailable, use an available GPT-4-class judge; absolute scores can drift versus the published numbers because the judge model changed. - If the dataset is already on disk, pass `dataset_args={'air_bench_chat': {'local_path': '/path/to/AIR-Bench-Dataset'}}`; the local root should contain `Chat/`. -## Evaluation Protocol - -- The judge LLM (default: GPT-4) receives the question, the textual audio description (`meta_info` from the dataset), the reference answer (`answer_gt`), and the model's response. It outputs a single line with two integer scores in `[1, 10]`. -- To remove position bias, every sample is judged twice with the order of reference and prediction swapped, then averaged. This mirrors `cal_score.py` in the official repository — disable it via `extra_params={'do_swap': False}` to halve judge cost. -- Reported metric `judge_score` is the model's mean judge score; `win_rate` records how often the model strictly beats the reference. - -```{warning} -The official leaderboard uses `gpt-4-0125-preview` as the judge model. If that exact snapshot is unavailable, use an available GPT-4-class judge; absolute scores can drift versus the published numbers because the judge model changed. -``` - -## Implementation Notes - -- The judge model is selected via `--judge-model-args`; ensure the model id supports long contexts (`meta_info` may exceed 4k tokens for dialogue tasks). -- Set `extra_params={'tasks': [...]}` to evaluate only specific Chat task names — useful for partial runs. - ## Properties diff --git a/docs/en/benchmarks/baby_vision.md b/docs/en/benchmarks/baby_vision.md index f4cbaa3b9..27f260c9f 100644 --- a/docs/en/benchmarks/baby_vision.md +++ b/docs/en/benchmarks/baby_vision.md @@ -30,7 +30,7 @@ recognition, and visual tracking. - Primary metric: **Accuracy** via LLM-as-judge - Subsets organized by `type` field (4 categories) - LLM judge evaluates both choice and blank answer types uniformly -- Requires `judge_model_args` configuration for LLM judge +- Requires an LLM judge configured through `judge.models` ## Properties @@ -48,30 +48,7 @@ recognition, and visual tracking. ## Data Statistics -| Metric | Value | -|--------|-------| -| Total Samples | 388 | -| Prompt Length (Mean) | 167.37 chars | -| Prompt Length (Min/Max) | 33 / 450 chars | - -**Per-Subset Statistics:** - -| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max | -|--------|---------|-------------|------------|------------| -| `Fine-grained Discrimination` | 163 | 152.09 | 33 | 450 | -| `Spatial Perception` | 91 | 157.18 | 73 | 370 | -| `Visual Pattern Recognition` | 51 | 178.92 | 94 | 319 | -| `Visual Tracking` | 83 | 201.45 | 97 | 389 | - -**Image Statistics:** - -| Metric | Value | -|--------|-------| -| Total Images | 388 | -| Images per Sample | min: 1, max: 1, mean: 1 | -| Resolution Range | 174x144 - 2378x1448 | -| Formats | jpeg, png, webp | - +*Statistics not available.* ## Sample Example @@ -81,13 +58,13 @@ recognition, and visual tracking. { "input": [ { - "id": "8b83904b", + "id": "1256c961", "content": [ { "image": "[BASE64_IMAGE: jpeg, ~77.0KB]" }, { - "text": "The image shows a total of 49 tiger patterns arranged in 7 rows and 7 columns. One of them is different from the others. Which row and column is it in? The answer format is (x,y). (For example, the answer for the 2nd row and 3rd column is (2,3))." + "text": "The image shows a total of 49 tiger patterns arranged in 7 rows and 7 columns. One of them is different from the others. Which row and column is it in? The answer format is (x,y). (For example, the answer for the 2nd row and 3rd column is (2,3)).\nThink about the question and give your final answer in \\boxed{Answer} format." } ] } @@ -134,11 +111,6 @@ task_cfg = TaskConfig( api_url='OPENAI_API_COMPAT_URL', api_key='EMPTY_TOKEN', datasets=['baby_vision'], - dataset_args={ - 'baby_vision': { - # subset_list: ['Fine-grained Discrimination', 'Spatial Perception', 'Visual Pattern Recognition'] # optional, evaluate specific subsets - } - }, limit=10, # Remove this line for formal evaluation ) diff --git a/docs/en/benchmarks/charxiv.md b/docs/en/benchmarks/charxiv.md index 00708fffa..477cec561 100644 --- a/docs/en/benchmarks/charxiv.md +++ b/docs/en/benchmarks/charxiv.md @@ -31,7 +31,7 @@ chart element perception (descriptive) and high-level reasoning about chart data - Each chart yields 5 samples: 4 descriptive + 1 reasoning - Primary metric: **Accuracy** via LLM-as-judge - Subsets: `descriptive` and `reasoning` (also by category) -- Requires `judge_model_args` configuration for LLM judge +- Requires an LLM judge configured through `judge.models` - [Paper](https://arxiv.org/abs/2406.18521) | [GitHub](https://github.com/princeton-nlp/CharXiv) @@ -50,28 +50,7 @@ chart element perception (descriptive) and high-level reasoning about chart data ## Data Statistics -| Metric | Value | -|--------|-------| -| Total Samples | 5,000 | -| Prompt Length (Mean) | 276.24 chars | -| Prompt Length (Min/Max) | 80 / 687 chars | - -**Per-Subset Statistics:** - -| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max | -|--------|---------|-------------|------------|------------| -| `descriptive` | 4,000 | 261.51 | 156 | 432 | -| `reasoning` | 1,000 | 335.14 | 80 | 687 | - -**Image Statistics:** - -| Metric | Value | -|--------|-------| -| Total Images | 5,000 | -| Images per Sample | min: 1, max: 1, mean: 1 | -| Resolution Range | 1023x139 - 1024x1024 | -| Formats | jpeg | - +*Statistics not available.* ## Sample Example @@ -81,7 +60,7 @@ chart element perception (descriptive) and high-level reasoning about chart data { "input": [ { - "id": "44ff5b8a", + "id": "dd588b4d", "content": [ { "image": "[BASE64_IMAGE: jpeg, ~70.0KB]" @@ -133,11 +112,6 @@ task_cfg = TaskConfig( api_url='OPENAI_API_COMPAT_URL', api_key='EMPTY_TOKEN', datasets=['charxiv'], - dataset_args={ - 'charxiv': { - # subset_list: ['descriptive', 'reasoning'] # optional, evaluate specific subsets - } - }, limit=10, # Remove this line for formal evaluation ) diff --git a/docs/en/benchmarks/count_qa.md b/docs/en/benchmarks/count_qa.md index 18b952d1f..b8b7e8282 100644 --- a/docs/en/benchmarks/count_qa.md +++ b/docs/en/benchmarks/count_qa.md @@ -34,7 +34,7 @@ well-separated objects. rule the paper states for its rewriter LLM. A reply with no digit scores 0, so `max_tokens` must leave the model room to reach its answer; a model that narrates its count ("row 1 has 3 ...") is scored on the first number it mentions rather than on its stated total -- Scoring is deterministic arithmetic and needs no LLM judge: keep `judge_strategy` at `rule` or +- Scoring is deterministic arithmetic and needs no LLM judge: keep `judge.strategy` at `rule` or `auto`, since `llm` replaces both metrics with a generic judge score. To read a different number out of a model that ignores the output format, prepend a per-run filter such as `filters={'regex': {'regex_pattern': '(\d+)', 'group_select': -1}}` (last number) via @@ -57,56 +57,11 @@ well-separated objects. ## Data Statistics -| Metric | Value | -|--------|-------| -| Total Samples | 1,528 | -| Prompt Length (Mean) | 467.52 chars | -| Prompt Length (Min/Max) | 459 / 508 chars | - -**Image Statistics:** - -| Metric | Value | -|--------|-------| -| Total Images | 1,528 | -| Images per Sample | min: 1, max: 1, mean: 1 | -| Resolution Range | 1714x178 - 2482x2500 | -| Formats | jpeg, png | - +*Statistics not available.* ## Sample Example -**Subset**: `default` - -```json -{ - "input": [ - { - "id": "dd3eebdf", - "content": "You are a helpful assistant that counts the number of items in an image. The user will provide an image and ask a question about the number of a certain type of item in the image. If the user question is referring to multiple objects, it means that you need to provide a sum of the number of items. You will count the number of items and return the number as an integer. Your output should STRICTLY be a single integer and nothing else." - }, - { - "id": "ba0d838c", - "content": [ - { - "image": "[BASE64_IMAGE: jpeg, ~202.2KB]" - }, - { - "text": "How many tiles are on the wall with the shower?" - } - ] - } - ], - "target": "18", - "id": 0, - "group_id": 0, - "metadata": { - "is_focused": false, - "categories": [ - "Outdoor, Vehicles & Structural" - ] - } -} -``` +*Sample example not available.* ## Prompt Template diff --git a/docs/en/benchmarks/hipho.md b/docs/en/benchmarks/hipho.md index 3da8cc508..39bdf72a5 100644 --- a/docs/en/benchmarks/hipho.md +++ b/docs/en/benchmarks/hipho.md @@ -28,8 +28,8 @@ range from text-only problems to diagram-based problems. ## Evaluation Notes -- Requires an LLM judge: run with `judge_strategy='llm'` (or `'auto'`, which enables the judge for this benchmark) - and provide `judge_model_args`. `judge_strategy='rule'` is not supported. +- Requires an LLM judge: set `judge.strategy='llm'` (or `'auto'`, which enables the judge for this benchmark) + and provide `judge.models`. `judge.strategy='rule'` is not supported. - Primary metric: `accuracy`, the per-problem awarded/attainable point ratio in `[0, 1]`, aggregated by mean per subset. For step-level problems the attainable maximum is the sum of the marking criteria; for problems with several official schemes (EuPhO, NBPhO) the highest-scoring scheme is used, matching the paper. @@ -102,7 +102,7 @@ range from text-only problems to diagram-based problems. { "input": [ { - "id": "9ec40cee", + "id": "41868500", "content": [ { "text": "You are participating in a high school physics Olympiad exam.\nPlease read the following question carefully and provide a clear, step-by-step solution with full reasoning.\nInstructions:\n1. Use LaTeX to format all variables, equations, and calc ... [TRUNCATED 3334 chars] ... gamma} R^{\\delta}$ \nwhere $G$ is the gravitational constant, and $\\beta, \\gamma$ and $\\delta$ are constant exponents.\nQuestion (Answer only the question stated below):\nFind the values of exponents: (1) $\\beta$, (2) $\\gamma$, and (3) $\\delta$." diff --git a/docs/en/benchmarks/job_bench.md b/docs/en/benchmarks/job_bench.md index 81300bb9f..23dce42ed 100644 --- a/docs/en/benchmarks/job_bench.md +++ b/docs/en/benchmarks/job_bench.md @@ -18,7 +18,7 @@ deliverables, and reconciling multi-source information. This adapter uses the Mo ## Evaluation Notes - The default evaluation split is `main`. -- Configure `judge_model_args` for rubric scoring. +- Configure `judge.models` for rubric scoring. - `normalized_score` is the primary weighted score (raw total / max score); `pass_rate` is the unweighted proportion of fully passed rubrics; `judge_score` is the raw sum of passed rubric weights, reported as a diagnostic. - Docker runs use `python:3.11-slim-bookworm` by default. For formal evaluation, provide an image with the Office, @@ -54,7 +54,7 @@ deliverables, and reconciling multi-source information. This adapter uses the Mo { "input": [ { - "id": "203fd1aa", + "id": "9cdb8ee0", "content": "You are preparing the Statistical Analysis Plan for Phase III trial XR-2847 evaluating cardiovascular disease prevention. The sponsor requires validation of statistical assumptions against empirical data and regulatory alignment before protoc ... [TRUNCATED 2026 chars] ... erables.\n\nWrite every final deliverable file under `jobbench_output`. Do not put intermediate scratch files there.\nYour final message may summarize what you produced, but files requested by the task must be actual files in\n`jobbench_output`.\n" } ], diff --git a/docs/en/benchmarks/mia_bench.md b/docs/en/benchmarks/mia_bench.md index 218e09bf8..de10de86c 100644 --- a/docs/en/benchmarks/mia_bench.md +++ b/docs/en/benchmarks/mia_bench.md @@ -24,8 +24,8 @@ MIA-Bench is a multimodal instruction-following benchmark designed to evaluate v - Default evaluation uses the **test** split (400 samples) - Primary metric: **judge_score** (mean of per-sample normalized 0–1 total scores) -- Requires a capable LLM judge (e.g., GPT-4o, Qwen-Max) configured via `judge_model_args` -- Judge strategy should be set to `JudgeStrategy.LLM` +- Requires a capable LLM judge (e.g., GPT-4o, Qwen-Max) configured through `judge.models` +- Judge strategy should be set to `judge.strategy='llm'` ## Properties @@ -67,7 +67,7 @@ MIA-Bench is a multimodal instruction-following benchmark designed to evaluate v { "input": [ { - "id": "9156e81c", + "id": "c09c1ebf", "content": [ { "image": "[BASE64_IMAGE: jpeg, ~195.9KB]" diff --git a/docs/en/benchmarks/perception_bench.md b/docs/en/benchmarks/perception_bench.md index 925c48913..b5302100c 100644 --- a/docs/en/benchmarks/perception_bench.md +++ b/docs/en/benchmarks/perception_bench.md @@ -38,7 +38,7 @@ single capability, so difficulty stems from perception rather than reasoning or (`[reason]` / `[judge] True|False`); the paper uses GPT-oss-120B, whose agreement with human judgment is 99.7% on a 300-sample audit - Empty or failed generations are scored 0 without invoking the judge -- Requires `judge_model_args` configuration for the LLM judge +- Requires an LLM judge configured through `judge.models` - The dataset embeds images as base64 data URIs (~1.6 GB download on first use) diff --git a/docs/en/benchmarks/phyx_mc.md b/docs/en/benchmarks/phyx_mc.md index 9c9b5c04f..b935a655f 100644 --- a/docs/en/benchmarks/phyx_mc.md +++ b/docs/en/benchmarks/phyx_mc.md @@ -35,7 +35,7 @@ with a figure and four answer options, and the model has to name the correct opt - Default scoring is the official string-level match: the chosen letter is extracted from the reply and compared with the ground truth, accepting replies that mark the correct option the way the prompt prints it (`D:`) or emphasises it (`**D**`). -- Setting `judge_strategy='llm'` (with `judge_model_args`) reproduces the official LLM-judged mode. +- Setting `judge.strategy='llm'` with `judge.models` reproduces the official LLM-judged mode. The judge is only consulted for replies whose option letter could not be extracted, matching upstream. - Figures are sent inline as base64 and the largest is ~5 MB; set `max_image_bytes` in `dataset_args` diff --git a/docs/en/benchmarks/phyx_oe.md b/docs/en/benchmarks/phyx_oe.md index 9c08bad7b..44afe07d3 100644 --- a/docs/en/benchmarks/phyx_oe.md +++ b/docs/en/benchmarks/phyx_oe.md @@ -36,9 +36,9 @@ answer of a university-level physics problem from the figure and state it. statement, else the whole reply is compared. A reply truncated before its answer therefore scores 0 for reasons unrelated to physics ability; give the model a generous `generation_config.max_tokens`. - Answers are free-form values with units, so an LLM judge is used by default (the official - recommendation): run with `judge_strategy='auto'` or `'llm'` and provide `judge_model_args`. The + recommendation): set `judge.strategy='auto'` or `'llm'` and provide `judge.models`. The judge is only consulted when the answer does not already match as a string. -- `judge_strategy='rule'` falls back to the official string-level mode, which understates accuracy +- `judge.strategy='rule'` falls back to the official string-level mode, which understates accuracy because equivalent spellings (`0.5 m` vs `50 cm`) do not match literally. - Figures are sent inline as base64 and the largest is ~5 MB; set `max_image_bytes` in `dataset_args` if the served model enforces a smaller per-image limit. diff --git a/docs/en/benchmarks/plawbench.md b/docs/en/benchmarks/plawbench.md index 72db62de9..beeaa8fa3 100644 --- a/docs/en/benchmarks/plawbench.md +++ b/docs/en/benchmarks/plawbench.md @@ -33,8 +33,8 @@ rubric rather than against a single reference answer. ## Evaluation Notes -- Requires an LLM judge: run with `judge_strategy='llm'` (or `'auto'`, which enables the judge for this benchmark) - and provide `judge_model_args`. `judge_strategy='rule'` is not supported. +- Requires an LLM judge: set `judge.strategy='llm'` (or `'auto'`, which enables the judge for this benchmark) + and provide `judge.models`. `judge.strategy='rule'` is not supported. - Metrics are point ratios in `[0, 1]`. `acc` is reported for every subset; `case_analysis` additionally reports `conclusion_acc`, `fact_acc`, `reasoning_acc`, and `law_acc`. These map one-to-one onto the official leaderboard columns: `legal_consultation` is Task1, `case_analysis` is Task2-Avg with its four dimensions, and the two @@ -47,10 +47,10 @@ rubric rather than against a single reference answer. `[0, max_points]`, so a judge that mis-reports the denominator cannot distort the score. - The judge output template for `case_analysis` is repaired relative to the official script, which ships malformed JSON and pins the conclusion section to zero points; every section is graded on its rubric allocation here. -- Judge requests are retried up to `judge_retries` times when the response cannot be parsed; a sample that still - fails is scored 0 and flagged via `judge_failed` in the review metadata. +- The judge model's transport retry policy is configured through its `generation_config`. A reply that still + fails the output contract is unavailable and excluded rather than silently scored as zero. - Case-analysis judging returns a long per-item breakdown. Give the judge a generous `max_tokens` - (for example 8192) in `judge_model_args.generation_config`. + (for example 8192) in `judge.models[].generation_config`. - The drafting subsets ask for a 2,500-3,000 character legal document, so the evaluated model also needs a generous `generation_config.max_tokens`. A truncated filing is graded as an incomplete document and scores near zero, which depresses Task3 for reasons unrelated to legal ability. @@ -126,12 +126,6 @@ Resources: [GitHub](https://github.com/skylenage/PLawbench) | {question} ``` -## Extra Parameters - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `judge_retries` | `int` | `3` | Maximum attempts per rubric judge request before the sample is scored as 0. | - ## Usage ### Using CLI @@ -159,7 +153,6 @@ task_cfg = TaskConfig( dataset_args={ 'plawbench': { # subset_list: ['case_analysis', 'legal_consultation', 'plaintiff_statement'] # optional, evaluate specific subsets - # extra_params: {} # uses default extra parameters } }, limit=10, # Remove this line for formal evaluation diff --git a/docs/en/benchmarks/researchrubrics.md b/docs/en/benchmarks/researchrubrics.md index 8994ac768..d4e5a102e 100644 --- a/docs/en/benchmarks/researchrubrics.md +++ b/docs/en/benchmarks/researchrubrics.md @@ -15,6 +15,15 @@ references, communication quality, and instruction following. - **Dataset**: 101 tasks and 2,593 weighted rubric criteria - **Metric**: Binary rubric compliance score +## Key Features + +- 101 open-ended Deep Research tasks paired with 2,593 expert-written, weighted rubric criteria. +- Rubrics cover explicit and implicit requirements, information synthesis, references, communication + quality, and instruction following, and each is graded independently. +- Negative-weight criteria capture undesirable behaviours and subtract from the score when present. +- Long reports are graded with the official chunk-evidence-synthesis procedure once they exceed the + configured judge context threshold. + ## Agent Environment - Uses EvalScope's built-in agent environment by default and does not require ``agent_config``. The agent can use ``bash`` @@ -30,7 +39,7 @@ references, communication quality, and instruction following. ## Evaluation Notes -- ResearchRubrics requires ``judge_model_args`` and ``judge_strategy='auto'`` or ``'llm'``. Gemini 2.5 Pro is the +- ResearchRubrics requires ``judge.models`` and ``judge.strategy='auto'`` or ``'llm'``. Gemini 2.5 Pro is the recommended judge for comparison with the paper, but no provider or model is hard-coded. - Every rubric is graded independently as Satisfied (1) or Not Satisfied (0), matching the public binary grader. The paper's ternary scores are not directly comparable. @@ -45,7 +54,6 @@ references, communication quality, and instruction following. - ``judge_context_limit``: 150,000 estimated tokens - ``judge_chunk_size``: 100,000 estimated tokens -- ``judge_retries``: 3 attempts per judge request The judge must be configured explicitly. For example: @@ -55,12 +63,14 @@ from evalscope import TaskConfig, run_task run_task(TaskConfig( model='YOUR_AGENT_MODEL', datasets=['researchrubrics'], - judge_strategy='llm', - judge_model_args={ - 'model_id': 'YOUR_JUDGE_MODEL', - 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL', - 'api_key': 'YOUR_JUDGE_API_KEY', - 'generation_config': {'temperature': 0.0}, + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'YOUR_JUDGE_MODEL', + 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL', + 'api_key': 'YOUR_JUDGE_API_KEY', + 'generation_config': {'temperature': 0.0}, + }, }, limit=1, )) @@ -154,7 +164,6 @@ Resources: [Paper](https://arxiv.org/abs/2511.07685) | |-----------|------|---------|-------------| | `judge_context_limit` | `int` | `150000` | Estimated token limit before rubric judging switches to chunking. | | `judge_chunk_size` | `int` | `100000` | Maximum estimated tokens in each document chunk sent to the judge. | -| `judge_retries` | `int` | `3` | Maximum attempts for each rubric judge request and JSON parse. | ## Usage diff --git a/docs/en/benchmarks/slake.md b/docs/en/benchmarks/slake.md index 82063468c..d9f08cb5b 100644 --- a/docs/en/benchmarks/slake.md +++ b/docs/en/benchmarks/slake.md @@ -42,7 +42,7 @@ and medical knowledge that has to be recalled on top of what the image shows. accuracy on the knowledge-graph questions is expected to be low - Strictness also costs answers that differ from the reference only in wording or granularity (`Right` answered as `Right Side`, 胸腔 answered as 胸部, one item of a multi-item reference). Set - `judge_strategy='llm_recall'` to have an LLM judge re-check only the rule-based failures; that + `judge.strategy='llm_recall'` to have an LLM judge re-check only the rule-based failures; that score is more lenient than the published numbers and is not directly comparable to them - Images ship as a single `imgs.zip` (about 200 MB) and are read directly from the archive - [Paper](https://arxiv.org/abs/2102.09542) | [Project page](https://www.med-vqa.com/slake/) diff --git a/docs/en/benchmarks/tir_bench.md b/docs/en/benchmarks/tir_bench.md index f31f3bbd8..46f7483a9 100644 --- a/docs/en/benchmarks/tir_bench.md +++ b/docs/en/benchmarks/tir_bench.md @@ -28,7 +28,7 @@ diverse task categories requiring spatial, compositional, and multi-step visual - Images are downloaded as `data.zip` from ModelScope and extracted automatically - Rule-based scoring: OCR (substring match), jigsaw (grid IoU), spot_difference (set IoU), word_search (numeric match), all other tasks (MCQ / numeric judge) -- **Recommended**: set `judge_strategy=JudgeStrategy.LLM_RECALL` and provide `judge_model_args` +- **Recommended**: set `judge.strategy='llm_recall'` and provide `judge.models` to activate LLM-as-judge as a recall mechanism — the judge is called only when rule-based scoring gives 0, providing more accurate evaluation without unnecessary API overhead - [Paper](https://arxiv.org/abs/2511.01833) | [GitHub](https://github.com/agents-x-project/TIR-Bench) diff --git a/docs/en/benchmarks/wide_search.md b/docs/en/benchmarks/wide_search.md index 2aee5f5a7..e776a68b3 100644 --- a/docs/en/benchmarks/wide_search.md +++ b/docs/en/benchmarks/wide_search.md @@ -23,7 +23,7 @@ atomic facts and return one structured Markdown table. EvalScope uses the ModelS ## Evaluation Notes - Uses the official Markdown table alignment and hybrid rule/LLM scoring semantics. -- Requires ``judge_strategy='auto'`` or ``'llm'`` with explicit ``judge_model_args``; rule-only scoring is unsupported. +- Requires ``judge.strategy='auto'`` or ``'llm'`` with at least one ``judge.models`` entry; rule-only scoring is unsupported. - See the [WideSearch usage guide](https://evalscope.readthedocs.io/en/latest/third_party/wide_search.html) for runtime examples and paper-style repeat settings. diff --git a/docs/en/best_practice/index_collection.md b/docs/en/best_practice/index_collection.md index 6d4c0b56f..e88bd0d68 100644 --- a/docs/en/best_practice/index_collection.md +++ b/docs/en/best_practice/index_collection.md @@ -170,10 +170,13 @@ task_cfg = TaskConfig( 'temperature': 0.0 # Usually set to 0 for reproducible results in evaluation }, # Use a powerful model as judge for automatic scoring (required for chinese_simpleqa) - judge_model_args={ - 'model_id': 'qwen3-max', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': os.getenv('DASHSCOPE_API_KEY'), + judge={ + 'strategy': 'auto', + 'models': { + 'model_id': 'qwen3-max', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': os.getenv('DASHSCOPE_API_KEY'), + }, }, eval_batch_size=5, # Eval concurrency, adjust according to actual situation ) @@ -359,4 +362,4 @@ Building evaluation indices is no longer exclusive to big companies or academia. 4. Use Case analysis to pinpoint model weaknesses -Don't be limited to general ranking scores anymore. Start building your own **Index** now and select the "best model" that truly fits your business! \ No newline at end of file +Don't be limited to general ranking scores anymore. Start building your own **Index** now and select the "best model" that truly fits your business! diff --git a/docs/en/best_practice/qwen3_coder.md b/docs/en/best_practice/qwen3_coder.md index 2ed849ebd..2bd9c35c5 100644 --- a/docs/en/best_practice/qwen3_coder.md +++ b/docs/en/best_practice/qwen3_coder.md @@ -123,10 +123,13 @@ task_cfg = TaskConfig( # limit=20, # Limit number of evaluations for quick tests; remove for full evaluation ignore_errors=True, # Ignore errors; some test cases may be refused by the model stream=True, # Enable streaming output - judge_model_args={ # Judge model configuration - 'model_id': 'qwen2.5-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': os.getenv('DASHSCOPE_API_KEY'), + judge={ + 'strategy': 'auto', + 'models': { # Judge model configuration + 'model_id': 'qwen2.5-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': os.getenv('DASHSCOPE_API_KEY'), + }, 'generation_config': { 'temperature': 0.0, 'max_tokens': 4096 diff --git a/docs/en/get_started/basic_usage.md b/docs/en/get_started/basic_usage.md index 1fc3c42d4..fc4db03bd 100644 --- a/docs/en/get_started/basic_usage.md +++ b/docs/en/get_started/basic_usage.md @@ -199,7 +199,6 @@ For certain subjective or open-ended tasks (such as `simple_qa`), you can specif ```python import os from evalscope import TaskConfig, run_task -from evalscope.constants import JudgeStrategy task_cfg = TaskConfig( model='qwen2.5-7b-instruct', @@ -208,11 +207,13 @@ task_cfg = TaskConfig( eval_type='openai_api', datasets=['chinese_simpleqa'], limit=5, - judge_strategy=JudgeStrategy.AUTO, - judge_model_args={ - 'model_id': 'qwen2.5-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': os.getenv('DASHSCOPE_API_KEY'), + judge={ + 'strategy': 'auto', + 'models': { + 'model_id': 'qwen2.5-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': os.getenv('DASHSCOPE_API_KEY'), + }, } ) @@ -305,7 +306,7 @@ evalscope eval \ ### Continue Evaluation from Existing/Interrupted Results -If a previous evaluation task was interrupted, or you want to continue evaluating based on existing results, you can use the `--use-cache` parameter to specify the previous output directory. This will skip completed samples and only evaluate the remaining ones. Additionally, you can use the `--rerun-review` parameter to force re-execution of the scoring step for all samples. +If a previous evaluation task was interrupted, or you want to continue evaluating based on existing results, you can use the `--use-cache` parameter to specify the previous output directory. This will skip completed samples and only evaluate the remaining ones. Additionally, `--rerun-review` re-scores cached predictions and atomically replaces the review cache after a successful run. ```shell evalscope eval \ diff --git a/docs/en/get_started/faq.md b/docs/en/get_started/faq.md index 010896df7..c3225d943 100644 --- a/docs/en/get_started/faq.md +++ b/docs/en/get_started/faq.md @@ -69,15 +69,15 @@ Note: most thinking models (Qwen3, DeepSeek series, etc.) prefill `` into --dataset-args '{"ifeval": {"filters": {"remove_until": ""}}}' ``` If your model uses different thinking tags like `<|end_of_thinking|>`, simply replace it accordingly. -3. **Fall back to a judge model**: set `judge_strategy=JudgeStrategy.LLM_RECALL` so samples that fail rule-based extraction are re-judged by the judge model. See "Abnormal Results & Troubleshooting" below. +3. **Fall back to a judge model**: set `judge={'strategy': 'llm_recall', 'models': {...}}` so rule-based misses are re-judged. See "Abnormal Results & Troubleshooting" below. **Q: How to use a local model as a Judge Model?** -**A:** You can deploy the local model as an API service using frameworks like vLLM, then specify its service address in `--judge-model-args`. +**A:** You can deploy the local model as an API service using frameworks like vLLM, then set its service address in `--judge` under `models`. **Q: How to set timeout for judge models?** -**A:** Set the `timeout` parameter in the `generation_config` of `--judge-model-args`. +**A:** Set `generation_config.timeout` for the relevant entry in `judge.models`. **Q: How to add custom request headers when evaluating API services?** @@ -127,11 +127,9 @@ task_config = TaskConfig( **A:** Math problem answer formats are complex, and rule-based parsing can't cover all cases. We recommend using LLM as an auxiliary judge to improve accuracy: ```python # Set in TaskConfig or DataAdapter -judge_strategy=JudgeStrategy.LLM_RECALL, -judge_model_args={ - 'model_id': 'qwen2.5-72b-instruct', - 'api_url': '...', - 'api_key': '...' +judge={ + 'strategy': 'llm_recall', + 'models': {'model_id': 'qwen2.5-72b-instruct', 'api_url': '...', 'api_key': '...'}, } ``` Reference documentation: [Judge Model Parameters](https://evalscope.readthedocs.io/zh-cn/latest/get_started/parameters.html#judge). @@ -143,14 +141,13 @@ Reference documentation: [Judge Model Parameters](https://evalscope.readthedocs. 2. Then add a judge model as a fallback so failed rule-based extractions are recalled: ```python # Set in TaskConfig - judge_strategy=JudgeStrategy.LLM_RECALL, - judge_model_args={'model_id': '...', 'api_url': '...', 'api_key': '...'} + judge={'strategy': 'llm_recall', 'models': {'model_id': '...', 'api_url': '...', 'api_key': '...'}} ``` `llm_recall` only calls the judge model when the rule-based score is not perfect, so it adds no across-the-board overhead. **Q: `Connection error` when evaluating `alpaca_eval`?** -**A:** `alpaca_eval` requires specifying a Judge Model for scoring. It uses OpenAI API by default, and connection will fail if related keys aren't configured. Please specify an available judge model through `--judge-model-args`. +**A:** `alpaca_eval` requires a Judge Model for scoring. It uses OpenAI API by default, and connection will fail if related keys are not configured. Specify one through `--judge`. **Q: How to continue from checkpoint after evaluation interruption?** @@ -285,4 +282,4 @@ You can also use `--data-source` to specify the data source (defaults to `models year={2024}, url={https://github.com/modelscope/evalscope} } -``` \ No newline at end of file +``` diff --git a/docs/en/get_started/parameters.md b/docs/en/get_started/parameters.md index 6d95ff464..9ecb73885 100644 --- a/docs/en/get_started/parameters.md +++ b/docs/en/get_started/parameters.md @@ -161,73 +161,59 @@ Refer to the [other backend usage guide](../user_guides/backend/index.md) ## Judge Parameters -LLM-as-a-Judge evaluation parameters using a judge model to determine correctness: - -| Parameter | Type | Description | Default | -|-----------|------|-------------|---------| -| `--judge-strategy` | `str` | Judge model strategy
• `auto`: Automatically decide based on dataset requirements
• `llm`: Always use judge model
• `rule`: Use rule-based judgment only
• `llm_recall`: Use judge model after rule-based judgment fails | `auto` | -| `--judge-worker-num` | `int` | **[Deprecated]** Use `--eval-batch-size` instead. Will be removed in v2.0.0. | `1` | -| `--judge-model-args` | `str` | Judge model configuration (JSON string), see table below | - | -| `--analysis-report` | `bool` | Whether to generate analysis report (language auto-detected) | `false` | - -### judge-model-args Configuration Options - -| Parameter | Type | Description | Default | -|-----------|------|-------------|---------| -| `api_key` | `str` | API key | Read from `MODELSCOPE_SDK_TOKEN`, default `EMPTY` | -| `api_url` | `str` | API endpoint | Read from `MODELSCOPE_API_BASE`,
default `https://api-inference.modelscope.cn/v1/` | -| `model_id` | `str` | Model ID | Read from `MODELSCOPE_JUDGE_LLM`,
default `Qwen/Qwen3-235B-A22B` | -| `system_prompt` | `str` | System prompt | - | -| `prompt_template` | `str` | Prompt template | Auto-selected based on `score_type` | -| `generation_config` | `dict` | Generation parameters (same as `--generation-config`) | - | -| `model_args` | `dict` | Judge model loading parameters (same as `--model-args`), e.g. `{"default_headers": {"X-API-KEY": "your-api-key"}}` | `{}` | -| `score_type` | `str` | Scoring method
• `pattern`: Judge if answer matches reference
• `numeric`: Score without reference (0-1) | `pattern` | -| `score_pattern` | `str` | Regex to parse output | `pattern` mode: `(A\|B)`
`numeric` mode: `\[\[(\d+(?:\.\d+)?)\]\]` | -| `score_mapping` | `dict` | Score mapping for `pattern` mode | `{'A': 1.0, 'B': 0.0}` | - -```{seealso} -For more information on ModelScope model inference services, refer to [ModelScope API Inference Services](https://modelscope.cn/docs/model-service/API-Inference/intro) -``` - -
pattern Mode Default Prompt Template - -```text -Your job is to look at a question, a gold target, and a predicted answer, and return a letter "A" or "B" to indicate whether the predicted answer is correct or incorrect. - -[Question] -{question} - -[Reference Answer] -{gold} - -[Predicted Answer] -{pred} - -Evaluate the model's answer based on correctness compared to the reference answer. -Grade the predicted answer of this new question as one of: -A: CORRECT -B: INCORRECT - -Just return the letters "A" or "B", with no text around it. +Native LLM judging is configured through one typed `judge` object. In Python/YAML use `judge={...}`; on the CLI use +`--judge ''`. + +```python +TaskConfig( + model='MODEL', + datasets=['simple_qa'], + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'JUDGE_MODEL', + 'api_url': 'OPENAI_COMPATIBLE_URL', + 'api_key': 'JUDGE_API_KEY', + 'generation_config': {'temperature': 0.0, 'retries': 3}, + }, + 'repeats': 1, + 'position_swap': 'auto', + 'aggregation': 'mean', + 'min_valid_judges': 1, + }, +) ``` -
- -
numeric Mode Default Prompt Template -```text -Please act as an impartial judge and evaluate the quality of the response provided by an AI assistant to the user question displayed below. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of the response. +`models` accepts one object or a list of objects. A list enables independent judges; every entry needs a unique +`judge_id` when the same `model_id` occurs more than once. `judge_id` defaults to a unique `model_id`. -Begin your evaluation by providing a short explanation. Be as objective as possible. - -After providing your explanation, you must rate the response on a scale of 0 (worst) to 1 (best) by strictly following this format: "[[rating]]", for example: "Rating: [[0.5]]" - -[Question] -{question} - -[Response] -{pred} -``` -
+| Field | Type | Description | Default | +|-------|------|-------------|---------| +| `strategy` | `auto\|rule\|llm\|llm_recall` | `auto` follows benchmark policy; `llm_recall` judges only rule-based misses and takes `max(rule, judge)`. | `auto` | +| `models` | `object\|list[object]` | One or more Judge model configurations. `model_id` is required for reproducible review caching. | `[]` | +| `repeats` | `int >= 1` | Independent verdict observations per Judge, distinct from transport retries. | `1` | +| `position_swap` | `auto\|on\|off` | `auto` preserves the benchmark's official position-swap policy. | `auto` | +| `aggregation` | `mean\|median\|majority_vote` | Cross-observation aggregation for ordinary metrics. | `mean` | +| `min_valid_judges` | `int >= 1` | Minimum valid Judge verdicts required for a metric. | `1` | + +Each entry in `models` supports `judge_id`, `model_id`, `api_key`, `api_url`, `eval_type`, `model_args`, and +`generation_config`. Provider-specific model initialization options belong in `model_args`; transport retry belongs +in `generation_config.retries`. + +`judge.contract` configures the generic single-verdict judge only: `system_prompt`, `prompt_template`, +`score_mapping`, and `score_type`. `pattern` asks the Judge for a JSON verdict label selected from `score_mapping`; +`numeric` asks for a JSON score in `[0, 1]`. The framework appends the JSON-format instruction, parses the normal +model response once, and never uses constrained decoding, regex score extraction, or corrective follow-up prompts. +An invalid reply is unavailable and excluded from the metric rather than reported as zero. + +Reports include `JudgeSummary` with coverage, failure counts, and disagreement for samples reviewed by an LLM. +When an adapter resolves a sample through a deterministic judge short-circuit, its score metadata records +`judge_skipped=true` and `judge_skip_reason`; the web review panel labels it as rule-based scoring instead of an +LLM verdict. With `rerun_review=True`, predictions are reused and the review cache is atomically replaced after +success. + +`judge_strategy` and a single mapping `judge_model_args` remain accepted only as a deprecated input migration. The +removed `judge_worker_num` and `score_pattern` are rejected. ## Sandbox Parameters @@ -275,7 +261,7 @@ For full usage, examples and Trace visualization, see [Agent Evaluation](../user | `--work-dir` | `str` | Evaluation output path (see directory structure below) | `./outputs` | | `--no-timestamp` | `bool` | Do not add timestamp to work_dir | `false` | | `--use-cache` | `str` | Reuse local cache path (e.g., `outputs/20241210_194434`)
Reuses inference and evaluation results | `None` | -| `--rerun-review` | `bool` | Used with `--use-cache`: deletes the existing reviews cache and re-runs the review/scoring stage while still reusing prediction cache | `false` | +| `--rerun-review` | `bool` | Used with `--use-cache`: re-runs review/scoring from cached predictions and atomically replaces the review cache after success | `false` | | `--enable-progress-tracker` | `bool` | Whether to enable progress tracking, writing hierarchical evaluation progress to `progress.json` in real time, queryable via the service API | `false` | | `--collect-perf` | `bool` | Collect per-request performance metrics (latency, TTFT, token usage) and write them into the evaluation report. TTFT requires `--generation-config stream=true`. Use `--no-collect-perf` to disable | `true` | | `--seed` | `int` | Random seed | `42` | diff --git a/docs/en/third_party/deepsearchqa.md b/docs/en/third_party/deepsearchqa.md index e2e429920..44657a79b 100644 --- a/docs/en/third_party/deepsearchqa.md +++ b/docs/en/third_party/deepsearchqa.md @@ -54,12 +54,14 @@ task_config = TaskConfig( datasets=['deepsearchqa'], eval_batch_size=1, limit=10, - judge_strategy='llm', - judge_model_args={ - 'model_id': 'YOUR_JUDGE_MODEL', - 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL', - 'api_key': os.getenv('JUDGE_API_KEY'), - 'generation_config': {'temperature': 0.0}, + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'YOUR_JUDGE_MODEL', + 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL', + 'api_key': os.getenv('JUDGE_API_KEY'), + 'generation_config': {'temperature': 0.0}, + }, }, agent_config=NativeAgentConfig( strategy='function_calling', @@ -101,8 +103,7 @@ evalscope eval \ --datasets deepsearchqa \ --limit 10 \ --eval-batch-size 1 \ - --judge-strategy llm \ - --judge-model-args '{"model_id":"YOUR_JUDGE_MODEL","api_url":"OPENAI_COMPATIBLE_JUDGE_URL","api_key":"'"$JUDGE_API_KEY"'","generation_config":{"temperature":0.0}}' \ + --judge '{"strategy":"llm","models":{"model_id":"YOUR_JUDGE_MODEL","api_url":"OPENAI_COMPATIBLE_JUDGE_URL","api_key":"'"$JUDGE_API_KEY"'","generation_config":{"temperature":0.0}}}' \ --agent-config '{"mode":"native","strategy":"function_calling","max_steps":30,"mcp_servers":[{"type":"stdio","name":"fetch","command":"python","args":["-m","mcp_server_fetch","--ignore-robots-txt"]},{"type":"http","name":"search","url":"https://mcp.api-inference.modelscope.net//mcp","headers":{"Authorization":"Bearer "}}],"kwargs":{"system_prompt":"Use search and fetch tools to research the question. Cross-check important facts before answering. When finished, call submit(answer=...) with only the final answer."}}' ``` diff --git a/docs/en/third_party/needle_haystack.md b/docs/en/third_party/needle_haystack.md index ed7eb75cd..fedcc1473 100644 --- a/docs/en/third_party/needle_haystack.md +++ b/docs/en/third_party/needle_haystack.md @@ -61,11 +61,11 @@ task_cfg = TaskConfig( 'max_tokens': 512, # Maximum number of tokens to generate }, eval_batch_size=5, - judge_model_args={ + judge={'models': { 'model_id': 'qwen2.5-72b-instruct', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': os.getenv('DASHSCOPE_API_KEY'), - } + }}, ) run_task(task_cfg=task_cfg) ``` @@ -160,13 +160,13 @@ task_cfg = TaskConfig( 'max_tokens': 512, # Maximum number of tokens to generate }, eval_batch_size=5, - judge_model_args={ + judge={'models': { 'model_id': 'qwen2.5-72b-instruct', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': os.getenv('DASHSCOPE_API_KEY'), - } + }}, ) run_task(task_cfg=task_cfg) ``` -The output example is similar to the one above. \ No newline at end of file +The output example is similar to the one above. diff --git a/docs/en/third_party/wide_search.md b/docs/en/third_party/wide_search.md index 8cddace3d..0f88df1aa 100644 --- a/docs/en/third_party/wide_search.md +++ b/docs/en/third_party/wide_search.md @@ -41,11 +41,13 @@ run_task(TaskConfig( api_key=os.getenv('MODEL_API_KEY'), eval_type='openai_api', datasets=['wide_search'], - judge_strategy='llm', - judge_model_args={ - 'model_id': 'YOUR_JUDGE_MODEL', - 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL', - 'api_key': os.getenv('JUDGE_API_KEY'), + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'YOUR_JUDGE_MODEL', + 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL', + 'api_key': os.getenv('JUDGE_API_KEY'), + }, 'generation_config': {'temperature': 0.0}, }, eval_batch_size=1, @@ -77,8 +79,7 @@ run_task(TaskConfig( 'network_enabled': True, }, ), - judge_strategy='llm', - judge_model_args={'model_id': 'YOUR_JUDGE_MODEL'}, + judge={'strategy': 'llm', 'models': {'model_id': 'YOUR_JUDGE_MODEL'}}, limit=1, )) ``` @@ -112,8 +113,7 @@ run_task(TaskConfig( ) ], ), - judge_strategy='llm', - judge_model_args={'model_id': 'YOUR_JUDGE_MODEL'}, + judge={'strategy': 'llm', 'models': {'model_id': 'YOUR_JUDGE_MODEL'}}, )) ``` diff --git a/docs/en/user_guides/aigc/image_edit.md b/docs/en/user_guides/aigc/image_edit.md index fca5335a4..51b222294 100644 --- a/docs/en/user_guides/aigc/image_edit.md +++ b/docs/en/user_guides/aigc/image_edit.md @@ -24,7 +24,7 @@ from dotenv import dotenv_values env = dotenv_values('.env') from evalscope import TaskConfig, run_task -from evalscope.constants import EvalType, JudgeStrategy, ModelTask +from evalscope.constants import EvalType, ModelTask task_config = TaskConfig( model='Qwen/Qwen-Image-Edit', # Model ID or local path @@ -50,15 +50,14 @@ task_config = TaskConfig( }, eval_batch_size=5, limit=5, - judge_strategy=JudgeStrategy.AUTO, - judge_model_args={ # Configure a VLM model for automatic scoring - 'model_id': 'qwen2.5-vl-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': env.get('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096, - } + judge={ + 'strategy': 'auto', + 'models': { # Configure a VLM model for automatic scoring + 'model_id': 'qwen2.5-vl-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': env.get('DASHSCOPE_API_KEY'), + 'generation_config': {'temperature': 0.0, 'max_tokens': 4096}, + }, }, ) @@ -173,7 +172,7 @@ from dotenv import dotenv_values env = dotenv_values('.env') from evalscope import TaskConfig, run_task -from evalscope.constants import EvalType, JudgeStrategy, ModelTask +from evalscope.constants import EvalType, ModelTask task_config = TaskConfig( model_id='offline-model', # No need to configure a model, just provide a custom ID @@ -191,15 +190,14 @@ task_config = TaskConfig( }, eval_batch_size=5, limit=5, - judge_strategy=JudgeStrategy.AUTO, - judge_model_args={ # Configure a VLM model for automatic scoring - 'model_id': 'qwen2.5-vl-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': env.get('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096, - } + judge={ + 'strategy': 'auto', + 'models': { # Configure a VLM model for automatic scoring + 'model_id': 'qwen2.5-vl-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': env.get('DASHSCOPE_API_KEY'), + 'generation_config': {'temperature': 0.0, 'max_tokens': 4096}, + }, }, ) @@ -247,4 +245,4 @@ Examples: *Score Distribution* ![image](https://sail-moe.oss-cn-hangzhou.aliyuncs.com/yunlin/images/evalscope/image_edit/compare.png) -*Effect Comparison* \ No newline at end of file +*Effect Comparison* diff --git a/docs/en/user_guides/arena.md b/docs/en/user_guides/arena.md index c0fd87ec5..bdb9210af 100644 --- a/docs/en/user_guides/arena.md +++ b/docs/en/user_guides/arena.md @@ -196,15 +196,12 @@ task_cfg = TaskConfig( } } }, - # Configure judge model parameters - judge_model_args={ + judge={'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': os.getenv('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 8000 - }, + 'generation_config': {'temperature': 0.0, 'max_tokens': 8000}, + }, }, eval_batch_size=5, # use_cache='outputs/xxx' # Optional: to add new candidate models to existing results, specify the existing results path diff --git a/docs/zh/advanced_guides/add_benchmark.md b/docs/zh/advanced_guides/add_benchmark.md index f19415eb9..50b2523fc 100644 --- a/docs/zh/advanced_guides/add_benchmark.md +++ b/docs/zh/advanced_guides/add_benchmark.md @@ -47,7 +47,7 @@ DataAdapter采用Pipeline架构，支持通过钩子方法自定义行为。以` calculate_metrics() ├── filter_prediction() │ └── extract_answer() [用户可选实现] - ├── match_score() / llm_match_score() + ├── match_score() / score_with_judge_contracts() └── 返回 SampleScore 4. 结果聚合阶段 @@ -61,6 +61,31 @@ DataAdapter采用Pipeline架构，支持通过钩子方法自定义行为。以` └── 返回 Report ``` +### 接入 LLM Judge Benchmark + +使用者通过 [Judge 参数](../get_started/parameters.md#judge参数) 中的 typed `judge` 配置启用判别： + +```python +TaskConfig( + model='MODEL_UNDER_TEST', + datasets=['your_benchmark'], + judge={ + 'strategy': 'llm', + 'models': {'model_id': 'JUDGE_MODEL', 'api_url': 'OPENAI_COMPATIBLE_URL', 'api_key': 'API_KEY'}, + }, +) +``` + +对于 adapter 开发者，judge 的 I/O 统一由 `evalscope.api.judge` 处理；不要在 adapter 中调用 `self.llm_judge.judge()`，也不要自行解析模型回复。 + +1. 声明 `scoring_policy`：规则评分没有意义时使用 `JUDGE_ONLY`；规则评分仍可用、但 `auto` 应使用 Judge 时使用 `JUDGE_DEFAULT`；`auto` 应保留规则评分时使用 `RULE_DEFAULT`。 +2. 实现 adapter 唯一的入口 `judge_definition(context)`。普通的单 verdict 任务返回带有 Pydantic verdict schema 的 `JudgeDefinition.labels(...)` 或 `JudgeDefinition.numeric(...)`。这些 helper 会追加 JSON 输出要求，并将 prompt、schema 与指标映射保持在同一处。 +3. 对 rubric、多 claim 或分阶段任务，定义 Pydantic verdict schema 和 `OutputContract`，然后返回 `JudgeDefinition.workflow(cases=..., request=..., reduce=...)`。只有工作流需要时才传入 `expand=...`、`fallback=...` 或 `finalize=...`。这些 callback 可以嵌套在 `judge_definition()` 中，也可以作为 adapter 的私有 helper，但必须由返回的 definition 持有，不能再作为 adapter hook 暴露。自定义 request 中应追加 `case.output_contract.instruction()`；仅当官方固定 JSON 要求与 schema 完全一致时才可保留官方格式。通过 `CaseVerdict.metadata` 传递上下文，不要把状态编码到 `case_id`。 +4. 当确定性的规则无需调用模型即可判定样本时，返回 `JudgeDefinition.skip(score, reason='...')`。`reason` 为必填字段，会以 `Score.metadata['judge_skip_reason']` 及 `Score.metadata['judge_skipped'] = True` 保存；Web review 面板会将其标为规则直接判分，而非 LLM verdict。 +5. 添加 scripted judge 测试，覆盖有效 JSON verdict、错误 JSON/自然语言回复和 transport error。无效 Judge 回复会从指标中排除，不会记为 0，executor 也不会自动纠错重试。 + +executor 负责请求调度、位置交换、重复次数、多 Judge quorum、聚合和 review 诊断；模型 transport 的重试策略由 `generation_config` 负责。 + ### 核心数据结构 #### 1. Sample对象 diff --git a/docs/zh/advanced_guides/custom_dataset/llm.md b/docs/zh/advanced_guides/custom_dataset/llm.md index fbe77287a..e26215b27 100644 --- a/docs/zh/advanced_guides/custom_dataset/llm.md +++ b/docs/zh/advanced_guides/custom_dataset/llm.md @@ -239,14 +239,18 @@ run_task(task_cfg=task_cfg) **方法2. 基于LLM的评测** -基于LLM的评测可以方便的评测模型的正确性（或其他维度的指标，需要设置自定义的prompt）。下面是一个示例，配置了`judge_model_args`相关的参数，使用预置的`pattern`模式来判断模型输出是否正确。 +基于 LLM 的评测可以方便地评测模型的正确性（或其他维度的指标，需要设置自定义 prompt）。下面是一个使用预置 `pattern` JSON 契约判断模型输出是否正确的示例。 完整的judge参数说明请[参考文档](../../get_started/parameters.md#judge参数)。 +```{note} +裁判模型以单个 JSON 对象回复。无法解析为 JSON 的回复会被**排除**在指标之外而不是记为 0 分， +因此 `Num` 可能小于样本数。自定义 `prompt_template` 只需描述打分标准，回复格式要求会自动追加。 +``` + ```python import os from evalscope import TaskConfig, run_task -from evalscope.constants import JudgeStrategy task_cfg = TaskConfig( model='Qwen/Qwen2.5-0.5B-Instruct', @@ -261,22 +265,18 @@ task_cfg = TaskConfig( ], } }, - # judge 相关参数 - judge_model_args={ - 'model_id': 'qwen2.5-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': os.getenv('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096 + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'qwen2.5-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': os.getenv('DASHSCOPE_API_KEY'), + 'generation_config': {'temperature': 0.0, 'max_tokens': 4096}, }, - # 根据参考答案和模型输出，判断模型输出是否正确 - 'score_type': 'pattern', + 'contract': {'score_type': 'pattern'}, }, # 评测并发数 eval_batch_size=5, - # 使用 LLM 进行评测 - judge_strategy=JudgeStrategy.LLM, ) run_task(task_cfg=task_cfg) @@ -297,13 +297,12 @@ run_task(task_cfg=task_cfg) 若数据集没有参考答案，可以使用LLM裁判来评测模型输出的答案，不配置LLM将不会有打分结果。 -下面是一个示例，配置了`judge_model_args`相关的参数，使用预置的`numeric`模式，从准确性、相关性、有用性等维度，自动综合判断模型输出得分，分数越高表示模型输出越好。 +下面是一个使用预置 `numeric` JSON 契约的示例，从准确性、相关性、有用性等维度自动综合判断模型输出得分，分数越高表示模型输出越好。 完整的judge参数说明请[参考文档](../../get_started/parameters.md#judge参数)。 ```python import os from evalscope import TaskConfig, run_task -from evalscope.constants import JudgeStrategy task_cfg = TaskConfig( model='Qwen/Qwen2.5-0.5B-Instruct', @@ -318,22 +317,18 @@ task_cfg = TaskConfig( ], } }, - # judge 相关参数 - judge_model_args={ - 'model_id': 'qwen2.5-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': os.getenv('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096 + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'qwen2.5-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': os.getenv('DASHSCOPE_API_KEY'), + 'generation_config': {'temperature': 0.0, 'max_tokens': 4096}, }, - # 直接打分 - 'score_type': 'numeric', + 'contract': {'score_type': 'numeric'}, }, # 评测并发数 eval_batch_size=5, - # 使用 LLM 进行评测 - judge_strategy=JudgeStrategy.LLM, ) run_task(task_cfg=task_cfg) diff --git a/docs/zh/advanced_guides/custom_dataset/vlm.md b/docs/zh/advanced_guides/custom_dataset/vlm.md index 64962b275..0bba6066b 100644 --- a/docs/zh/advanced_guides/custom_dataset/vlm.md +++ b/docs/zh/advanced_guides/custom_dataset/vlm.md @@ -266,7 +266,7 @@ evalscope eval \ ```python from evalscope.run import run_task -from evalscope.constants import EvalType, JudgeStrategy +from evalscope.constants import EvalType from os import environ as env task_cfg = TaskConfig( @@ -282,17 +282,16 @@ task_cfg = TaskConfig( } }, limit=5, - judge_model_args={ - 'model_id': 'qwen-plus', # 无需是多模态模型 - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': env.get('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096 + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'qwen-plus', # 无需是多模态模型 + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': env.get('DASHSCOPE_API_KEY'), + 'generation_config': {'temperature': 0.0, 'max_tokens': 4096}, }, }, eval_batch_size=5, - judge_strategy=JudgeStrategy.LLM, ) result = run_task(task_cfg=task_cfg) ``` @@ -307,9 +306,8 @@ evalscope eval \ --datasets general_vqa \ --dataset-args '{"general_vqa": {"local_path": "custom_eval/multimodal/vqa", "subset_list": ["example_openai"]}}' \ --limit 5 \ - --judge-model-args '{"model_id": "qwen-plus", "api_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", "api_key": "$DASHSCOPE_API_KEY", "generation_config": {"temperature": 0.0, "max_tokens": 4096}}' \ - --judge-worker-num 5 \ - --judge-strategy llm + --eval-batch-size 5 \ + --judge '{"strategy": "llm", "models": {"model_id": "qwen-plus", "api_url": "https://dashscope.aliyuncs.com/compatible-mode/v1", "api_key": "$DASHSCOPE_API_KEY", "generation_config": {"temperature": 0.0, "max_tokens": 4096}}}' ``` 评测将输出准确率指标（输出指标较ROUGE和BLEU更直观）： diff --git a/docs/zh/benchmarks/air_bench_chat.md b/docs/zh/benchmarks/air_bench_chat.md index aef3fbc9f..df002a4a5 100644 --- a/docs/zh/benchmarks/air_bench_chat.md +++ b/docs/zh/benchmarks/air_bench_chat.md @@ -3,57 +3,40 @@ ## 概述 -AIR-Bench Chat 是 [AIR-Bench](https://arxiv.org/abs/2402.07729)（Audio InstRuction Benchmark，ACL 2024 主会）的生成式部分——这是首个面向大音频语言模型（LALMs）的指令遵循基准测试，涵盖**人类语音、自然声音和音乐**。该数据集包含约 2,000 个开放式音频问答对，覆盖语音、声音、音乐及混合音频场景；模型的回答由 GPT-4 作为裁判，对照参考答案进行评分。 +AIR-Bench Chat 是 [AIR-Bench](https://arxiv.org/abs/2402.07729)（Audio InstRuction Benchmark，ACL 2024 主会）的生成式部分——这是首个面向大音频语言模型（LALMs）的指令遵循基准测试，涵盖**人类语音、自然声音和音乐**。该数据集包含约 2,000 个开放式音频问答对，覆盖语音、声音、音乐及混合音频场景；模型的回答由 GPT-4 评判器根据参考答案进行评分。 ## 任务描述 - **任务类型**：开放式音频问答。 - **输入**：一段音频片段加一个自由形式的问题。 -- **输出**：一段文本答案，将根据参考答案进行评估。 +- **输出**：文本答案，与参考答案进行对比评估。 +- **模态**：音频（人类语音、自然声音、音乐）+ 文本。 -## 类别（8 项任务 → 5 个报告类别） +## 核心特性 -官方 `cal_score.py` 脚本将 8 项 Chat 任务聚合为以下五个类别： +- 包含约 2k 个开放式音频问答对，覆盖语音、声音、音乐及混合音频场景；是 AIR-Bench（ACL 2024）的生成式部分。 +- 官方 `cal_score.py` 将 8 个 Chat 任务聚合为 5 个报告类别：`speech`（`speech_QA`, `speech_dialogue_QA`）、`sound`（`sound_QA`, `sound_generation_QA`）、`music`（`music_QA`, `music_generation_analysis_QA`）、`speech_and_sound`（`speech_and_sound_QA`）、`speech_and_music`（`speech_and_music_QA`）。论文中的 Mixed-audio = mean(speech_and_sound, speech_and_music)。 +- 通过将每个样本判断两次（交换参考答案与模型预测的顺序）并取平均值来消除位置偏差（可通过设置 `extra_params={'do_swap': False}` 禁用此机制，以节省一半评判成本）。 +- 数据托管于 ModelScope（[`evalscope/AIR-Bench-Dataset`](https://modelscope.cn/datasets/evalscope/AIR-Bench-Dataset)），采用 audiofolder + JSON 格式；完整数据集约 49 GB，可通过 `extra_params={'tasks': [...]}` 限制任务范围以进行部分运行。 -- `speech`（语音）: `speech_QA`, `speech_dialogue_QA` -- `sound`（声音）: `sound_QA`, `sound_generation_QA` -- `music`（音乐）: `music_QA`, `music_generation_analysis_QA` -- `speech_and_sound`（语音与声音）: `speech_and_sound_QA` -- `speech_and_music`（语音与音乐）: `speech_and_music_QA` +## 评估说明 -论文中的 **Mixed-audio（混合音频）= mean(speech_and_sound, speech_and_music)**。 - -## 数据集获取 - -- 该数据集托管在 ModelScope 上：[`evalscope/AIR-Bench-Dataset`](https://modelscope.cn/datasets/evalscope/AIR-Bench-Dataset)。采用 *audiofolder + JSON metadata* 的布局方式。evalscope 在首次运行时通过 `modelscope.dataset_snapshot_download` 按需下载；完整数据集约 49 GB，建议通过 `extra_params` 参数限制仅拉取所需任务。 -- 如果数据集已存在于本地磁盘，请传入 `dataset_args={'air_bench_chat': {'local_path': '/path/to/AIR-Bench-Dataset'}}`；本地根目录应包含 `Chat/` 文件夹。 - -## 评估协议 - -- 裁判 LLM（默认为 GPT-4）接收问题、音频的文本描述（来自数据集的 `meta_info`）、参考答案（`answer_gt`）以及模型的回答，并输出一行包含两个 `[1, 10]` 范围内的整数分数。 -- 为消除位置偏差，每个样本会被评估两次，分别交换参考答案与模型预测的顺序，然后取平均分。此做法与官方仓库中的 `cal_score.py` 一致；可通过设置 `extra_params={'do_swap': False}` 禁用此行为，以节省一半的裁判成本。 -- 报告指标 `judge_score` 表示模型的平均裁判分数；`win_rate` 记录模型严格优于参考答案的频率。 - -```{warning} -官方排行榜使用 `gpt-4-0125-preview` 作为裁判模型。如果该特定快照不可用，请使用其他可用的 GPT-4 级别裁判模型；由于裁判模型变更，绝对分数可能与已发表结果存在偏差。 -``` - -## 实现说明 - -- 通过 `--judge-model-args` 指定裁判模型；请确保所选模型支持长上下文（对话类任务的 `meta_info` 可能超过 4k tokens）。 -- 设置 `extra_params={'tasks': [...]}` 可仅评估指定的 Chat 任务名称——适用于部分运行场景。 +- **指标**：`judge_score` 表示模型的平均评判分数；`win_rate` 记录模型严格优于参考答案的频率。 +- 评判 LLM 接收问题、音频的文本描述（`meta_info`）、参考答案（`answer_gt`）和模型回答，并输出两个 `[1, 10]` 范围内的整数分数。由于对话类任务的 `meta_info` 可能超过 4k tokens，请使用支持长上下文的评判模型。 +- 官方排行榜使用 `gpt-4-0125-preview`。若该特定快照不可用，请使用其他可用的 GPT-4 级别评判模型；由于评判模型变更，绝对分数可能与已发表结果存在偏差。 +- 若数据集已下载至本地，可通过 `dataset_args={'air_bench_chat': {'local_path': '/path/to/AIR-Bench-Dataset'}}` 指定路径；本地根目录应包含 `Chat/` 文件夹。 ## 属性 | 属性 | 值 | |----------|-------| | **基准测试名称** | `air_bench_chat` | -| **数据集 ID** | [evalscope/AIR-Bench](https://modelscope.cn/datasets/evalscope/AIR-Bench/summary) | +| **数据集ID** | [evalscope/AIR-Bench](https://modelscope.cn/datasets/evalscope/AIR-Bench/summary) | | **论文** | [Paper](https://aclanthology.org/2024.acl-long.109/) | | **标签** | `Audio`, `InstructionFollowing`, `QA` | | **指标** | `judge_score`, `win_rate` | | **默认示例数** | 0-shot | -| **评估分割** | `test` | +| **评估划分** | `test` | ## 数据统计 @@ -66,7 +49,7 @@ AIR-Bench Chat 是 [AIR-Bench](https://arxiv.org/abs/2402.07729)（Audio InstRuc **各子集统计信息：** -| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 | +| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 | |--------|---------|-------------|------------|------------| | `speech_QA` | 400 | 64.33 | 23 | 148 | | `speech_dialogue_QA` | 400 | 77.03 | 29 | 206 | @@ -133,7 +116,7 @@ AIR-Bench Chat 是 [AIR-Bench](https://arxiv.org/abs/2402.07729)（Audio InstRuc | 参数 | 类型 | 默认值 | 描述 | |-----------|------|---------|-------------| | `tasks` | `list` | `None` | 可选的 Chat 任务名称列表（子集包括 ['music_QA', 'music_generation_analysis_QA', 'sound_QA', 'sound_generation_QA', 'speech_QA', 'speech_and_music_QA', 'speech_and_sound_QA', 'speech_dialogue_QA']）。默认评估所有任务。 | -| `do_swap` | `bool` | `True` | 若为 True（默认），每个样本会评判两次（交换参考答案与模型预测的顺序），然后取平均分。禁用可节省一半裁判成本，但会引入位置偏差。 | +| `do_swap` | `bool` | `True` | 若为 True（默认），每个样本会被评判两次（交换参考答案与模型预测的顺序），然后取平均分。禁用此选项可节省一半评判成本，但会引入位置偏差。 | ## 使用方法 diff --git a/docs/zh/benchmarks/baby_vision.md b/docs/zh/benchmarks/baby_vision.md index e32d2aac3..ece5648dd 100644 --- a/docs/zh/benchmarks/baby_vision.md +++ b/docs/zh/benchmarks/baby_vision.md @@ -3,7 +3,7 @@ ## 概述 -BabyVision 是一个视觉感知基准测试，通过受婴儿及幼儿视觉发展启发的任务，评估多模态大语言模型的基础视觉能力。该基准聚焦于细粒度辨别、空间感知、视觉模式识别和视觉追踪能力。 +BabyVision 是一个视觉感知基准测试，通过受婴儿及幼儿视觉发展启发的任务，评估多模态大语言模型的基础视觉能力。该基准聚焦于细粒度辨别、空间感知、视觉模式识别和视觉追踪。 ## 任务描述 @@ -15,18 +15,18 @@ BabyVision 是一个视觉感知基准测试，通过受婴儿及幼儿视觉发 ## 主要特点 - 包含 388 个测试样本，覆盖 4 大视觉能力类别和 22 个子类型 -- 两种答案类型：选择题（135 个样本）和填空题（253 个样本） +- 两种答案类型：选择题（135 个样本）和填空题（235 个样本） - 子类型包括：找不同、找相同、数簇、迷宫、立方体展开图、图案补全、折纸、旋转图案等 -- 侧重于低层次视觉感知，而非高层次推理或知识 +- 测试低层次视觉感知能力，而非高层次推理或知识 - 提供思维链（Chain-of-Thought, CoT）参考用于分析 ## 评估说明 - 默认评估使用 **train** 切分（388 个样本，单切分数据集） - 主要指标：通过 LLM-as-judge 计算的 **准确率（Accuracy）** -- 子集按 `type` 字段组织（4 个类别） +- 子集按 `type` 字段组织（共 4 个类别） - LLM 评判器统一评估选择题和填空题两种答案类型 -- 需要配置 `judge_model_args` 以启用 LLM 评判器 +- 需通过 `judge.models` 配置 LLM 评判器 ## 属性 @@ -43,30 +43,7 @@ BabyVision 是一个视觉感知基准测试，通过受婴儿及幼儿视觉发 ## 数据统计 -| 指标 | 值 | -|--------|-------| -| 总样本数 | 388 | -| 提示词长度（平均） | 167.37 字符 | -| 提示词长度（最小/最大） | 33 / 450 字符 | - -**各子集统计信息：** - -| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 | -|--------|---------|-------------|------------|------------| -| `Fine-grained Discrimination` | 163 | 152.09 | 33 | 450 | -| `Spatial Perception` | 91 | 157.18 | 73 | 370 | -| `Visual Pattern Recognition` | 51 | 178.92 | 94 | 319 | -| `Visual Tracking` | 83 | 201.45 | 97 | 389 | - -**图像统计信息：** - -| 指标 | 值 | -|--------|-------| -| 图像总数 | 388 | -| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 | -| 分辨率范围 | 174x144 - 2378x1448 | -| 格式 | jpeg, png, webp | - +*统计数据不可用。* ## 样例示例 @@ -76,13 +53,13 @@ BabyVision 是一个视觉感知基准测试，通过受婴儿及幼儿视觉发 { "input": [ { - "id": "8b83904b", + "id": "1256c961", "content": [ { "image": "[BASE64_IMAGE: jpeg, ~77.0KB]" }, { - "text": "The image shows a total of 49 tiger patterns arranged in 7 rows and 7 columns. One of them is different from the others. Which row and column is it in? The answer format is (x,y). (For example, the answer for the 2nd row and 3rd column is (2,3))." + "text": "The image shows a total of 49 tiger patterns arranged in 7 rows and 7 columns. One of them is different from the others. Which row and column is it in? The answer format is (x,y). (For example, the answer for the 2nd row and 3rd column is (2,3)).\nThink about the question and give your final answer in \\boxed{Answer} format." } ] } @@ -107,7 +84,7 @@ BabyVision 是一个视觉感知基准测试，通过受婴儿及幼儿视觉发 ## 使用方法 -### 通过命令行（CLI） +### 使用 CLI ```bash evalscope eval \ @@ -118,7 +95,7 @@ evalscope eval \ --limit 10 # 正式评估时请删除此行 ``` -### 通过 Python +### 使用 Python ```python from evalscope import run_task @@ -129,11 +106,6 @@ task_cfg = TaskConfig( api_url='OPENAI_API_COMPAT_URL', api_key='EMPTY_TOKEN', datasets=['baby_vision'], - dataset_args={ - 'baby_vision': { - # subset_list: ['Fine-grained Discrimination', 'Spatial Perception', 'Visual Pattern Recognition'] # 可选，用于评估特定子集 - } - }, limit=10, # 正式评估时请删除此行 ) diff --git a/docs/zh/benchmarks/charxiv.md b/docs/zh/benchmarks/charxiv.md index d9b575cea..976d1238a 100644 --- a/docs/zh/benchmarks/charxiv.md +++ b/docs/zh/benchmarks/charxiv.md @@ -3,7 +3,7 @@ ## 概述 -CharXiv 是 NeurIPS 2024 提出的一个全面的图表理解基准测试，用于评估多模态大语言模型在来自 arXiv 论文的真实科学图表上的表现。该基准同时考察模型对图表元素的低层次感知能力（描述性任务）和对图表数据的高层次推理能力。 +CharXiv 是 NeurIPS 2024 提出的一个全面的图表理解基准测试，用于评估多模态大语言模型在来自 arXiv 论文的真实科学图表上的表现。该基准同时考察模型对图表元素的底层感知能力（描述性任务）和对图表数据的高层推理能力。 ## 任务描述 @@ -17,11 +17,11 @@ CharXiv 是 NeurIPS 2024 提出的一个全面的图表理解基准测试，用 - 包含来自 8 个学科 arXiv 论文的 2,323 张真实科学图表 - 两种问题类型： - **描述性**（每张图表 4 个）：基础元素识别（标题、坐标轴、图例、趋势等） - - **推理型**（每张图表 1 个）：需要综合数据进行高阶推理 + - **推理型**（每张图表 1 个）：需要数据综合的高阶推理 - 19 种描述性问题模板，涵盖信息提取、枚举、模式识别、计数和组合性任务 - 4 种推理答案类型：图表内文本、通用文本、图表内数值、通用数值 - 验证集（1,000 张图表）和测试集（1,323 张图表） -- 依据官方 CharXiv 评分协议，通过 LLM 作为评判者进行评估 +- 依据官方 CharXiv 评分协议，通过 LLM 作为评判器进行评估 ## 评估说明 @@ -29,7 +29,7 @@ CharXiv 是 NeurIPS 2024 提出的一个全面的图表理解基准测试，用 - 每张图表生成 5 个样本：4 个描述性 + 1 个推理型 - 主要指标：通过 LLM-as-judge 计算的 **准确率（Accuracy）** - 子集划分：`descriptive` 和 `reasoning`（也可按类别细分） -- 需配置 `judge_model_args` 以指定 LLM 评判模型 +- 需通过 `judge.models` 配置 LLM 评判器 - [论文](https://arxiv.org/abs/2406.18521) | [GitHub](https://github.com/princeton-nlp/CharXiv) @@ -48,28 +48,7 @@ CharXiv 是 NeurIPS 2024 提出的一个全面的图表理解基准测试，用 ## 数据统计 -| 指标 | 值 | -|--------|-------| -| 总样本数 | 5,000 | -| 提示词长度（平均） | 276.24 字符 | -| 提示词长度（最小/最大） | 80 / 687 字符 | - -**各子集统计数据：** - -| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 | -|--------|---------|-------------|------------|------------| -| `descriptive` | 4,000 | 261.51 | 156 | 432 | -| `reasoning` | 1,000 | 335.14 | 80 | 687 | - -**图像统计数据：** - -| 指标 | 值 | -|--------|-------| -| 图像总数 | 5,000 | -| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 | -| 分辨率范围 | 1023x139 - 1024x1024 | -| 格式 | jpeg | - +*统计数据暂不可用。* ## 样例示例 @@ -79,7 +58,7 @@ CharXiv 是 NeurIPS 2024 提出的一个全面的图表理解基准测试，用 { "input": [ { - "id": "44ff5b8a", + "id": "dd588b4d", "content": [ { "image": "[BASE64_IMAGE: jpeg, ~70.0KB]" @@ -109,7 +88,7 @@ CharXiv 是 NeurIPS 2024 提出的一个全面的图表理解基准测试，用 ## 使用方法 -### 使用 CLI +### 通过 CLI 使用 ```bash evalscope eval \ @@ -120,7 +99,7 @@ evalscope eval \ --limit 10 # 正式评估时请删除此行 ``` -### 使用 Python +### 通过 Python 使用 ```python from evalscope import run_task @@ -131,11 +110,6 @@ task_cfg = TaskConfig( api_url='OPENAI_API_COMPAT_URL', api_key='EMPTY_TOKEN', datasets=['charxiv'], - dataset_args={ - 'charxiv': { - # subset_list: ['descriptive', 'reasoning'] # 可选，用于评估特定子集 - } - }, limit=10, # 正式评估时请删除此行 ) diff --git a/docs/zh/benchmarks/count_qa.md b/docs/zh/benchmarks/count_qa.md index 3fc0642d1..4bb5853d7 100644 --- a/docs/zh/benchmarks/count_qa.md +++ b/docs/zh/benchmarks/count_qa.md @@ -3,20 +3,20 @@ ## 概述 -CountQA 探究物体计数这一基本感知能力，而多模态模型在此方面大多未经充分评估。该数据集的图像均在日常环境中手工拍摄，并刻意包含高密度物体、杂乱背景和遮挡情况，使得仅靠检测少量分离良好的物体无法完成计数任务。 +CountQA 用于测试物体计数能力，这是一种基础的感知技能，而多模态模型在此方面基本未经过充分评估。该基准的数据集图像均在日常环境中手工拍摄，并刻意包含高密度物体、杂乱背景和遮挡情况，使得仅靠检测少量分离良好的物体无法完成计数任务。 ## 任务描述 - **任务类型**：自由形式视觉问答（物体计数） - **输入**：一张真实世界照片 + 一个计数问题（例如：“有多少件夹克？”） - **输出**：一个整数 -- **领域**：日常场景——食品杂货、厨房用具、工具、衣物、办公及户外物品 +- **领域**：日常场景——包括食品杂货、厨房用具、工具、衣物、办公用品及户外物品 -## 关键特性 +## 主要特点 - 包含 1,528 个问答对，覆盖 1,001 张图像；每张图像可能对应多个问题 - 真实计数值在拍摄过程中现场标注（而非事后标注），范围从 0 到 400 -- 问题包括组合型问题，需对多种物体类型分别计数后求和 +- 问题中包含组合型问题，需对多种物体类型分别计数后求和 - 约一半图像属于杂乱场景而非聚焦于单一主体（每个样本元数据中标记为 ``is_focused``），场景类别记录在 ``categories`` 字段中 ## 评估说明 @@ -24,9 +24,9 @@ CountQA 探究物体计数这一基本感知能力，而多模态模型在此方 - 默认评估使用 **test** 划分作为单一子集 - 主要指标：**Accuracy** (`accuracy`) —— 与真实整数值完全匹配 - 次要指标：**relaxed_acc** —— 论文中定义的宽松准确率，当预测值在真实值的 ±5% 范围内即视为正确 -- 使用论文中的系统提示词原样不变；该提示词强制模型仅输出纯整数 -- 答案解析规则：若回复本身是整数则直接采用；否则取其中第一个整数——此规则与论文中用于重写答案的大语言模型一致。若回复不含任何数字，则得分为 0，因此 `max_tokens` 必须为模型留出足够空间输出答案；若模型以叙述方式计数（如“第一行有 3 个……”），则以其提到的第一个数字评分，而非其最终陈述的总数 -- 评分基于确定性算术规则，无需大语言模型裁判：请保持 `judge_strategy` 为 `rule` 或 `auto`，因为 `llm` 会用通用裁判分数替代上述两个指标。若需从忽略输出格式的模型回复中提取不同数字，可通过 `dataset_args` 添加运行时过滤器（例如 `filters={'regex': {'regex_pattern': '(\d+)', 'group_select': -1}}` 提取最后一个数字），而非修改适配器 +- 使用论文中的系统提示词原样不变；该提示强制模型仅输出一个纯整数 +- 答案解析规则：若回复本身是整数则直接采用；否则提取其中第一个整数——此规则与论文中重写器 LLM 所用规则一致。若回复中不含任何数字，则得分为 0，因此 `max_tokens` 必须为模型留出足够空间输出答案；若模型以叙述方式计数（如“第一行有 3 个……”），则以其提到的第一个数字评分，而非其最终陈述的总数 +- 评分基于确定性算术运算，无需 LLM 评判器：请保持 `judge.strategy` 为 `rule` 或 `auto`，因为若设为 `llm` 将会用通用评判分数替代上述两个指标。若需从忽略输出格式的模型回复中提取不同数字，可通过 `dataset_args` 添加运行时过滤器（例如 `filters={'regex': {'regex_pattern': '(\d+)', 'group_select': -1}}` 提取最后一个数字），而非修改适配器 - [论文](https://arxiv.org/abs/2508.06585) @@ -40,62 +40,17 @@ CountQA 探究物体计数这一基本感知能力，而多模态模型在此方 | **论文** | [Paper](https://arxiv.org/abs/2508.06585) | | **标签** | `MultiModal`, `QA`, `Reasoning` | | **指标** | `accuracy`, `relaxed_acc` | -| **默认示例数** | 0-shot | +| **默认示例数量** | 0-shot | | **评估划分** | `test` | ## 数据统计 -| 指标 | 值 | -|--------|-------| -| 总样本数 | 1,528 | -| 提示词长度（平均） | 467.52 字符 | -| 提示词长度（最小/最大） | 459 / 508 字符 | - -**图像统计信息：** - -| 指标 | 值 | -|--------|-------| -| 总图像数 | 1,528 | -| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 | -| 分辨率范围 | 1714x178 - 2482x2500 | -| 格式 | jpeg, png | - +*统计数据暂不可用。* ## 样例示例 -**子集**: `default` - -```json -{ - "input": [ - { - "id": "dd3eebdf", - "content": "You are a helpful assistant that counts the number of items in an image. The user will provide an image and ask a question about the number of a certain type of item in the image. If the user question is referring to multiple objects, it means that you need to provide a sum of the number of items. You will count the number of items and return the number as an integer. Your output should STRICTLY be a single integer and nothing else." - }, - { - "id": "ba0d838c", - "content": [ - { - "image": "[BASE64_IMAGE: jpeg, ~202.2KB]" - }, - { - "text": "How many tiles are on the wall with the shower?" - } - ] - } - ], - "target": "18", - "id": 0, - "group_id": 0, - "metadata": { - "is_focused": false, - "categories": [ - "Outdoor, Vehicles & Structural" - ] - } -} -``` +*样例示例暂不可用。* ## 提示模板 diff --git a/docs/zh/benchmarks/hipho.md b/docs/zh/benchmarks/hipho.md index 6d30342fe..b7cb4b80e 100644 --- a/docs/zh/benchmarks/hipho.md +++ b/docs/zh/benchmarks/hipho.md @@ -3,32 +3,32 @@ ## 概述 -HiPhO 是首个专注于高中物理奥林匹克竞赛并采用人类对齐评估的基准测试。它汇集了13场近期（2024-2025年）的国际及区域性奥赛试题，涵盖多种模态，包括纯文本题目和基于图表的题目。 +HiPhO 是首个专注于高中物理奥林匹克竞赛并采用人类对齐评估的基准测试。它汇集了 13 场近期（2024–2025 年）的国际及区域性奥赛试题，涵盖多种模态，包括纯文本题目和基于图表的题目。 ## 任务描述 - **任务类型**：自由形式的物理问题求解，依据官方评分标准进行评分 - **输入**：一道物理题（包含常数表、上下文和问题），可选附带图表 -- **输出**：分步解答，最终答案用 `...` 包裹并放入 `\boxed{}` 中 -- **模态**：纯文本 和 文本+图表（示意图 / 变量图 / 数据图） +- **输出**：逐步推导的解答，最终答案用 `...` 包裹并以 `\boxed{}` 标注 +- **模态**：纯文本，以及文本+图表（示意图 / 变量图 / 数据图） -## 核心特性 +## 主要特性 -- 共计403道题目，来自14套试卷（IPhO、APhO、EuPhO、NBPhO、PanPhO、PanMechanics、CPhO、F=MA），每套试卷作为一个独立子集。 -- 英语试卷使用英文提示，中文试卷（CPhO、PanMechanics）使用中文提示，遵循官方语言对应关系。 +- 共计 403 道题目，来自 14 套试卷（IPhO、APhO、EuPhO、NBPhO、PanPhO、PanMechanics、CPhO、F=MA），每套试卷作为一个独立子集。 +- 英文试卷使用英文提示，中文试卷（CPhO、PanMechanics）使用中文提示，遵循官方语言对应关系。 - 复现论文中的两种评分机制，按题目分别应用： - - **步骤级评分**：适用于提供官方评分细则的题目，LLM 评委对每个评分项打分，总分为各项得分之和。 - - **答案级评分**：适用于无评分细则的题目，通过基于规则的数学检查匹配 `\boxed{}` 中的最终答案；若规则检查失败，则回退至 LLM 评委判断。 + - **步骤级评分**：适用于提供官方评分标准的题目，LLM 评委会对每个评分项打分，并将得分累加。 + - **答案级评分**：适用于无评分标准的题目，通过基于规则的数学检查匹配 `\boxed{}` 中的最终答案；若规则检查失败，则由 LLM 评委会作为后备方案。 ## 评估说明 -- 需要 LLM 评委：运行时需设置 `judge_strategy='llm'`（或 `'auto'`，该选项会为此基准自动启用评委），并提供 `judge_model_args`。不支持 `judge_strategy='rule'`。 +- 需要 LLM 评委会：设置 `judge.strategy='llm'`（或 `'auto'`，该选项会为此基准自动启用评委会），并提供 `judge.models`。不支持 `judge.strategy='rule'`。 - 主要指标：`accuracy`，即每道题得分与满分之比（范围 `[0, 1]`），按子集取平均值聚合。 - - 对于步骤级题目，满分为所有评分项分数之和； + - 对于步骤级题目，满分为各评分项分数之和； - 对于拥有多个官方评分方案的题目（如 EuPhO、NBPhO），采用得分最高的方案，与论文一致。 -- 报告的是每场考试的标准化得分，**不计算**论文中金/银/铜牌的分数线，因为这需要原始总分和官方截止分数。 -- 解答可能很长，且图表题需要视觉输入；请为被测模型设置较大的 `generation_config.max_tokens`。若解答在 `` 块前被截断，则无法提取答案，得分接近零，但这与物理能力无关。 -- 图表以内联 base64 形式发送，最大约 1.5 MB；若所用模型对单图有更小限制，请在 `dataset_args` 中设置 `max_image_bytes`。 +- 本基准报告的是每场考试的标准化得分，**不计算**论文中金/银/铜牌的分数线，后者需要原始总分和官方截断值。 +- 解答可能很长，且图表题需要视觉输入；请为被测模型设置较大的 `generation_config.max_tokens`。若解答在 `` 块前被截断，则无法提取最终答案，导致得分接近零，这与模型的物理能力无关。 +- 图表以内联 base64 形式发送，最大约 1.5 MB；若所用模型对单图有更小的限制，请在 `dataset_args` 中设置 `max_image_bytes`。 - 资源链接：[论文](https://arxiv.org/abs/2509.07894) | [GitHub](https://github.com/SciYu/HiPhO) | [排行榜](https://phyarena.github.io/) @@ -90,7 +90,7 @@ HiPhO 是首个专注于高中物理奥林匹克竞赛并采用人类对齐评 { "input": [ { - "id": "9ec40cee", + "id": "41868500", "content": [ { "text": "You are participating in a high school physics Olympiad exam.\nPlease read the following question carefully and provide a clear, step-by-step solution with full reasoning.\nInstructions:\n1. Use LaTeX to format all variables, equations, and calc ... [TRUNCATED 3334 chars] ... gamma} R^{\\delta}$ \nwhere $G$ is the gravitational constant, and $\\beta, \\gamma$ and $\\delta$ are constant exponents.\nQuestion (Answer only the question stated below):\nFind the values of exponents: (1) $\\beta$, (2) $\\gamma$, and (3) $\\delta$." @@ -129,7 +129,7 @@ HiPhO 是首个专注于高中物理奥林匹克竞赛并采用人类对齐评 } ``` -*注：部分内容因展示需要已被截断。* +*注：部分内容为显示目的已截断。* ## 提示模板 diff --git a/docs/zh/benchmarks/job_bench.md b/docs/zh/benchmarks/job_bench.md index 76fadac3d..e5987e3ce 100644 --- a/docs/zh/benchmarks/job_bench.md +++ b/docs/zh/benchmarks/job_bench.md @@ -3,20 +3,20 @@ ## 概述 -JobBench 评估智能体系统在真实专业工作场景中的表现，这些任务需要阅读参考文件、生成交付成果，并整合多源信息。此适配器使用 ModelScope 数据集 `evalscope/job-bench`。 +JobBench 评估智能体系统在真实专业工作场景中的表现，这些任务要求阅读参考文件、生成交付物，并整合多源信息。本适配器使用 ModelScope 数据集 `evalscope/job-bench`。 ## 任务描述 -- **任务类型**：智能体专业工作 / 交付成果生成 +- **任务类型**：智能体专业工作 / 交付物生成 - **输入**：职场风格的任务提示，可选包含 `reference_files/` 目录下的参考文件 -- **输出**：最终交付成果文件写入 `jobbench_output/` 目录 +- **输出**：最终交付物文件写入 `jobbench_output/` 目录 - **数据集**：ModelScope `evalscope/job-bench` - **评估指标**：加权 LLM 评分标准得分（`normalized_score`） ## 评估说明 -- 默认评估划分（split）为 `main`。 -- 配置 `judge_model_args` 用于评分标准打分。 +- 默认评估划分是 `main`。 +- 配置 `judge.models` 用于评分标准打分。 - `normalized_score` 是主要的加权得分（原始总分 / 满分）；`pass_rate` 是完全通过评分项的未加权比例；`judge_score` 是通过评分项权重的原始总和，仅作为诊断指标报告。 - Docker 运行默认使用 `python:3.11-slim-bookworm` 镜像。正式评估时，请提供包含任务所需 Office、PDF 和电子表格工具的镜像。 @@ -49,7 +49,7 @@ JobBench 评估智能体系统在真实专业工作场景中的表现，这些 { "input": [ { - "id": "203fd1aa", + "id": "9cdb8ee0", "content": "You are preparing the Statistical Analysis Plan for Phase III trial XR-2847 evaluating cardiovascular disease prevention. The sponsor requires validation of statistical assumptions against empirical data and regulatory alignment before protoc ... [TRUNCATED 2026 chars] ... erables.\n\nWrite every final deliverable file under `jobbench_output`. Do not put intermediate scratch files there.\nYour final message may summarize what you produced, but files requested by the task must be actual files in\n`jobbench_output`.\n" } ], diff --git a/docs/zh/benchmarks/mia_bench.md b/docs/zh/benchmarks/mia_bench.md index d7db901e1..6888009d1 100644 --- a/docs/zh/benchmarks/mia_bench.md +++ b/docs/zh/benchmarks/mia_bench.md @@ -3,7 +3,7 @@ ## 概述 -MIA-Bench 是一个专为评估视觉-语言模型而设计的多模态指令遵循基准测试，重点考察模型在图像基础上遵循复杂、组合式指令的能力。每个样本包含一张图像和一条多组件指令，模型的回答将由大语言模型（LLM）评委按组件分别评分。 +MIA-Bench 是一个**多模态指令遵循基准测试**，旨在评估视觉-语言模型在图像基础上遵循复杂、组合式指令的能力。每个样本包含一张图像和一个多组件指令，模型的响应由大语言模型（LLM）评委按组件分别打分。 ## 任务描述 @@ -12,20 +12,20 @@ MIA-Bench 是一个专为评估视觉-语言模型而设计的多模态指令遵 - **输出**：自由形式的回答，需满足所有指令组件的要求 - **领域**：视觉理解、指令遵循、语言生成 -## 主要特点 +## 核心特性 - 包含 400 个测试样本，涵盖从基础到高级的多样化指令类型 - 每条指令可分解为 1–5 个评分组件，并配有加权分数 -- 组件类型包括：描述（describe）、长度限制（length_limit）、语言学要求（linguistics）、格式（format）等 +- 组件类型包括：describe（描述）、length_limit（长度限制）、linguistics（语言学要求）、format（格式）等 - 采用 LLM 作为评委进行评分：评委独立评估每个组件，并给出加权总分（0–10 分范围，归一化至 0–1） -- 不提供预定义的标准答案；评分完全依赖评委判断 +- **无预设参考答案**；评分完全依赖评委判断 ## 评估说明 - 默认使用 **test** 划分（400 个样本）进行评估 -- 主要指标：**judge_score**（各样本归一化总分的平均值，范围 0–1） -- 需配置一个能力强的 LLM 评委（例如 GPT-4o、Qwen-Max），通过 `judge_model_args` 参数指定 -- 评委策略应设置为 `JudgeStrategy.LLM` +- 主要指标：**judge_score**（各样本归一化总分 [0–1] 的平均值） +- 需配置一个能力强的 LLM 评委（例如 GPT-4o、Qwen-Max），通过 `judge.models` 设置 +- 评委策略应设为 `judge.strategy='llm'` ## 属性 @@ -36,7 +36,7 @@ MIA-Bench 是一个专为评估视觉-语言模型而设计的多模态指令遵 | **论文** | N/A | | **标签** | `InstructionFollowing`, `MultiModal`, `QA` | | **指标** | `judge_score` | -| **默认示例数量** | 0-shot | +| **默认示例数** | 0-shot | | **评估划分** | `test` | @@ -52,7 +52,7 @@ MIA-Bench 是一个专为评估视觉-语言模型而设计的多模态指令遵 | 指标 | 值 | |--------|-------| -| 总图像数 | 400 | +| 图像总数 | 400 | | 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 | | 分辨率范围 | 135x240 - 3264x4928 | | 格式 | jpeg, mpo, webp | @@ -66,7 +66,7 @@ MIA-Bench 是一个专为评估视觉-语言模型而设计的多模态指令遵 { "input": [ { - "id": "9156e81c", + "id": "c09c1ebf", "content": [ { "image": "[BASE64_IMAGE: jpeg, ~195.9KB]" diff --git a/docs/zh/benchmarks/perception_bench.md b/docs/zh/benchmarks/perception_bench.md index a277fb8a7..5a76da4b7 100644 --- a/docs/zh/benchmarks/perception_bench.md +++ b/docs/zh/benchmarks/perception_bench.md @@ -3,7 +3,7 @@ ## 概述 -PerceptionBench 是由 Moonshot AI 提出的一项基准测试，用于评估多模态大语言模型（MLLM）的原子级视觉感知能力。该基准采用自底向上的构建方式：通过对前沿 MLLM 在 42 个现有基准上最早出现的失败点进行诊断，归纳出一个错误分类体系，其中感知分支定义了十种原子级感知能力。每个问题仅聚焦于单一能力，因此其难度来源于感知本身，而非推理或知识。 +PerceptionBench 是由 Moonshot AI 推出的一项基准测试，用于评估多模态大语言模型（MLLM）的原子级视觉感知能力。该基准采用自下而上的构建方式：通过对前沿 MLLM 在 42 个现有基准上的最早失败点进行诊断，归纳出一个错误分类体系，其中“感知”分支定义了十种原子级感知能力。每个问题仅聚焦于单一能力，因此其难度来源于感知本身，而非推理或知识。 ## 任务描述 @@ -12,21 +12,21 @@ PerceptionBench 是由 Moonshot AI 提出的一项基准测试，用于评估多 - **输出**：自由格式的简短答案，且具有唯一确定的参考答案 - **领域**：涵盖十种原子级视觉感知能力 -## 主要特性 +## 核心特性 - 包含 3,000 个经过验证的问题，覆盖十种原子级感知能力 - 其中 1,800 个问题（60%）是从源基准中归因失败案例分解出的原子子问题；1,200 个问题（40%）是在补充图像上全新编写的问题 -- 子集对应十种 `error_category` 标签：视觉关系、计数、属性、深度与 3D 感知、定位、比较、细粒度识别、上下文整合、OCR 和感知相关幻觉 -- 支持多图像问题：图像通过 `<|image_N|>` 占位符嵌入到问题中 -- 对于带有 `hint`（坐标约定或图像尺寸）的样本，会以系统消息形式传递，与官方消息构建器保持一致 +- 子集遵循十种 `error_category` 标签：视觉关系、计数、属性、深度与 3D 感知、定位、比较、细粒度识别、上下文整合、OCR 和感知相关幻觉 +- 支持多图像问题：图像通过 `<|image_N|>` 占位符交错嵌入问题中 +- 对于包含 `hint`（坐标约定或图像尺寸）的样本，会通过系统消息传递该提示，与官方消息构建器保持一致 ## 评估说明 - 默认评估使用 **train** 切分（3,000 个样本，单切分数据集） - 主要指标：**准确率（Accuracy）**，报告整体及各能力维度的结果 -- 评分遵循官方协议：使用 LLM 评判器，根据教师评分提示（teacher-grading prompt）将自由格式答案与参考答案对比，并对每个样本返回严格的 0/1 判定（`[reason]` / `[judge] True|False`）；论文中使用的 GPT-oss-120B 模型在 300 个样本的人工审核中与人类判断的一致性达 99.7% -- 空输出或生成失败的样本直接记为 0 分，不调用评判器 -- 需要配置 `judge_model_args` 以指定 LLM 评判器 +- 评分遵循官方协议：使用 LLM 评判器，通过教师评分提示（teacher-grading prompt）将自由格式答案与参考答案对比，并对每个样本返回严格的 0/1 判定（`[reason]` / `[judge] True|False`）；论文中使用 GPT-oss-120B 作为评判器，在 300 个样本的人工审核中与人类判断的一致性达 99.7% +- 空输出或生成失败的答案直接记为 0 分，不调用评判器 +- 需通过 `judge.models` 配置 LLM 评判器 - 数据集中图像以 base64 data URI 形式嵌入（首次使用时下载约 1.6 GB） ## 属性 @@ -114,7 +114,7 @@ PerceptionBench 是由 Moonshot AI 提出的一项基准测试，用于评估多 ## 使用方法 -### 使用 CLI +### 通过 CLI 使用 ```bash evalscope eval \ @@ -125,7 +125,7 @@ evalscope eval \ --limit 10 # 正式评估时请删除此行 ``` -### 使用 Python +### 通过 Python 使用 ```python from evalscope import run_task diff --git a/docs/zh/benchmarks/phyx_mc.md b/docs/zh/benchmarks/phyx_mc.md index 3bf83f971..6040506a9 100644 --- a/docs/zh/benchmarks/phyx_mc.md +++ b/docs/zh/benchmarks/phyx_mc.md @@ -7,35 +7,34 @@ PhyX 是首个面向现实、视觉接地场景中物理推理的大规模基准 ## 任务描述 -- **任务类型**：视觉多项选择物理问题求解 +- **任务类型**：视觉多选物理问题求解 - **输入**：一张图像，以及问题描述、提问和四个带标签的选项 - **输出**：单个选项字母（A、B、C 或 D） -- **领域**：大学水平物理（力学、电磁学、热力学、波动/声学、光学、现代物理） +- **领域**：大学物理（力学、电磁学、热力学、波动/声学、光学、近代物理） ## 核心特性 -- 包含 3,000 道大学水平题目（`test`），覆盖 6 个核心领域和 25 个子领域，每个领域作为独立子集提供；设置 `eval_split='test_mini'` 可选用官方 1,000 题的 testmini 测试集。 +- 包含 3,000 道大学水平题目（`test`），覆盖 6 个核心领域和 25 个子领域，每个领域作为独立子集提供；设置 `eval_split='test_mini'` 可选择官方指定的 1,000 题 testmini 测试集。 - 每道题均基于一张图像，其中包含文本未重复说明的关键信息，因此模型必须结合视觉线索与隐含的物理定律进行推理。 - 涵盖 6 种推理类型（物理模型接地、多公式、空间关系、数值、预测性及隐含条件推理）。 -- 采用论文默认的 *Text-DeRedundancy* 输入格式：简化后的问题描述加提问，并附上图像。 +- 采用论文中默认的 *Text-DeRedundancy* 输入风格：简化后的问题描述加提问，并附上图像。 -- 官方提示词被逐字复现，包括要求仅输出选项字母的指令，以确保评估结果与已发表数据具有可比性。 +- 官方提示词被逐字复现，包括要求仅输出选项字母的指令，以确保评分结果可与已发表数据直接比较。 ## 评估说明 -- 主要指标：`acc`（准确率），对所有题目取平均值，并分别报告整体及各领域的得分。 -- 默认评分方式为官方字符串级匹配：从模型回复中提取选项字母并与标准答案比较，接受如 `D:` 或 `**D**` 等符合提示格式的正确选项标记形式。 -- 设置 `judge_strategy='llm'`（配合 `judge_model_args`）可复现官方 LLM 判定模式。仅当无法从回复中提取选项字母时，才调用判别模型，与上游实现一致。 -- 图像以内联 base64 形式发送，最大约 5 MB；若所用模型对单张图像有更小的大小限制，请在 `dataset_args` 中设置 `max_image_bytes`。 +- 主要指标：`acc`（准确率），对所有题目取平均值，并报告整体及各领域的得分。 +- 默认评分方式为官方字符串级匹配：从模型回复中提取选项字母，并与标准答案比对，接受如 `D:` 或 `**D**` 等符合提示格式的正确选项标记形式。 +- 若设置 `judge.strategy='llm'` 并指定 `judge.models`，则复现官方 LLM 判定模式。仅当无法从回复中提取选项字母时，才调用判别模型，与上游实现一致。 +- 图像以内联 base64 形式发送，最大约 5 MB；若所用模型对单图大小有限制，可在 `dataset_args` 中设置 `max_image_bytes`。 - 资源链接：[论文](https://arxiv.org/abs/2505.15929) | [GitHub](https://github.com/NastyMarcus/PhyX) | [项目主页](https://killthefullmoon.github.io/projects/PhyX/index.html) - ## 属性 | 属性 | 值 | |----------|-------| | **基准测试名称** | `phyx_mc` | -| **数据集ID** | [evalscope/PhyX](https://modelscope.cn/datasets/evalscope/PhyX/summary) | +| **数据集 ID** | [evalscope/PhyX](https://modelscope.cn/datasets/evalscope/PhyX/summary) | | **论文** | [Paper](https://arxiv.org/abs/2505.15929) | | **标签** | `MCQ`, `MultiModal`, `Reasoning` | | **指标** | `accuracy` | diff --git a/docs/zh/benchmarks/phyx_oe.md b/docs/zh/benchmarks/phyx_oe.md index 9cc4a67e5..38cccbc26 100644 --- a/docs/zh/benchmarks/phyx_oe.md +++ b/docs/zh/benchmarks/phyx_oe.md @@ -3,7 +3,7 @@ ## 概述 -PhyX 是首个面向现实、视觉接地场景中物理推理的大规模基准测试。这是其开放式变体：不提供选项，模型必须从图示中推导出大学水平物理问题的答案并直接陈述。 +PhyX 是首个面向现实、视觉接地场景中物理推理的大规模基准测试。这是其开放式变体：不提供选项，模型必须从图示中推导出大学水平物理问题的答案并明确陈述。 ## 任务描述 @@ -12,9 +12,9 @@ PhyX 是首个面向现实、视觉接地场景中物理推理的大规模基准 - **输出**：包含逐步推导过程并以最终答案（带单位的数值或公式）结尾的解答 - **领域**：大学水平物理（力学、电磁学、热力学、波动/声学、光学、现代物理） -## 核心特性 +## 关键特性 -- 包含 3,000 道大学水平题目（`test`），覆盖 6 个核心领域和 25 个子领域，每个领域作为独立子集提供；使用 `eval_split='test_mini'` 可选择官方指定的 1,000 题 testmini 测试集。 +- 包含 3,000 道大学水平题目（`test`），覆盖 6 个核心领域和 25 个子领域，每个领域作为独立子集提供；设置 `eval_split='test_mini'` 可选择官方的 1,000 题 testmini 测试集。 - 每道题均基于一张图示，其中包含文本未重复说明的信息，因此模型必须结合视觉线索与隐含的物理定律进行推理。 - 涵盖 6 种推理类型（物理模型接地、多公式、空间关系、数值、预测性及隐含条件推理）。 - 采用论文默认的 *Text-DeRedundancy* 输入风格：简化后的问题描述加提问，并附上图示。 @@ -24,9 +24,9 @@ PhyX 是首个面向现实、视觉接地场景中物理推理的大规模基准 ## 评估说明 - 主要指标：`acc`，即各题目的平均准确率，整体及按领域分别报告。 -- 最终答案从 `\boxed{...}` 中提取；若无，则从 'final answer:' / 'correct answer:' 语句中提取；否则将整个回复内容用于比对。若回复在给出答案前被截断，则得分为 0（此情况与模型物理能力无关）；请为模型设置足够大的 `generation_config.max_tokens`。 -- 答案为自由格式的带单位数值，因此默认使用 LLM 评判器（官方推荐）：运行时设置 `judge_strategy='auto'` 或 `'llm'` 并提供 `judge_model_args`。仅当答案字符串不完全匹配时才调用评判器。 -- 若使用 `judge_strategy='rule'`，则回退到官方的字符串级匹配模式，但由于等效写法（如 `0.5 m` 与 `50 cm`）无法字面匹配，会导致准确率被低估。 +- 最终答案从 `\boxed{...}` 中提取；若无，则从 'final answer:' / 'correct answer:' 语句中提取；若仍无，则将整个回复与标准答案比较。若回复在给出答案前被截断，则得分为 0（此情况与模型物理能力无关）；请为模型设置足够大的 `generation_config.max_tokens`。 +- 答案为自由格式的带单位数值，因此默认使用 LLM 评判器（官方推荐）：设置 `judge.strategy='auto'` 或 `'llm'` 并提供 `judge.models`。仅当答案字符串不完全匹配时才调用评判器。 +- 若设置 `judge.strategy='rule'`，则回退到官方的字符串级匹配模式，但会低估准确率，因为等效表达（如 `0.5 m` 与 `50 cm`）在字面上并不相同。 - 图像以内联 base64 形式发送，最大约 5 MB；若所用模型对单张图像有更小的大小限制，请在 `dataset_args` 中设置 `max_image_bytes`。 - 资源链接：[论文](https://arxiv.org/abs/2505.15929) | [GitHub](https://github.com/NastyMarcus/PhyX) | [项目主页](https://killthefullmoon.github.io/projects/PhyX/index.html) diff --git a/docs/zh/benchmarks/plawbench.md b/docs/zh/benchmarks/plawbench.md index cd4209ea6..6ccf85ac5 100644 --- a/docs/zh/benchmarks/plawbench.md +++ b/docs/zh/benchmarks/plawbench.md @@ -3,36 +3,36 @@ ## 概述 -PLawBench 是一个基于评分标准（rubric）的基准测试，用于评估大语言模型在真实中国法律实务场景中的表现。 +PLawBench 是一个基于评分标准（rubric）的基准测试，用于评估大语言模型在真实中文法律实务场景中的表现。 该基准模拟执业律师的工作流程，涵盖三个层级的任务：在公共法律咨询中梳理事实、通过结构化法律推理分析案件，以及起草专业法律文书。 -每个样本均附有法律专家标注的评分标准，评分由 LLM 裁判模型依据该评分标准进行，而非与单一参考答案比对。 +每个样本均附有法律专家标注的评分标准，评分由 LLM 评判模型依据该标准进行，而非与单一参考答案比对。 ## 任务描述 -- **任务类型**：开放式中文法律生成任务，使用专家制定的评分标准进行评分 +- **任务类型**：基于专家评分标准的开放式中文法律生成任务 - **输入**：客户陈述，或案件描述加法律问题 - **输出**：问题清单、结构化案件分析，或完整的法律文书 -- **领域**：中国法律实务（个人事务、婚姻家庭、公司治理、知识产权、刑事与民事诉讼、跨境事务、劳动、环境安全等） +- **领域**：中文法律实务（个人事务、婚姻家庭、公司治理、知识产权、刑事与民事诉讼、跨境事务、劳动法、环境安全等） ## 核心特性 - 共 280 个样本，划分为四个子集，对应 PLawBench 的四项任务： - - `case_analysis`（250 个）：案件分析，从四个维度评分——结论、案件事实、推理过程、引用法条。回答必须遵循【结论】/【案件事实】/【推理过程】/【法条依据】的结构。 + - `case_analysis`（250 个）：案件分析，从四个维度评分——结论、案件事实、推理过程、引用法条。回答必须严格遵循【结论】/【案件事实】/【推理过程】/【法条依据】的结构。 - `legal_consultation`（18 个）：模型扮演律师，需提出 10–25 个可验证的后续问题，以揭示客户遗漏或歪曲的事实。 - `plaintiff_statement`（6 个）：根据客户陈述起草起诉状。 - `defendant_statement`（6 个）：根据客户陈述及对方起诉状起草答辩状。 - 客户陈述故意设计得模糊、情绪化或具有误导性，因此模型必须识别陷阱，而非简单复述客户主张。 -- 任务提示词和裁判提示词直接沿用官方发布版本，`case_analysis` 的评分标准保留了各维度的分值分配。 +- 任务提示词和评判提示词均直接采用官方发布版本，`case_analysis` 子集的评分标准保留了各维度的原始分值分配。 ## 评估说明 -- 需要 LLM 裁判模型：运行时需设置 `judge_strategy='llm'`（或 `'auto'`，该选项会为此基准自动启用裁判模型），并提供 `judge_model_args`。不支持 `judge_strategy='rule'`。 -- 指标为 `[0, 1]` 区间内的得分比例。每个子集均报告 `acc`；`case_analysis` 额外报告 `conclusion_acc`、`fact_acc`、`reasoning_acc` 和 `law_acc`。这些指标与官方排行榜列一一对应：`legal_consultation` 对应 Task1，`case_analysis` 对应 Task2-Avg 及其四个维度，两个文书起草子集分别对应 Task3-Plaintiff 和 Task3-Defendant。 -- 应比较各子集得分，而非 `OVERALL` 行。`OVERALL` 是按样本数量计算的平均值，因此被 `case_analysis` 主导（250/280 个样本）。论文中的 `Overall` 列是三项任务得分的等权重平均值，与官方表格更吻合（在官方排名的 24 个模型上拟合，平均绝对误差为 0.72，而样本加权平均的误差为 2.87）。 -- 评分标准的总分来自数据集本身，而非裁判模型输出，且实际得分会被限制在 `[0, max_points]` 范围内，因此即使裁判模型错误报告分母，也不会扭曲最终得分。 -- 相较于官方脚本，本实现修复了 `case_analysis` 的裁判输出模板。官方脚本输出格式错误的 JSON，并将结论部分固定为零分；此处每个部分均按其评分标准分配进行评分。 -- 当裁判响应无法解析时，会重试最多 `judge_retries` 次；若仍失败，则该样本得分为 0，并在评审元数据中标记 `judge_failed`。 -- 案件分析的裁判会返回详细的逐项分解。建议在 `judge_model_args.generation_config` 中为裁判模型设置较大的 `max_tokens`（例如 8192）。 +- **必须使用 LLM 评判模型**：需设置 `judge.strategy='llm'`（或 `'auto'`，该选项会为此基准自动启用评判模型），并提供 `judge.models`。不支持 `judge.strategy='rule'`。 +- **指标为 [0, 1] 区间内的得分比例**。所有子集均报告 `acc`；`case_analysis` 额外报告 `conclusion_acc`、`fact_acc`、`reasoning_acc` 和 `law_acc`。这些指标与官方排行榜列一一对应：`legal_consultation` 对应 Task1，`case_analysis` 对应 Task2-Avg 及其四个维度，两个文书起草子集分别对应 Task3-Plaintiff 和 Task3-Defendant。 +- **应比较各子集得分，而非 `OVERALL` 行**。`OVERALL` 是按样本数量计算的平均值，因此被 `case_analysis` 主导（250/280）。论文中的 `Overall` 列是三项任务得分的等权重平均值，与官方公布表格更吻合（在官方排名的 24 个模型上拟合，平均绝对误差为 0.72，而样本加权平均的误差为 2.87）。 +- **评分标准的总分来自数据集本身，而非评判模型输出**，且实际得分会被限制在 `[0, max_points]` 范围内，因此即使评判模型错误报告分母，也不会扭曲最终分数。 +- `case_analysis` 的评判输出模板已修复。官方脚本存在格式错误的 JSON，并将结论部分固定为零分；本实现中每个部分均按其评分标准独立打分。 +- 评判模型的重试策略通过其 `generation_config` 配置。若回复仍不符合输出格式要求，则视为无效并排除，而非静默记为零分。 +- 案件分析评判会返回详细的逐项分解结果。建议为评判模型设置较大的 `max_tokens`（例如 8192），配置于 `judge.models[].generation_config`。 - 文书起草子集要求生成 2,500–3,000 字符的法律文书，因此被评估模型也需要较大的 `generation_config.max_tokens`。若生成内容被截断，将被视为不完整文书，得分接近零，从而因非法律能力原因拉低 Task3 得分。 资源：[GitHub](https://github.com/skylenage/PLawbench) | @@ -60,9 +60,9 @@ PLawBench 是一个基于评分标准（rubric）的基准测试，用于评估 | 提示词长度（平均） | 2669.88 字符 | | 提示词长度（最小/最大） | 1267 / 5890 字符 | -**各子集统计：** +**各子集统计信息：** -| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 | +| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 | |--------|---------|-------------|------------|------------| | `case_analysis` | 250 | 2720.18 | 2137 | 4873 | | `legal_consultation` | 18 | 1851.94 | 1512 | 2611 | @@ -97,7 +97,7 @@ PLawBench 是一个基于评分标准（rubric）的基准测试，用于评估 } ``` -*注：部分内容因显示需要已被截断。* +*注：部分内容因展示需要已被截断。* ## 提示模板 @@ -106,12 +106,6 @@ PLawBench 是一个基于评分标准（rubric）的基准测试，用于评估 {question} ``` -## 额外参数 - -| 参数 | 类型 | 默认值 | 描述 | -|-----------|------|---------|-------------| -| `judge_retries` | `int` | `3` | 每次评分标准裁判请求的最大重试次数，超过后该样本得分为 0。 | - ## 使用方法 ### 使用 CLI @@ -138,8 +132,7 @@ task_cfg = TaskConfig( datasets=['plawbench'], dataset_args={ 'plawbench': { - # subset_list: ['case_analysis', 'legal_consultation', 'plaintiff_statement'] # 可选，用于评估指定子集 - # extra_params: {} # 使用默认额外参数 + # subset_list: ['case_analysis', 'legal_consultation', 'plaintiff_statement'] # 可选，用于评估特定子集 } }, limit=10, # 正式评估时请删除此行 diff --git a/docs/zh/benchmarks/researchrubrics.md b/docs/zh/benchmarks/researchrubrics.md index a23936520..1cbb37763 100644 --- a/docs/zh/benchmarks/researchrubrics.md +++ b/docs/zh/benchmarks/researchrubrics.md @@ -3,39 +3,45 @@ ## 概述 -ResearchRubrics 用于评估深度研究（Deep Research）智能体在真实、开放式研究任务上的表现。每个任务包含一个用户提示，以及由专家编写的细粒度评分标准（rubrics），涵盖显式与隐式需求、信息综合、参考文献、沟通质量以及指令遵循等方面。 +ResearchRubrics 用于评估深度研究（Deep Research）智能体在真实、开放式研究任务上的表现。每个任务包含一个用户提示，以及由专家编写的细粒度评分标准（rubrics），涵盖显性和隐性需求、信息综合、参考文献、沟通质量以及指令遵循等方面。 ## 任务描述 - **任务类型**：多轮研究智能体 / 长篇报告生成 - **输入**：一个开放式研究提示 -- **输出**：通过迭代使用工具后生成的 Markdown 格式研究报告 +- **输出**：通过迭代使用工具生成的 Markdown 研究报告 - **数据集**：101 个任务，共包含 2,593 条带权重的评分标准 - **评估指标**：二元评分标准符合度得分（Binary rubric compliance score） +## 核心特性 + +- 包含 101 个开放式深度研究任务，配以 2,593 条由专家编写并赋予权重的评分标准。 +- 评分标准覆盖显性与隐性要求、信息综合、参考文献、沟通质量及指令遵循等多个维度，每条标准独立评分。 +- 负权重标准用于捕捉不良行为，当此类行为出现时会从总分中扣除相应分数。 +- 当报告长度超过配置的评判器上下文阈值时，将采用官方的分块-证据-综合（chunk-evidence-synthesis）流程进行评分。 + ## 智能体环境 -- 默认使用 EvalScope 内置的智能体环境，无需提供 ``agent_config``。智能体可通过 ``bash`` 访问网络、收集信息并生成最终报告。 +- 默认使用 EvalScope 内置的智能体环境，无需显式提供 ``agent_config``。智能体可通过 ``bash`` 访问网络、收集信息并生成最终报告。 - 默认环境使用主机网络和临时工作目录，但不提供完整的文件系统隔离。请勿在共享或敏感机器上运行不可信模型。 -- 默认策略为 ``function_calling``，最多执行 50 步。可通过 ``NativeAgentConfig`` 覆盖策略或步数限制；也可选择 ``react`` 策略。两种策略均需模型原生支持函数调用。 +- 默认策略为 ``function_calling``，最多执行 50 步。可通过 ``NativeAgentConfig`` 覆盖策略或步数限制；也可选用 ``react`` 策略。两种策略均需模型原生支持函数调用。 - 可通过 ``NativeAgentConfig`` 添加专用搜索或网页抓取工具，或使用 ``ExternalAgentConfig`` 在其他智能体框架中运行任务。 -- 当达到步数限制时，模型会被要求基于已收集的信息生成最终报告，以便进行人工评审和评分。 +- 若达到步数上限，模型将被要求基于已收集的信息生成最终报告，以便后续评审和打分。 ## 评估说明 -- ResearchRubrics 要求显式配置 ``judge_model_args``，且 ``judge_strategy`` 必须为 ``'auto'`` 或 ``'llm'``。论文推荐使用 Gemini 2.5 Pro 作为评判模型，但未硬编码任何特定提供商或模型。 -- 每条评分标准独立判分为“满足”（1）或“不满足”（0），与公开的二元评分器一致。论文中使用的三元评分无法直接比较。 -- 负权重标准在出现不良行为时会从分子中扣除相应分数，最终得分不会被截断。 -- 当报告长度超过配置的评判模型上下文阈值时，将采用官方的分块-证据-综合（chunk-evidence-synthesis）方法进行评估。 -- 完整运行需执行 2,593 次评分标准评估，成本较高。涉及近期事件的任务对评估时的日期和可用网络资源较为敏感。 +- ResearchRubrics 要求配置 ``judge.models``，且 ``judge.strategy`` 必须为 ``'auto'`` 或 ``'llm'``。论文推荐使用 Gemini 2.5 Pro 作为评判器，但未硬编码任何特定提供商或模型。 +- 每条评分标准独立判为“满足”（1）或“不满足”（0），与公开的二元评分器一致。论文中使用的三元评分不可直接比较。 +- 当出现不良行为时，负权重标准会从分子中扣除对应分数，且最终得分不会被截断（clipped）。 +- 当报告长度超过配置的评判器上下文阈值时，将采用官方的分块-证据-综合方法进行评估。 +- 完整运行需执行 2,593 次评分标准评估，成本较高。此外，涉及当前事件的任务对评估时的日期和可用网络资源较为敏感。 ## 配置 - ``judge_context_limit``: 150,000 个估算 token - ``judge_chunk_size``: 100,000 个估算 token -- ``judge_retries``: 每次评判请求最多重试 3 次 -评判模型必须显式配置。例如： +评判器必须显式配置。例如： ```python from evalscope import TaskConfig, run_task @@ -43,12 +49,14 @@ from evalscope import TaskConfig, run_task run_task(TaskConfig( model='YOUR_AGENT_MODEL', datasets=['researchrubrics'], - judge_strategy='llm', - judge_model_args={ - 'model_id': 'YOUR_JUDGE_MODEL', - 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL', - 'api_key': 'YOUR_JUDGE_API_KEY', - 'generation_config': {'temperature': 0.0}, + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'YOUR_JUDGE_MODEL', + 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL', + 'api_key': 'YOUR_JUDGE_API_KEY', + 'generation_config': {'temperature': 0.0}, + }, }, limit=1, )) @@ -68,7 +76,7 @@ run_task(TaskConfig( | **论文** | [Paper](https://arxiv.org/abs/2511.07685) | | **标签** | `Agent`, `MultiTurn`, `Reasoning`, `Retrieval` | | **指标** | `compliance_score` | -| **默认示例数** | 0-shot | +| **默认示例数量** | 0-shot | | **评估划分** | `train` | @@ -127,7 +135,7 @@ run_task(TaskConfig( } ``` -*注：部分内容因显示需要已被截断。* +*注：部分内容因展示需要已被截断。* ## 提示模板 @@ -140,9 +148,8 @@ run_task(TaskConfig( | 参数 | 类型 | 默认值 | 描述 | |-----------|------|---------|-------------| -| `judge_context_limit` | `int` | `150000` | 评判前允许的最大估算 token 数，超过此值将启用分块处理。 | -| `judge_chunk_size` | `int` | `100000` | 发送给评判模型的每个文档分块的最大估算 token 数。 | -| `judge_retries` | `int` | `3` | 每次评分标准评判请求及 JSON 解析的最大重试次数。 | +| `judge_context_limit` | `int` | `150000` | 切换至分块评分前的估算 token 上限。 | +| `judge_chunk_size` | `int` | `100000` | 发送给评判器的每个文档分块的最大估算 token 数。 | ## 使用方法 diff --git a/docs/zh/benchmarks/slake.md b/docs/zh/benchmarks/slake.md index 9e30a901b..0b891e132 100644 --- a/docs/zh/benchmarks/slake.md +++ b/docs/zh/benchmarks/slake.md @@ -3,33 +3,33 @@ ## 概述 -SLAKE 是一个双语（英语 / 中文）的放射学视觉问答基准数据集，由医生基于 CT、MRI 和 X 光图像构建而成。问题既涵盖影像本身的纯视觉属性，也包含需结合医学知识才能回答的内容。 +SLAKE 是一个双语（英语 / 中文）放射学视觉问答基准数据集，由医生基于 CT、MRI 和 X 光图像构建而成。问题既涵盖影像的纯视觉属性，也包含需要结合医学知识才能回答的内容。 ## 任务描述 -- **任务类型**：医学视觉问答（自由形式的简短回答） -- **输入**：一张放射学图像以及一道英语或中文问题 -- **输出**：一个单词或短语，语言与问题一致 -- **领域**：放射学（胸部、腹部、脑部、骨盆、颈部） +- **任务类型**：医学视觉问答（自由形式的简短答案） +- **输入**：一张放射学图像以及一条英文或中文问题 +- **输出**：单个单词或短语，语言需与问题一致 +- **领域**：放射学（胸部、腹部、脑部、盆腔、颈部） ## 主要特点 -- 包含 180 张图像上的 2,094 道测试问题，英语（1,061）和中文（1,033）大致均衡 -- 每个问题均标注为 `OPEN`（自由回答）或 `CLOSED`（答案来自一个小的封闭集合，多为是/否类问题），这是原始论文所采用的划分方式 -- 问题涵盖十种语义类型：器官、位置、异常、知识图谱、模态、大小、平面、数量、颜色和形状 +- 包含 180 张图像上的 2,094 个测试问题，英语（1,061）和中文（1,033）大致均衡 +- 每个问题均标注为 `OPEN`（自由作答）或 `CLOSED`（答案来自小型封闭集合，多为是/否类问题），此分类方式与原始论文一致 +- 问题涵盖十种语义类型：器官、位置、异常、知识图谱、模态、大小、层面、数量、颜色和形状 - 知识图谱类问题（`base_type=kvqa`）涉及病因、症状、治疗和功能等无法直接从图像中读取的信息 ## 评估说明 -- **主要指标**：通过归一化后的精确匹配（exact match）计算**准确率**，与单一参考答案对比 +- **主要指标**：通过归一化精确匹配（normalized exact match）计算**准确率**，并与唯一参考答案对比 - 结果按四个子集报告：`_`，分为英语和中文两类；总体得分为样本加权平均值 -- 归一化遵循官方答案预处理规则（转小写、去除标点和冠词、将文字数字转换为阿拉伯数字、统一 `x ray` 为 `xray`），并额外加入生成式回答所需处理： - - 英语中 yes/no 的同义词统一归为一个标签； - - 中文中表达相同极性的 是的 / 有 / 包含 / 可以 或 不是 / 没有 / 不包含 / 不可以 也统一归为一个标签； - - 中文中的 X光 / X射线 和 两个 / 二 分别映射为英文的 `X-Ray` 和数字形式（中文部分保留数字引用） +- 归一化遵循官方答案预处理规则（转小写、去除标点和冠词、将文字数字转换为阿拉伯数字、统一 `x ray` 为 `xray`），并额外加入生成式问答所需处理： + - 英语中 yes/no 的同义词统一归为一个标签 + - 中文中表达相同极性的 是的 / 有 / 包含 / 可以 或 不是 / 没有 / 不包含 / 不可以 也统一归为一个标签 + - 中文中的 X光 / X射线 和 两个 / 二 统一映射为英文的 `X-Ray` 和对应数字形式 - 答案从提示中要求的 `ANSWER:` 行读取；若模型未输出该行，则对整个回复进行归一化处理，因此仅复述问题的回复得分为 0 -- 精确匹配设计上较为严格，与原始分类式评估一致：例如参考答案为 `Lung, Spinal Cord`、知识图谱中的治疗列表，或 `T2` 被回答为 `T2-weighted`，只有当模型完全复现参考措辞时才计为正确，因此知识图谱类开放问题的准确率预期较低 -- 严格性也会导致仅在措辞或粒度上与参考答案不同的回答被判错（如 `Right` 回答为 `Right Side`，胸腔 回答为 胸部，或多答案参考中只答对其中一项）。设置 `judge_strategy='llm_recall'` 可让大语言模型重新检查仅因规则判断失败的情况；该得分比已发表结果更宽松，不可直接与其比较 +- 精确匹配设计上较为严格，与原始分类式评估一致：例如参考答案为 `Lung, Spinal Cord`、知识图谱中的治疗列表，或 `T2` 被回答为 `T2-weighted` 时，仅当模型完全复现参考措辞才算正确，因此知识图谱类开放问题的准确率预期较低 +- 严格性也会导致仅在措辞或粒度上与参考答案不同的回答被判错（如 `Right` 回答为 `Right Side`，胸腔 回答为 胸部，或多答案参考中只答对其中一项）。设置 `judge.strategy='llm_recall'` 可让大语言模型（LLM）重新审核仅因规则判断失败的情况；该分数比已发表结果更宽松，不可直接比较 - 图像以单个 `imgs.zip` 文件形式提供（约 200 MB），直接从压缩包中读取 - [论文](https://arxiv.org/abs/2102.09542) | [项目主页](https://www.med-vqa.com/slake/) diff --git a/docs/zh/benchmarks/tir_bench.md b/docs/zh/benchmarks/tir_bench.md index b0630ddb5..3055737e6 100644 --- a/docs/zh/benchmarks/tir_bench.md +++ b/docs/zh/benchmarks/tir_bench.md @@ -9,28 +9,28 @@ TIR-Bench（Thinking-with-Images Reasoning Benchmark）是一个全面的多模 - **任务类型**：多任务视觉推理（包括选择题、OCR、单词搜索、找不同、拼图等） - **输入**：一张或两张图像 + 问题（大多数任务采用选择题格式） -- **输出**：答案字母（选择题）或数字/文本形式的答案（取决于任务类型） -- **领域**：instrument（仪器读数）、color（颜色识别）、refcoco（指代表达理解）、rotation_game（旋转游戏）、math（数学题）、word_search（单词搜索）、visual_search（视觉搜索）、ocr（光学字符识别）、symbolic（符号推理）、spot_difference（找不同）、contrast（对比分析）、jigsaw（拼图）、maze（迷宫） +- **输出**：答案字母（选择题）或数字/文本响应（取决于任务类型） +- **领域**：instrument（仪器）、color（颜色）、refcoco、rotation_game（旋转游戏）、math（数学）、word_search（单词搜索）、visual_search（视觉搜索）、ocr、symbolic（符号）、spot_difference（找不同）、contrast（对比）、jigsaw（拼图）、maze（迷宫） ## 核心特性 - 包含13种多样化的视觉推理任务类别，共计1,215个测试样本 - 覆盖单图和双图推理场景 - 答案形式包括字母选项（A-J）、整数、浮点数和文本 -- 采用任务特定的评分机制，并在必要时回退至基于大语言模型（LLM-as-judge）的评分方式，以实现稳健评估 +- 采用任务特定评分机制，并在必要时回退至LLM-as-judge以实现稳健评估 ## 评估说明 -- 默认使用 **test** 分割进行评估（共1,215个样本） +- 默认使用 **test** 划分进行评估（共1,215个样本） - 主要指标：**准确率**（`accuracy`） - 图像数据通过 ModelScope 下载为 `data.zip` 并自动解压 - 基于规则的评分方式： - - OCR：子字符串匹配 - - 拼图（jigsaw）：网格 IoU - - 找不同（spot_difference）：集合 IoU - - 单词搜索（word_search）：数值匹配 - - 其他所有任务：选择题或数值判断 -- **推荐设置**：将 `judge_strategy=JudgeStrategy.LLM_RECALL` 并提供 `judge_model_args`，以启用 LLM-as-judge 作为召回机制——仅当基于规则的评分结果为0时才调用评判模型，从而在避免不必要 API 开销的同时提升评估准确性 + - OCR（子串匹配） + - 拼图（grid IoU） + - 找不同（set IoU） + - 单词搜索（数值匹配） + - 其他所有任务（选择题 / 数值判断） +- **推荐设置**：将 `judge.strategy='llm_recall'` 并提供 `judge.models`，以启用 LLM-as-judge 作为召回机制——仅当基于规则的评分结果为0时才调用判别器，在避免不必要API开销的同时提升评估准确性 - [论文](https://arxiv.org/abs/2511.01833) | [GitHub](https://github.com/agents-x-project/TIR-Bench) diff --git a/docs/zh/benchmarks/wide_search.md b/docs/zh/benchmarks/wide_search.md index f32e4e6a8..650f297a3 100644 --- a/docs/zh/benchmarks/wide_search.md +++ b/docs/zh/benchmarks/wide_search.md @@ -3,26 +3,26 @@ ## 概述 -WideSearch 用于评估搜索智能体在广泛网络信息检索任务上的表现。每个任务要求智能体收集大量原子事实，并返回一个结构化的 Markdown 表格。EvalScope 使用 ModelScope 上的 `bytedance-community/WideSearch` 数据集。 +WideSearch 用于评估搜索智能体在广泛网络信息检索任务上的表现。每个任务要求智能体收集多个原子事实，并返回一个结构化的 Markdown 表格。EvalScope 使用 ModelScope 上的 `bytedance-community/WideSearch` 数据集。 ## 任务描述 - **任务类型**：多轮搜索智能体 -- **输入**：包含明确表格结构的自然语言信息收集请求 +- **输入**：包含显式表格结构的自然语言信息收集请求 - **输出**：完整的 Markdown 表格 -- **数据集**：`full` 划分中共有 200 个任务；其中 100 个为英文，100 个为中文 +- **数据集**：``full`` 划分中共有 200 个任务；其中 100 个为英文，100 个为中文 ## 核心特性 -- 官方单智能体协议：支持语言特定的系统提示词、``function_calling``，以及默认 50 步的最大执行步数。 -- 默认为每个样本提供临时本地目录中的 Bash 环境；Docker 沙箱和 MCP 服务器为可选项。 +- 官方单智能体协议：语言特定的系统提示词、``function_calling`` 机制，以及默认 50 步的最大执行步数。 +- 默认为每个样本提供一个临时本地目录并启用 Bash；Docker 沙箱和 MCP 服务器为可选项。 - 单次完整运行即可生成 ``all``、``en`` 和 ``zh`` 三份报告，无需重复推理。 ## 评估说明 -- 采用官方的 Markdown 表格对齐方式及混合规则/LLM 评分语义。 -- 必须设置 ``judge_strategy='auto'`` 或 ``'llm'`` 并显式指定 ``judge_model_args``；不支持仅使用规则评分。 -- 运行示例及论文风格的重复实验设置，请参阅 [WideSearch 使用指南](https://evalscope.readthedocs.io/zh-cn/latest/third_party/wide_search.html)。 +- 使用官方的 Markdown 表格对齐方式及混合规则/大语言模型（LLM）评分语义。 +- 要求设置 ``judge.strategy='auto'`` 或 ``'llm'``，且至少指定一个 ``judge.models`` 条目；不支持仅使用规则评分。 +- 运行时示例及论文风格的重复实验设置，请参阅 [WideSearch 使用指南](https://evalscope.readthedocs.io/zh-cn/latest/third_party/wide_search.html)。 ## 属性 diff --git a/docs/zh/best_practice/index_collection.md b/docs/zh/best_practice/index_collection.md index db525bc22..1dce23c4b 100644 --- a/docs/zh/best_practice/index_collection.md +++ b/docs/zh/best_practice/index_collection.md @@ -170,11 +170,11 @@ task_cfg = TaskConfig( 'temperature': 0.0 # 评测通常设为 0 以保证结果可复现 }, # 使用强大的模型作为 judge 进行自动评分 (chinese_simpleqa 需要) - judge_model_args={ + judge={'models': { 'model_id': 'qwen3-max', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': os.getenv('DASHSCOPE_API_KEY'), - }, + }}, eval_batch_size=5, # 并发数，根据实际情况调整 ) @@ -359,4 +359,4 @@ run_task(task_cfg=task_cfg) 4. 用 Case 分析定位了模型的弱点 -不要再局限于通用榜单的分数，现在就开始构建属于你自己的 **Index**，选出真正适合你业务的“最佳模型”吧！ \ No newline at end of file +不要再局限于通用榜单的分数，现在就开始构建属于你自己的 **Index**，选出真正适合你业务的“最佳模型”吧！ diff --git a/docs/zh/best_practice/qwen3_coder.md b/docs/zh/best_practice/qwen3_coder.md index ce86d4afc..1666614da 100644 --- a/docs/zh/best_practice/qwen3_coder.md +++ b/docs/zh/best_practice/qwen3_coder.md @@ -122,15 +122,12 @@ task_cfg = TaskConfig( # limit=20, # 限制评测数量，便于快速测试，正式评测时建议去掉此项 ignore_errors=True, # 忽略错误，可能会被模型拒绝的测试用例 stream=True, # 启用流式输出 - judge_model_args={ # 配置Judge模型参数 + judge={'models': { # 配置 Judge 模型参数 'model_id': 'qwen2.5-72b-instruct', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': os.getenv('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096 - } - }, + 'generation_config': {'temperature': 0.0, 'max_tokens': 4096}, + }}, ) run_task(task_cfg=task_cfg) diff --git a/docs/zh/get_started/basic_usage.md b/docs/zh/get_started/basic_usage.md index a7d0e8f25..b7d4da57a 100644 --- a/docs/zh/get_started/basic_usage.md +++ b/docs/zh/get_started/basic_usage.md @@ -216,7 +216,6 @@ evalscope eval \ ```python import os from evalscope import TaskConfig, run_task -from evalscope.constants import JudgeStrategy task_cfg = TaskConfig( model='qwen2.5-7b-instruct', @@ -225,11 +224,13 @@ task_cfg = TaskConfig( eval_type='openai_api', datasets=['chinese_simpleqa'], limit=5, - judge_strategy=JudgeStrategy.AUTO, - judge_model_args={ - 'model_id': 'qwen2.5-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': os.getenv('DASHSCOPE_API_KEY'), + judge={ + 'strategy': 'auto', + 'models': { + 'model_id': 'qwen2.5-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': os.getenv('DASHSCOPE_API_KEY'), + }, } ) @@ -322,7 +323,7 @@ evalscope eval \ ### 从已有/中断的评测结果继续评测 -如果之前的评测任务中断，或者您想基于已有的评测结果继续进行评测，可以使用 `--use-cache` 参数指定之前的输出目录。这将跳过已完成的样本，只评测剩余样本。此外，还可以通过 `--rerun-review` 参数强制对所有样本重新执行评分步骤。 +如果之前的评测任务中断，或者您想基于已有的评测结果继续进行评测，可以使用 `--use-cache` 参数指定之前的输出目录。这将跳过已完成的样本，只评测剩余样本。此外，`--rerun-review` 会基于缓存的 prediction 重新评分，并只在成功后原子替换 review 缓存。 ```shell evalscope eval \ diff --git a/docs/zh/get_started/faq.md b/docs/zh/get_started/faq.md index 9a59dd741..71ee74b46 100644 --- a/docs/zh/get_started/faq.md +++ b/docs/zh/get_started/faq.md @@ -70,15 +70,15 @@ --dataset-args '{"ifeval": {"filters": {"remove_until": ""}}}' ``` 如果您的模型使用不同的思考标签，如 `<|end_of_thinking|>`，只需替换即可。 -3. **用裁判模型兜底**：设置 `judge_strategy=JudgeStrategy.LLM_RECALL`，规则提取失败的样本会交给裁判模型重新判定，详见下文“结果异常与问题排查”。 +3. **用裁判模型兜底**：设置 `judge={'strategy': 'llm_recall', 'models': {...}}`。规则已满分的样本不调用 Judge；其余样本会用 Judge 复核，详见下文“结果异常与问题排查”。 **Q: 如何使用本地模型作为裁判模型（Judge Model）？** -**A:** 您可以使用 vLLM 等框架将本地模型部署为一个 API 服务，然后在 `--judge-model-args` 中指定其服务地址。 +**A:** 您可以使用 vLLM 等框架将本地模型部署为一个 API 服务，然后在 `--judge` 的 `models` 中指定其服务地址。 **Q: 如何为裁判模型设置超时时间？** -**A:** 在 `--judge-model-args` 的 `generation_config` 中设置 `timeout` 参数。 +**A:** 在 `--judge` 的 `models.generation_config` 中设置 `timeout` 参数。 **Q: 如何在评测 API 服务时添加自定义请求头（Header）？** @@ -128,12 +128,9 @@ task_config = TaskConfig( **A:** 数学问题的答案格式复杂，规则解析难以覆盖所有情况。建议使用 LLM 作为辅助裁判来提升准确率： ```python # 在 TaskConfig 中设置 -judge_strategy=JudgeStrategy.LLM_RECALL, -judge_model_args={ - 'model_id': 'qwen2.5-72b-instruct', - 'api_url': '...', - 'api_key': '...' -} +judge={'strategy': 'llm_recall', 'models': { + 'model_id': 'qwen2.5-72b-instruct', 'api_url': '...', 'api_key': '...' +}} ``` 参考文档：[裁判模型参数](https://evalscope.readthedocs.io/zh-cn/latest/get_started/parameters.html#judge)。 @@ -144,14 +141,13 @@ judge_model_args={ 2. 再配合裁判模型兜底，规则提取失败时自动召回： ```python # 在 TaskConfig 中设置 - judge_strategy=JudgeStrategy.LLM_RECALL, - judge_model_args={'model_id': '...', 'api_url': '...', 'api_key': '...'} + judge={'strategy': 'llm_recall', 'models': {'model_id': '...', 'api_url': '...', 'api_key': '...'}} ``` `llm_recall` 仅在规则得分不满分时才调用裁判模型，不会带来额外的全量开销。 **Q: 评测 `alpaca_eval` 时报错 `Connection error`？** -**A:** `alpaca_eval` 需要指定一个裁判模型（Judge Model）进行打分。默认使用 OpenAI API，如果未配置相关 Key 会导致连接失败。请通过 `--judge-model-args` 指定一个可用的裁判模型。 +**A:** `alpaca_eval` 需要指定一个裁判模型（Judge Model）进行打分。默认使用 OpenAI API，如果未配置相关 Key 会导致连接失败。请通过 `--judge` 指定一个可用的裁判模型。 **Q: 评测中断后，如何从断点处继续？** @@ -287,4 +283,4 @@ judge_model_args={ year={2024}, url={https://github.com/modelscope/evalscope} } -``` \ No newline at end of file +``` diff --git a/docs/zh/get_started/parameters.md b/docs/zh/get_started/parameters.md index 5af174ac1..a1af478c2 100644 --- a/docs/zh/get_started/parameters.md +++ b/docs/zh/get_started/parameters.md @@ -157,73 +157,57 @@ ## Judge参数 -LLM-as-a-Judge评测参数，使用裁判模型判断正误： - -| 参数 | 类型 | 说明 | 默认值 | -|------|------|------|--------| -| `--judge-strategy` | `str` | 裁判模型策略
• `auto`: 根据数据集自动决定
• `llm`: 总是使用裁判模型
• `rule`: 只使用规则判断
• `llm_recall`: 规则失败后使用裁判模型 | `auto` | -| `--judge-worker-num` | `int` | **[已废弃]** 请使用 `--eval-batch-size` 代替，将在 v2.0.0 中移除 | `1` | -| `--judge-model-args` | `dict` | 裁判模型配置（JSON字符串），详见下表 | - | -| `--analysis-report` | `bool` | 是否生成分析报告（自动判断语言） | `false` | - -### judge-model-args 配置项 - -| 参数 | 类型 | 说明 | 默认值 | -|------|------|------|--------| -| `api_key` | `str` | API密钥 | 从`MODELSCOPE_SDK_TOKEN`读取，默认`EMPTY` | -| `api_url` | `str` | API端点 | 从`MODELSCOPE_API_BASE`读取，
默认`https://api-inference.modelscope.cn/v1/` | -| `model_id` | `str` | 模型ID | 从`MODELSCOPE_JUDGE_LLM`读取，
默认`Qwen/Qwen3-235B-A22B` | -| `system_prompt` | `str` | 系统prompt | - | -| `prompt_template` | `str` | Prompt模板 | 根据`score_type`自动选择 | -| `generation_config` | `dict` | 生成参数（同`--generation-config`） | - | -| `model_args` | `dict` | 裁判模型加载参数（同`--model-args`），例如`{"default_headers": {"X-API-KEY": "your-api-key"}}` | `{}` | -| `score_type` | `str` | 打分方式
• `pattern`: 判断与参考答案是否相同
• `numeric`: 无参考答案打分（0-1） | `pattern` | -| `score_pattern` | `str` | 解析输出的正则表达式 | `pattern`模式：`(A\|B)`
`numeric`模式：`\[\[(\d+(?:\.\d+)?)\]\]` | -| `score_mapping` | `dict` | `pattern`模式的分数映射 | `{'A': 1.0, 'B': 0.0}` | - -```{seealso} -关于ModelScope模型推理服务，请参考[ModelScope API推理服务](https://modelscope.cn/docs/model-service/API-Inference/intro) -``` - -
pattern 模式默认prompt模板 - -```text -Your job is to look at a question, a gold target, and a predicted answer, and return a letter "A" or "B" to indicate whether the predicted answer is correct or incorrect. - -[Question] -{question} - -[Reference Answer] -{gold} - -[Predicted Answer] -{pred} - -Evaluate the model's answer based on correctness compared to the reference answer. -Grade the predicted answer of this new question as one of: -A: CORRECT -B: INCORRECT - -Just return the letters "A" or "B", with no text around it. +Native LLM Judge 通过一个 typed `judge` 对象配置：Python/YAML 使用 `judge={...}`，CLI 使用 +`--judge ''`。 + +```python +TaskConfig( + model='MODEL', + datasets=['simple_qa'], + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'JUDGE_MODEL', + 'api_url': 'OPENAI_COMPATIBLE_URL', + 'api_key': 'JUDGE_API_KEY', + 'generation_config': {'temperature': 0.0, 'retries': 3}, + }, + 'repeats': 1, + 'position_swap': 'auto', + 'aggregation': 'mean', + 'min_valid_judges': 1, + }, +) ``` -
- -
numeric 模式默认prompt模板 -```text -Please act as an impartial judge and evaluate the quality of the response provided by an AI assistant to the user question displayed below. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of the response. +`models` 可传单个对象或对象列表。列表表示独立 Judge；重复的 `model_id` 必须显式指定不同的 +`judge_id`，唯一 `model_id` 默认同时作为 `judge_id`。 -Begin your evaluation by providing a short explanation. Be as objective as possible. - -After providing your explanation, you must rate the response on a scale of 0 (worst) to 1 (best) by strictly following this format: "[[rating]]", for example: "Rating: [[0.5]]" - -[Question] -{question} - -[Response] -{pred} -``` -
+| 字段 | 类型 | 说明 | 默认值 | +|------|------|------|--------| +| `strategy` | `auto\|rule\|llm\|llm_recall` | `auto` 遵循 benchmark 策略；`llm_recall` 仅复核规则漏判，并取 `max(rule, judge)`。 | `auto` | +| `models` | `object\|list[object]` | 一个或多个 Judge 模型配置；为保证 review cache 可复现，必须给出 `model_id`。 | `[]` | +| `repeats` | `int >= 1` | 每个 Judge 的独立判分观测次数，不等同于 transport retry。 | `1` | +| `position_swap` | `auto\|on\|off` | `auto` 保持 benchmark 官方的位置交换策略。 | `auto` | +| `aggregation` | `mean\|median\|majority_vote` | 普通指标的跨观测聚合方式。 | `mean` | +| `min_valid_judges` | `int >= 1` | 一个指标所需的最少有效 Judge verdict 数。 | `1` | + +`models` 的每项支持 `judge_id`、`model_id`、`api_key`、`api_url`、`eval_type`、`model_args` 与 +`generation_config`。provider 私有的模型初始化参数放入 `model_args`；transport 重试放入 +`generation_config.retries`。 + +`judge.contract` 仅配置通用单 verdict Judge：`system_prompt`、`prompt_template`、`score_mapping` 和 +`score_type`。`pattern` 要求 Judge 在 JSON 中返回 `score_mapping` 之一；`numeric` 要求 JSON 分数位于 +`[0, 1]`。框架会在 prompt 中追加 JSON 格式要求，只解析一次普通模型回复；不使用 constrained decoding、 +正则提分或纠正性追问。无效回复显示为 unavailable，并从指标中排除，而非记为 0。 + +对于经过 LLM 判定的样本，报告包含 `JudgeSummary`：覆盖率、失败计数与分歧。当 adapter 通过确定性的 +Judge 短路直接判定样本时，得分 metadata 会记录 `judge_skipped=true` 和 `judge_skip_reason`；Web review +面板会将其标为规则直接判分，而非 LLM verdict。设置 `rerun_review=True` 可复用 prediction 并重算 review， +新的 review 文件只有成功后才原子替换旧文件。 + +旧 `judge_strategy` 和单个 mapping `judge_model_args` 仅保留一轮输入迁移并会告警。已删除的 +`judge_worker_num` 和 `score_pattern` 会明确报错。 ## Sandbox参数 @@ -271,7 +255,7 @@ EvalScope 使用嵌套的 `--sandbox` 配置（对应 `SandboxTaskConfig`）统 | `--work-dir` | `str` | 评测输出路径（详见下方目录结构） | `./outputs` | | `--no-timestamp` | `bool` | 是否不在工作目录中添加时间戳 | `false` | | `--use-cache` | `str` | 复用本地缓存路径（如`outputs/20241210_194434`）
重用推理结果和评测结果 | `None` | -| `--rerun-review` | `bool` | 配合 `--use-cache` 使用：删除已有 reviews 缓存并重跑评测/打分阶段，但仍复用 predictions 缓存 | `false` | +| `--rerun-review` | `bool` | 配合 `--use-cache` 使用：基于 predictions 缓存重跑评测/打分，并仅在成功后原子替换 reviews 缓存 | `false` | | `--enable-progress-tracker` | `bool` | 是否开启进度追踪，将层级评测进度实时写入`progress.json`，可通过服务接口查询 | `false` | | `--collect-perf` | `bool` | 采集每次推理请求的性能指标（延迟、TTFT、Token 用量），汇总后写入评测报告。采集 TTFT 需开启 `--generation-config stream=true`；使用 `--no-collect-perf` 可禁用 | `true` | | `--seed` | `int` | 随机种子 | `42` | diff --git a/docs/zh/third_party/deepsearchqa.md b/docs/zh/third_party/deepsearchqa.md index 336094039..5230993b3 100644 --- a/docs/zh/third_party/deepsearchqa.md +++ b/docs/zh/third_party/deepsearchqa.md @@ -52,12 +52,14 @@ task_config = TaskConfig( datasets=['deepsearchqa'], eval_batch_size=1, limit=10, - judge_strategy='llm', - judge_model_args={ - 'model_id': 'YOUR_JUDGE_MODEL', - 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL', - 'api_key': os.getenv('JUDGE_API_KEY'), - 'generation_config': {'temperature': 0.0}, + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'YOUR_JUDGE_MODEL', + 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL', + 'api_key': os.getenv('JUDGE_API_KEY'), + 'generation_config': {'temperature': 0.0}, + }, }, agent_config=NativeAgentConfig( strategy='function_calling', @@ -99,8 +101,7 @@ evalscope eval \ --datasets deepsearchqa \ --limit 10 \ --eval-batch-size 1 \ - --judge-strategy llm \ - --judge-model-args '{"model_id":"YOUR_JUDGE_MODEL","api_url":"OPENAI_COMPATIBLE_JUDGE_URL","api_key":"'"$JUDGE_API_KEY"'","generation_config":{"temperature":0.0}}' \ + --judge '{"strategy":"llm","models":{"model_id":"YOUR_JUDGE_MODEL","api_url":"OPENAI_COMPATIBLE_JUDGE_URL","api_key":"'"$JUDGE_API_KEY"'","generation_config":{"temperature":0.0}}}' \ --agent-config '{"mode":"native","strategy":"function_calling","max_steps":30,"mcp_servers":[{"type":"stdio","name":"fetch","command":"python","args":["-m","mcp_server_fetch","--ignore-robots-txt"]},{"type":"http","name":"search","url":"https://mcp.api-inference.modelscope.net//mcp","headers":{"Authorization":"Bearer "}}],"kwargs":{"system_prompt":"Use search and fetch tools to research the question. Cross-check important facts before answering. When finished, call submit(answer=...) with only the final answer."}}' ``` diff --git a/docs/zh/third_party/needle_haystack.md b/docs/zh/third_party/needle_haystack.md index 733df3366..30b20b6bb 100644 --- a/docs/zh/third_party/needle_haystack.md +++ b/docs/zh/third_party/needle_haystack.md @@ -61,11 +61,11 @@ task_cfg = TaskConfig( 'max_tokens': 512, # 最大生成token数 }, eval_batch_size=5, - judge_model_args={ + judge={'models': { 'model_id': 'qwen2.5-72b-instruct', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': os.getenv('DASHSCOPE_API_KEY'), - } + }}, ) run_task(task_cfg=task_cfg) ``` @@ -160,13 +160,13 @@ task_cfg = TaskConfig( 'max_tokens': 512, # 最大生成token数 }, eval_batch_size=5, - judge_model_args={ + judge={'models': { 'model_id': 'qwen2.5-72b-instruct', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': os.getenv('DASHSCOPE_API_KEY'), - } + }}, ) run_task(task_cfg=task_cfg) ``` -输出示例同上。 \ No newline at end of file +输出示例同上。 diff --git a/docs/zh/third_party/wide_search.md b/docs/zh/third_party/wide_search.md index 8e391a53a..860fab396 100644 --- a/docs/zh/third_party/wide_search.md +++ b/docs/zh/third_party/wide_search.md @@ -39,12 +39,14 @@ run_task(TaskConfig( api_key=os.getenv('MODEL_API_KEY'), eval_type='openai_api', datasets=['wide_search'], - judge_strategy='llm', - judge_model_args={ - 'model_id': 'YOUR_JUDGE_MODEL', - 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL', - 'api_key': os.getenv('JUDGE_API_KEY'), - 'generation_config': {'temperature': 0.0}, + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'YOUR_JUDGE_MODEL', + 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL', + 'api_key': os.getenv('JUDGE_API_KEY'), + 'generation_config': {'temperature': 0.0}, + }, }, eval_batch_size=1, limit=1, # 正式评测 200 题时删除 @@ -74,8 +76,7 @@ run_task(TaskConfig( 'network_enabled': True, }, ), - judge_strategy='llm', - judge_model_args={'model_id': 'YOUR_JUDGE_MODEL'}, + judge={'strategy': 'llm', 'models': {'model_id': 'YOUR_JUDGE_MODEL'}}, limit=1, )) ``` @@ -109,8 +110,7 @@ run_task(TaskConfig( ) ], ), - judge_strategy='llm', - judge_model_args={'model_id': 'YOUR_JUDGE_MODEL'}, + judge={'strategy': 'llm', 'models': {'model_id': 'YOUR_JUDGE_MODEL'}}, )) ``` diff --git a/docs/zh/user_guides/aigc/image_edit.md b/docs/zh/user_guides/aigc/image_edit.md index 5ad1b3643..3e3b67978 100644 --- a/docs/zh/user_guides/aigc/image_edit.md +++ b/docs/zh/user_guides/aigc/image_edit.md @@ -26,7 +26,7 @@ from dotenv import dotenv_values env = dotenv_values('.env') from evalscope import TaskConfig, run_task -from evalscope.constants import EvalType, JudgeStrategy, ModelTask +from evalscope.constants import EvalType, ModelTask task_config = TaskConfig( model='Qwen/Qwen-Image-Edit', # 模型ID 或 本地路径 @@ -52,15 +52,14 @@ task_config = TaskConfig( }, eval_batch_size=5, limit=5, - judge_strategy=JudgeStrategy.AUTO, - judge_model_args={ # 需要配置一个VLM模型用于自动打分 - 'model_id': 'qwen2.5-vl-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': env.get('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096, - } + judge={ + 'strategy': 'auto', + 'models': { # 需要配置一个VLM模型用于自动打分 + 'model_id': 'qwen2.5-vl-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': env.get('DASHSCOPE_API_KEY'), + 'generation_config': {'temperature': 0.0, 'max_tokens': 4096}, + }, }, ) @@ -175,7 +174,7 @@ from dotenv import dotenv_values env = dotenv_values('.env') from evalscope import TaskConfig, run_task -from evalscope.constants import EvalType, JudgeStrategy, ModelTask +from evalscope.constants import EvalType, ModelTask task_config = TaskConfig( model_id='offline-model', # 无需配置model，只需提供自定义的ID @@ -193,15 +192,14 @@ task_config = TaskConfig( }, eval_batch_size=5, limit=5, - judge_strategy=JudgeStrategy.AUTO, - judge_model_args={ # 需要配置一个VLM模型用于自动打分 - 'model_id': 'qwen2.5-vl-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': env.get('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096, - } + judge={ + 'strategy': 'auto', + 'models': { # 需要配置一个VLM模型用于自动打分 + 'model_id': 'qwen2.5-vl-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': env.get('DASHSCOPE_API_KEY'), + 'generation_config': {'temperature': 0.0, 'max_tokens': 4096}, + }, }, ) @@ -249,4 +247,4 @@ evalscope service *打分分布* ![image](https://sail-moe.oss-cn-hangzhou.aliyuncs.com/yunlin/images/evalscope/image_edit/compare.png) -*效果对比* \ No newline at end of file +*效果对比* diff --git a/docs/zh/user_guides/arena.md b/docs/zh/user_guides/arena.md index 0e1d3850b..74b592fb8 100644 --- a/docs/zh/user_guides/arena.md +++ b/docs/zh/user_guides/arena.md @@ -196,8 +196,8 @@ task_cfg = TaskConfig( } } }, - # 配置judge模型参数 - judge_model_args={ + # 配置 Judge 模型参数 + judge={'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': os.getenv('DASHSCOPE_API_KEY'), @@ -205,7 +205,7 @@ task_cfg = TaskConfig( 'temperature': 0.0, 'max_tokens': 8000 }, - }, + }}, eval_batch_size=5, # use_cache='outputs/xxx' # 可选，若想在已有评测结果上添加新的候选模型，可以指定已有评测结果路径 ) diff --git a/evalscope/api/benchmark/adapters/default_data_adapter.py b/evalscope/api/benchmark/adapters/default_data_adapter.py index b62fb1ea6..aaf5afe3f 100644 --- a/evalscope/api/benchmark/adapters/default_data_adapter.py +++ b/evalscope/api/benchmark/adapters/default_data_adapter.py @@ -704,23 +704,38 @@ def calculate_metrics(self, task_state: TaskState) -> SampleScore: task_state=task_state ) - # Step 2: Apply LLM judge if enabled and get final score - final_score = self.maybe_llm_match_score( - original_prediction=prediction, - filtered_prediction=filtered_prediction, - reference=task_state.target, - task_state=task_state, - rule_based_score=rule_based_score - ) + if float(rule_based_score.main_value or 0.0) > 0.99: + final_score = rule_based_score + else: + # A valid judge may raise the rule score; an unavailable judge preserves it. + judge_score = self.score_with_judge_contracts( + original_prediction=prediction, + filtered_prediction=filtered_prediction, + reference=task_state.target, + task_state=task_state, + ) + final_score = self._merge_scores(rule_based_score, judge_score) else: if self.use_llm_judge: - # Use LLM judge to compute the match score directly - final_score = self.llm_match_score( + # Judge-default benchmarks retain their usable rule score when the judge fails. + judge_score = self.score_with_judge_contracts( original_prediction=prediction, filtered_prediction=filtered_prediction, reference=task_state.target, task_state=task_state ) + if not judge_score.status.is_usable and self.scoring_policy.rule_supported: + final_score = self.fallback_to_rule_score( + self.match_score( + original_prediction=prediction, + filtered_prediction=filtered_prediction, + reference=task_state.target, + task_state=task_state, + ), + judge_score, + ) + else: + final_score = judge_score else: # Use standard match score calculation without LLM judge final_score = self.match_score( @@ -735,6 +750,7 @@ def calculate_metrics(self, task_state: TaskState) -> SampleScore: score=final_score, sample_id=task_state.sample_id, group_id=task_state.group_id, + generation_index=(task_state.sample_id % self.repeats) if self.repeats > 1 else 0, sample_metadata=task_state.metadata, ) diff --git a/evalscope/api/evaluator/cache.py b/evalscope/api/evaluator/cache.py index faa73239a..b917baf2a 100644 --- a/evalscope/api/evaluator/cache.py +++ b/evalscope/api/evaluator/cache.py @@ -1,7 +1,8 @@ import copy import os -from pydantic import BaseModel, Field, model_validator -from typing import Any, Dict, List, Optional, Tuple, Union +import uuid +from pydantic import BaseModel, ConfigDict, Field, model_validator +from typing import Any, Dict, List, Optional, Tuple from evalscope.api.agent import AgentTrace from evalscope.api.dataset import Dataset @@ -24,7 +25,12 @@ class CacheManager: avoid redundant computations. """ - def __init__(self, outputs: OutputsStructure, model_name: str, benchmark_name: str): + def __init__( + self, + outputs: OutputsStructure, + model_name: str, + benchmark_name: str, + ): """ Initialize the cache manager. @@ -37,6 +43,7 @@ def __init__(self, outputs: OutputsStructure, model_name: str, benchmark_name: s self.model_name = model_name self.benchmark_name = benchmark_name self._writers: Dict[str, JsonlWriter] = {} + self._review_reruns: Dict[str, str] = {} def _get_writer(self, cache_file: str) -> JsonlWriter: """Return a persistent writer for *cache_file*, opening on first use. @@ -169,7 +176,7 @@ def filter_review_cache(self, subset: str, # Process each cached review result for cache_item in cache_items: # Deserialize the cached review result - cached_review_result = ReviewResult.model_validate(cache_item) + cached_review_result = ReviewResult.from_cache_item(cache_item) cached_sample_scores.append(cached_review_result.to_sample_score()) # Filter out task states that already have review scores @@ -196,11 +203,32 @@ def get_review_cache_path(self, subset: str) -> str: return file_path def delete_review_cache(self, subset: str): - """Delete the review cache for a specific subset. If the cache exists, it will be removed.""" + """Start a transactional review rerun without touching the previous review file.""" file_path = self.get_review_cache_path(subset) - if os.path.exists(file_path): - logger.info(f'Deleting review cache file: {file_path}') - os.remove(file_path) + temporary = f'{file_path}.rerun-{uuid.uuid4().hex}' + self._review_reruns[file_path] = temporary + logger.debug(f'Rescoring reviews into temporary cache: {temporary}') + + def commit_review_reruns(self) -> None: + """Atomically publish every fully written review rerun.""" + for file_path, temporary in list(self._review_reruns.items()): + writer = self._writers.pop(temporary, None) + if writer is not None: + writer.close() + if not os.path.exists(temporary): + continue + os.replace(temporary, file_path) + del self._review_reruns[file_path] + + def discard_review_reruns(self) -> None: + """Close and remove incomplete transactional review files after a failed rerun.""" + for file_path, temporary in list(self._review_reruns.items()): + writer = self._writers.pop(temporary, None) + if writer is not None: + writer.close() + if os.path.exists(temporary): + os.unlink(temporary) + del self._review_reruns[file_path] def save_review_cache( self, @@ -221,6 +249,7 @@ def save_review_cache( The saved review result object """ cache_file = self.get_review_cache_path(subset) + cache_file = self._review_reruns.get(cache_file, cache_file) # Convert score and state to serializable review result review_result = ReviewResult.from_score_state(sample_score, task_state, save_metadata) # Serialize to dictionary, convert non-JSON types (numpy, datetime), append. @@ -367,6 +396,8 @@ class ReviewResult(BaseModel): including the computed score and relevant context. """ + model_config = ConfigDict(extra='ignore') + index: int """Index of the sample that was reviewed.""" @@ -405,6 +436,11 @@ def _migrate_legacy_input(cls, data: Any) -> Any: data.pop('trajectory', None) return data + @classmethod + def from_cache_item(cls, data: Any) -> 'ReviewResult': + """Load a review result from an on-disk cache row.""" + return cls.model_validate(data) + @property def messages_markdown(self) -> str: """Render ``messages`` as markdown (multi-modal aware).""" diff --git a/evalscope/api/judge/__init__.py b/evalscope/api/judge/__init__.py new file mode 100644 index 000000000..c162e6a3e --- /dev/null +++ b/evalscope/api/judge/__init__.py @@ -0,0 +1,47 @@ +"""Native LLM judge subsystem: a JSON output contract and a single execution path. + +Benchmark adapters declare judge cases and the JSON schema each reply must satisfy; +:class:`JudgeExecutor` owns every judge call, the retry boundaries, position swapping and +aggregation. Adapters never call a judge model and never parse a judge response. + +The judge must be a model that can follow a structured-output instruction reliably; a weak judge +shows up as a high parse-failure rate in ``JudgeSummary.failures``. +""" +from .contracts import OutputContract, ParseResult +from .definition import JudgeDefinition +from .executor import JudgeExecutor, JudgeExecutorConfig +from .summary import summarize_judge_disagreement, summarize_judge_runs +from .types import ( + CaseVerdict, + JudgeAttempt, + JudgeCase, + JudgeContext, + JudgeObservation, + JudgeRequest, + JudgeReview, + PairwiseOutcome, + PairwisePlacementOutcome, + Placement, + ReducedVerdict, +) + +__all__ = [ + 'CaseVerdict', + 'JudgeAttempt', + 'JudgeCase', + 'JudgeContext', + 'JudgeDefinition', + 'JudgeExecutor', + 'JudgeExecutorConfig', + 'JudgeObservation', + 'JudgeRequest', + 'JudgeReview', + 'OutputContract', + 'ParseResult', + 'Placement', + 'PairwiseOutcome', + 'PairwisePlacementOutcome', + 'ReducedVerdict', + 'summarize_judge_disagreement', + 'summarize_judge_runs', +] diff --git a/evalscope/api/judge/aggregation.py b/evalscope/api/judge/aggregation.py new file mode 100644 index 000000000..50ed21f00 --- /dev/null +++ b/evalscope/api/judge/aggregation.py @@ -0,0 +1,249 @@ +"""Pure aggregation helpers for Native judge reviews. + +The executor owns request execution and review lifecycle. This module owns only deterministic +reduction of already parsed verdicts, so aggregation rules can be inspected without following +transport or adapter code. +""" +import math +import statistics +from collections import Counter, defaultdict +from pydantic import BaseModel +from typing import Any, Dict, Iterable, List, Optional, Sequence + +from .types import JudgeObservation, PairwiseOutcome, PairwisePlacementOutcome + + +def aggregate_repeat_values(values: Sequence[Dict[str, float]], method: str) -> tuple[Dict[str, float], List[str]]: + """Reduce one judge's repeats and return metrics that needed a first-observation tie-break.""" + buckets = _metric_buckets(values) + if method == 'median': + return {name: float(statistics.median(items)) for name, items in buckets.items()}, [] + if method == 'majority_vote': + result: Dict[str, float] = {} + ties: List[str] = [] + for name, items in buckets.items(): + winner, tied = _majority(items) + if tied: + # Repeats have no independent primary judge. The first valid observation is + # deterministic; callers mark the review degraded and retain this diagnostic. + winner = items[0] + ties.append(name) + result[name] = winner + return result, ties + return {name: sum(items) / len(items) for name, items in buckets.items()}, [] + + +def aggregate_judge_values(judge_values: Dict[str, Dict[str, float]], method: str, + primary: str) -> tuple[Optional[Dict[str, float]], bool, List[str]]: + """Combine equally weighted judges, using the primary judge only for a tied majority vote.""" + buckets = _metric_buckets(judge_values.values()) + result: Dict[str, float] = {} + tie_broken = False + unresolved_ties: List[str] = [] + for name, values in buckets.items(): + if method == 'median': + result[name] = float(statistics.median(values)) + elif method == 'majority_vote': + winner, tied = _majority(values) + if tied: + if primary not in judge_values or name not in judge_values[primary]: + unresolved_ties.append(name) + continue + winner = judge_values[primary][name] + tie_broken = True + result[name] = winner + else: + result[name] = sum(values) / len(values) + return result or None, tie_broken, unresolved_ties + + +def aggregate_pairwise_outcomes(observations: Sequence[JudgeObservation], primary: str, + min_valid_judges: int) -> tuple[Optional[PairwiseOutcome], bool]: + """Vote candidate-oriented pairwise results and their presentation-order diagnostics.""" + per_judge: Dict[str, List[PairwiseOutcome]] = defaultdict(list) + for observation in observations: + outcome = observation.reduced.outcome if observation.reduced is not None else None + if outcome is not None: + per_judge[observation.judge_id].append(outcome) + if len(per_judge) < min_valid_judges: + return None, False + + judge_outcomes = {judge_id: _vote_pairwise(values, repeat_vote=True)[0] for judge_id, values in per_judge.items()} + outcome, tied = _vote_pairwise( + list(judge_outcomes.values()), repeat_vote=False, primary=judge_outcomes.get(primary) + ) + if tied and primary not in judge_outcomes: + return None, False + + placements: Dict[str, PairwisePlacementOutcome] = {} + by_placement: Dict[str, Dict[str, List[PairwisePlacementOutcome]]] = defaultdict(lambda: defaultdict(list)) + for judge_id, outcomes in per_judge.items(): + for candidate_outcome in outcomes: + for placement, placement_outcome in candidate_outcome.placements.items(): + by_placement[placement][judge_id].append(placement_outcome) + for placement, per_judge_placements in by_placement.items(): + if len(per_judge_placements) < min_valid_judges: + continue + judge_placements = { + judge_id: _vote_pairwise_placements(values, repeat_vote=True)[0] + for judge_id, values in per_judge_placements.items() + } + placement_outcome, placement_tied = _vote_pairwise_placements( + list(judge_placements.values()), + repeat_vote=False, + primary=judge_placements.get(primary), + ) + if placement_tied and primary not in judge_placements: + return None, False + placements[placement] = placement_outcome + tied = tied or placement_tied + return outcome.model_copy(update={'placements': placements}), tied + + +def judge_disagreement(per_judge: Dict[str, List[Dict[str, float]]], + observations: Sequence[JudgeObservation]) -> Dict[str, Any]: + """Build per-sample numeric, categorical, and position-consistency diagnostics.""" + all_values = _metric_buckets(value for values in per_judge.values() for value in values) + numeric = { + name: { + 'std': statistics.pstdev(values) if len(values) > 1 else 0.0, + 'range': max(values) - min(values), + } + for name, values in all_values.items() + } + repeat_numeric = { + judge_id: _numeric_disagreement(values) + for judge_id, values in per_judge.items() + if len(values) > 1 + } + per_judge_mean = {judge_id: aggregate_repeat_values(values, 'mean')[0] for judge_id, values in per_judge.items()} + categorical = _categorical_disagreement_by_case(observations) + positions = _position_consistency(observations) + return { + 'numeric': { + 'all_observations': numeric, + 'repeats': repeat_numeric, + 'cross_judge': _numeric_disagreement(list(per_judge_mean.values())), + }, + 'categorical': categorical, + 'position_consistency': sum(positions) / len(positions) if positions else None, + 'swap_flip_count': sum(not value for value in positions), + } + + +def _metric_buckets(values: Iterable[Dict[str, float]]) -> Dict[str, List[float]]: + buckets: Dict[str, List[float]] = defaultdict(list) + for value in values: + for name, item in value.items(): + buckets[name].append(float(item)) + return buckets + + +def _majority(values: Sequence[float]) -> tuple[float, bool]: + counts = Counter(values) + top = max(counts.values()) + winners = [value for value, count in counts.items() if count == top] + return winners[0], len(winners) > 1 + + +def _numeric_disagreement(values: Sequence[Dict[str, float]]) -> Dict[str, Dict[str, float]]: + return { + name: { + 'std': statistics.pstdev(items) if len(items) > 1 else 0.0, + 'range': max(items) - min(items), + } + for name, items in _metric_buckets(values).items() + } + + +def _categorical_disagreement_by_case(observations: Sequence[JudgeObservation]) -> Dict[str, Dict[str, Any]]: + categories: Dict[str, List[tuple[str, str]]] = defaultdict(list) + for observation in observations: + outcome = observation.reduced.outcome if observation.reduced is not None else None + if outcome is not None: + categories[f'pairwise/{outcome.metric_name}'].append((observation.judge_id, outcome.result)) + continue + for verdict in observation.case_verdicts: + label = _categorical_label(verdict.value) + if label is not None: + categories[verdict.case_id].append((observation.judge_id, label)) + + result: Dict[str, Dict[str, Any]] = {} + for case_id, labels in categories.items(): + per_judge: Dict[str, List[str]] = defaultdict(list) + for judge_id, label in labels: + per_judge[judge_id].append(label) + result[case_id] = { + **_categorical_disagreement([label for _, label in labels]), + 'repeats': { + judge_id: _categorical_disagreement(values) + for judge_id, values in per_judge.items() + if len(values) > 1 + }, + 'cross_judge': _categorical_disagreement([_majority_label(values) for values in per_judge.values()]), + } + return result + + +def _categorical_label(value: Any) -> Optional[str]: + if isinstance(value, BaseModel) and hasattr(value, 'verdict'): + return str(value.verdict) + return value if isinstance(value, str) else None + + +def _categorical_disagreement(labels: Sequence[str]) -> Dict[str, float]: + if not labels: + return {'agreement_ratio': 0.0, 'vote_entropy': 0.0} + counts = Counter(labels) + total = len(labels) + entropy = -sum((count / total) * math.log2(count / total) for count in counts.values()) + return {'agreement_ratio': max(counts.values()) / total, 'vote_entropy': entropy} + + +def _majority_label(labels: Sequence[str]) -> str: + return Counter(labels).most_common(1)[0][0] + + +def _position_consistency(observations: Sequence[JudgeObservation]) -> List[bool]: + positions: List[bool] = [] + for observation in observations: + reduced = observation.reduced + outcome = reduced.outcome if reduced is not None else None + if outcome is not None and outcome.placements: + positions.append(len({value.result for value in outcome.placements.values()}) == 1) + elif reduced is not None and reduced.position_results: + positions.append(len(set(reduced.position_results.values())) == 1) + return positions + + +def _vote_pairwise(outcomes: Sequence[PairwiseOutcome], + repeat_vote: bool, + primary: Optional[PairwiseOutcome] = None) -> tuple[PairwiseOutcome, bool]: + """Return a semantic majority; ties within a judge's repeats deliberately become a draw.""" + counts = Counter(outcome.result for outcome in outcomes) + top = max(counts.values()) + winners = [result for result, count in counts.items() if count == top] + tied = len(winners) > 1 + result = 'tie' if tied and repeat_vote else primary.result if tied and primary is not None else winners[0] + selected = [outcome for outcome in outcomes if outcome.result == result] + strength = 'weak' + if result != 'tie' and selected: + strength = 'strong' if sum(item.strength == 'strong' for item in selected) * 2 > len(selected) else 'weak' + template = selected[0] if selected else outcomes[0] + return PairwiseOutcome(metric_name=template.metric_name, result=result, strength=strength), tied + + +def _vote_pairwise_placements( + outcomes: Sequence[PairwisePlacementOutcome], + repeat_vote: bool, + primary: Optional[PairwisePlacementOutcome] = None, +) -> tuple[PairwisePlacementOutcome, bool]: + proxies = [ + PairwiseOutcome(metric_name='placement', result=item.result, strength=item.strength) for item in outcomes + ] + primary_proxy = ( + PairwiseOutcome(metric_name='placement', result=primary.result, strength=primary.strength) + if primary is not None else None + ) + voted, tied = _vote_pairwise(proxies, repeat_vote, primary_proxy) + return PairwisePlacementOutcome(result=voted.result, strength=voted.strength), tied diff --git a/evalscope/api/judge/contracts.py b/evalscope/api/judge/contracts.py new file mode 100644 index 000000000..077d13234 --- /dev/null +++ b/evalscope/api/judge/contracts.py @@ -0,0 +1,128 @@ +"""The judge's output contract: a single JSON object, validated against a Pydantic schema. + +The contract owns both halves of the format agreement -- the requirement written into the prompt +and the parser that reads the reply -- so the two cannot drift apart. A reply that does not +satisfy the schema is a ``parse_error``, never a silently-zero score. +""" +import json +from pydantic import BaseModel, ConfigDict, Field +from typing import Any, List, Literal, Optional, Sequence, Type, get_args, get_origin + + +class ParseResult(BaseModel): + """Outcome of parsing one judge response against a contract.""" + + model_config = ConfigDict(frozen=True) + + ok: bool + value: Any = Field(default=None) + error: Optional[str] = Field(default=None) + + @classmethod + def success(cls, value: Any) -> 'ParseResult': + return cls(ok=True, value=value) + + @classmethod + def failure(cls, error: str) -> 'ParseResult': + return cls(ok=False, error=error) + + +class OutputContract(BaseModel): + """The JSON reply one judge case requires.""" + + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + schema_model: Type[BaseModel] + + def instruction(self) -> str: + """The format requirement appended to the benchmark's own prompt.""" + # The alias is the key the judge must actually emit when a schema declares one. + fields = '\n'.join( + f'- "{field.alias or name}": {_describe(field)}' for name, field in self.schema_model.model_fields.items() + ) + return ( + '\n\nReply with a single JSON object and no other text, containing these required keys ' + '(additional keys are allowed):\n' + f'{fields}' + ) + + def parse(self, response: str) -> ParseResult: + """Parse one judge reply through the schema's normal Pydantic coercion rules.""" + if not response: + return ParseResult.failure('empty response') + + payloads = _payloads(response) + if not payloads: + return ParseResult.failure('no JSON object found in the reply') + if len(payloads) > 1: + return ParseResult.failure(f'{len(payloads)} JSON objects found where exactly one is required') + + try: + data = json.loads(payloads[0]) + except json.JSONDecodeError as exc: + return ParseResult.failure(f'reply is not valid JSON: {exc}') + if not isinstance(data, dict): + return ParseResult.failure(f'reply is a {type(data).__name__}, not a JSON object') + + try: + return ParseResult.success(self.schema_model.model_validate(data)) + except Exception as exc: + return ParseResult.failure(f'reply does not satisfy {self.schema_model.__name__}: {exc}') + + +def _payloads(response: str) -> List[str]: + """Return every top-level JSON object in the reply. + + Scanning rather than anchoring lets a reasoning judge wrap its answer in a ```` block + or a fence; requiring exactly one object keeps a two-verdict reply a failure, not a coin flip. + """ + found: List[str] = [] + depth = 0 + start = 0 + in_string = False + escaped = False + for index, char in enumerate(response): + if in_string: + if escaped: + escaped = False + elif char == '\\': + escaped = True + elif char == '"': + in_string = False + continue + if char == '"': + in_string = True + elif char == '{': + if depth == 0: + start = index + depth += 1 + elif char == '}' and depth: + depth -= 1 + if depth == 0: + found.append(response[start:index + 1]) + return found + + +def _describe(field: Any) -> str: + """Describe one schema field for the prompt, so the judge is told the exact allowed values.""" + annotation = field.annotation + if get_origin(annotation) is Literal: + allowed = ' or '.join(f'"{value}"' for value in get_args(annotation)) + return f'exactly one of {allowed}' + + bounds = [ + f'>= {meta.ge}' + if getattr(meta, 'ge', None) is not None else f'<= {meta.le}' if getattr(meta, 'le', None) is not None else '' + for meta in field.metadata + ] + bounds = [bound for bound in bounds if bound] + kind = ( + 'true or false' if annotation is bool else 'an integer' if annotation is int else + 'a number' if annotation is float else 'a list' if get_origin(annotation) is list else 'a string' + ) + if bounds: + return f'{kind} {" and ".join(bounds)}' + return f'{kind}' + (f' ({field.description})' if field.description else '') + + +__all__: Sequence[str] = ('OutputContract', 'ParseResult') diff --git a/evalscope/api/judge/definition.py b/evalscope/api/judge/definition.py new file mode 100644 index 000000000..3b48c9002 --- /dev/null +++ b/evalscope/api/judge/definition.py @@ -0,0 +1,221 @@ +"""Adapter-facing declarations for one Native LLM judge review.""" +from dataclasses import dataclass +from pydantic import BaseModel +from typing import Any, Callable, Dict, Optional, Sequence, Type + +from evalscope.api.messages import ChatMessageSystem, ChatMessageUser +from evalscope.api.metric import Score +from .contracts import OutputContract +from .types import CaseVerdict, JudgeCase, JudgeContext, JudgeRequest, JudgeReview, Placement, ReducedVerdict + +RequestBuilder = Callable[[JudgeCase, Placement, Sequence[CaseVerdict], JudgeContext], JudgeRequest] +VerdictReducer = Callable[[Sequence[CaseVerdict], JudgeContext], ReducedVerdict] +CaseExpander = Callable[[int, Sequence[CaseVerdict], JudgeContext], Sequence[JudgeCase]] +FallbackBuilder = Callable[[JudgeCase, JudgeContext], Optional[CaseVerdict]] +ScoreFinalizer = Callable[[Score, JudgeReview, JudgeContext], Score] + + +@dataclass +class JudgeDefinition: + """A complete, sample-scoped judge declaration. + + ``labels`` and ``numeric`` cover ordinary single-verdict tasks. ``workflow`` keeps the + rare multi-case and staged tasks expressible without exposing executor hooks on adapters. + """ + + cases: Sequence[JudgeCase] + request: RequestBuilder + reduce: VerdictReducer + main_score_name: Optional[str] = None + expand: Optional[CaseExpander] = None + fallback: Optional[FallbackBuilder] = None + immediate_score: Optional[Score] = None + skip_reason: Optional[str] = None + finalize: Optional[ScoreFinalizer] = None + + def __post_init__(self) -> None: + """Keep rule short-circuits observable and distinct from judge workflows.""" + if self.immediate_score is not None and not self.skip_reason: + raise ValueError('A skipped judge definition requires a non-empty skip_reason.') + if self.immediate_score is None and self.skip_reason is not None: + raise ValueError('skip_reason is only valid for a skipped judge definition.') + + @classmethod + def labels( + cls, + *, + prompt: str, + schema_model: Type[BaseModel], + scores: Dict[str, Dict[str, float]], + case_id: str = 'judge', + system_prompt: Optional[str] = None, + verdict_field: str = 'verdict', + fallback_verdict: Optional[str] = None, + main_score_name: Optional[str] = None, + finalize: Optional[ScoreFinalizer] = None, + ) -> 'JudgeDefinition': + """Declare a one-case label judge with a label-to-metrics mapping.""" + contract = OutputContract(schema_model=schema_model) + case = JudgeCase(case_id=case_id, output_contract=contract) + + def request_builder( + current: JudgeCase, + placement: Placement, + completed: Sequence[CaseVerdict], + context: JudgeContext, + ) -> JudgeRequest: + messages = [] + if system_prompt: + messages.append(ChatMessageSystem(content=system_prompt)) + messages.append(ChatMessageUser(content=prompt + current.output_contract.instruction())) + return JudgeRequest(messages=messages) + + def reducer(verdicts: Sequence[CaseVerdict], context: JudgeContext) -> ReducedVerdict: + label = str(getattr(verdicts[0].value, verdict_field)) + return ReducedVerdict(value=dict(scores[label])) + + fallback = None + if fallback_verdict is not None: + + def fallback_builder(current: JudgeCase, context: JudgeContext) -> CaseVerdict: + return CaseVerdict( + case_id=current.case_id, + value=schema_model.model_validate({verdict_field: fallback_verdict}), + ) + + fallback = fallback_builder + return cls( + cases=[case], + request=request_builder, + reduce=reducer, + fallback=fallback, + main_score_name=main_score_name, + finalize=finalize, + ) + + @classmethod + def numeric( + cls, + *, + prompt: str, + schema_model: Type[BaseModel], + metric_name: str = 'acc', + case_id: str = 'judge', + system_prompt: Optional[str] = None, + score_field: str = 'score', + main_score_name: Optional[str] = None, + finalize: Optional[ScoreFinalizer] = None, + ) -> 'JudgeDefinition': + """Declare a one-case numeric judge that copies a schema field into one metric.""" + contract = OutputContract(schema_model=schema_model) + case = JudgeCase(case_id=case_id, output_contract=contract) + + def request_builder( + current: JudgeCase, + placement: Placement, + completed: Sequence[CaseVerdict], + context: JudgeContext, + ) -> JudgeRequest: + messages = [] + if system_prompt: + messages.append(ChatMessageSystem(content=system_prompt)) + messages.append(ChatMessageUser(content=prompt + current.output_contract.instruction())) + return JudgeRequest(messages=messages) + + def reducer(verdicts: Sequence[CaseVerdict], context: JudgeContext) -> ReducedVerdict: + return ReducedVerdict(value={metric_name: float(getattr(verdicts[0].value, score_field))}) + + return cls( + cases=[case], + request=request_builder, + reduce=reducer, + main_score_name=main_score_name or metric_name, + finalize=finalize, + ) + + @classmethod + def workflow( + cls, + *, + cases: Sequence[JudgeCase], + request: RequestBuilder, + reduce: VerdictReducer, + main_score_name: Optional[str] = None, + expand: Optional[CaseExpander] = None, + fallback: Optional[FallbackBuilder] = None, + finalize: Optional[ScoreFinalizer] = None, + ) -> 'JudgeDefinition': + """Declare multiple or staged cases with callbacks scoped to this definition.""" + return cls( + cases=cases, + request=request, + reduce=reduce, + main_score_name=main_score_name, + expand=expand, + fallback=fallback, + finalize=finalize, + ) + + @classmethod + def skip(cls, score: Score, *, reason: str) -> 'JudgeDefinition': + """Return a rule-derived score without opening a judge session. + + ``reason`` is recorded in the resulting score metadata so consumers can distinguish a + deterministic short-circuit from an LLM judge verdict. + """ + return cls( + cases=[], + request=_unreachable_request, + reduce=_unreachable_reduce, + immediate_score=score, + skip_reason=reason, + ) + + # The following methods are executor-facing only. They deliberately live on the definition, + # rather than on benchmark adapters, so an adapter exposes one public extension point. + def build_cases(self, context: JudgeContext) -> Sequence[JudgeCase]: + """Return the initial cases declared for this review.""" + return self.cases + + def build_request( + self, + case: JudgeCase, + placement: Placement, + completed_cases: Sequence[CaseVerdict], + context: JudgeContext, + ) -> JudgeRequest: + """Render one declared case.""" + return self.request(case, placement, completed_cases, context) + + def expand_cases( + self, + stage: int, + completed_cases: Sequence[CaseVerdict], + context: JudgeContext, + ) -> Sequence[JudgeCase]: + """Return derived cases for the next stage, when configured.""" + return self.expand(stage, completed_cases, context) if self.expand is not None else [] + + def fallback_verdict(self, case: JudgeCase, context: JudgeContext) -> Optional[CaseVerdict]: + """Return the declared rule fallback, when configured.""" + return self.fallback(case, context) if self.fallback is not None else None + + def reduce_verdicts(self, verdicts: Sequence[CaseVerdict], context: JudgeContext) -> ReducedVerdict: + """Fold parsed case verdicts into this review's metric values.""" + return self.reduce(verdicts, context) + + +def _unreachable_request( + case: JudgeCase, + placement: Placement, + completed: Sequence[CaseVerdict], + context: JudgeContext, +) -> JudgeRequest: + raise RuntimeError('A skipped judge definition cannot build a request.') + + +def _unreachable_reduce(verdicts: Sequence[CaseVerdict], context: JudgeContext) -> ReducedVerdict: + raise RuntimeError('A skipped judge definition cannot reduce verdicts.') + + +__all__ = ['JudgeDefinition'] diff --git a/evalscope/api/judge/executor.py b/evalscope/api/judge/executor.py new file mode 100644 index 000000000..b9f6043be --- /dev/null +++ b/evalscope/api/judge/executor.py @@ -0,0 +1,343 @@ +"""The single execution path for Native LLM judge contracts.""" +import time +from collections import Counter, defaultdict +from pydantic import BaseModel, ConfigDict, Field +from typing import Any, Dict, List, Literal, Optional, Sequence + +from evalscope.api.metric import JudgeSummary, Score +from evalscope.constants import ScoreStatus +from evalscope.utils.logger import get_logger +from .aggregation import ( + aggregate_judge_values, + aggregate_pairwise_outcomes, + aggregate_repeat_values, + judge_disagreement, +) +from .definition import JudgeDefinition +from .types import CaseVerdict, JudgeAttempt, JudgeCase, JudgeContext, JudgeObservation, JudgeReview, Placement + +logger = get_logger() + +MAX_STAGES = 4 + + +class JudgeExecutorConfig(BaseModel): + """The resolved execution policy for one benchmark run.""" + + model_config = ConfigDict(frozen=True) + + repeats: int = Field(default=1, ge=1) + position_swap: bool = False + aggregation: Literal['mean', 'median', 'majority_vote'] = 'mean' + min_valid_judges: int = Field(default=1, ge=1) + + +class JudgeExecutor: + """Execute one complete judge session at a time, serially within each sample.""" + + def __init__(self, judges: Sequence[Any], config: Optional[JudgeExecutorConfig] = None) -> None: + self.judges = list(judges) + self.config = config or JudgeExecutorConfig() + if not self.judges: + raise ValueError('JudgeExecutor requires at least one judge model.') + if self.config.min_valid_judges > len(self.judges): + raise ValueError('min_valid_judges cannot exceed the configured judge count.') + + def execute(self, definition: JudgeDefinition, context: JudgeContext) -> JudgeReview: + """Score one sample through every configured judge and repeat.""" + review = JudgeReview() + for judge in self.judges: + for repeat_id in range(self.config.repeats): + review.observations.append(self._run_observation(definition, context, judge, repeat_id, review)) + review.failure_counts = dict( + Counter(attempt.status.value for attempt in review.attempts if not attempt.status.is_usable) + ) + self._aggregate(review) + return review + + def _run_observation( + self, + definition: JudgeDefinition, + context: JudgeContext, + judge: Any, + repeat_id: int, + review: JudgeReview, + ) -> JudgeObservation: + observation = JudgeObservation(judge_id=_judge_id(judge), repeat_id=repeat_id) + pending = list(definition.build_cases(context)) + declared = len(pending) + completed: List[CaseVerdict] = [] + for stage in range(MAX_STAGES): + for case in pending: + verdict = self._run_case(definition, context, judge, repeat_id, case, completed, review) + if verdict is None: + observation.status = ScoreStatus.INVALID_SESSION + observation.error = f'case {case.case_id} produced no usable verdict' + return observation + completed.append(verdict) + pending = list(definition.expand_cases(stage + 1, completed, context)) + declared += len(pending) + if pending: + raise RuntimeError(f'Judge case expansion exceeded {MAX_STAGES} stages; adapter bug.') + if declared and not completed: + observation.status = ScoreStatus.EXCLUDED + observation.error = 'no case produced a usable verdict' + return observation + observation.case_verdicts = completed + observation.reduced = definition.reduce_verdicts(completed, context) + if any(verdict.status is ScoreStatus.FALLBACK for verdict in completed): + observation.status = ScoreStatus.FALLBACK + return observation + + def _run_case( + self, + definition: JudgeDefinition, + context: JudgeContext, + judge: Any, + repeat_id: int, + case: JudgeCase, + completed: Sequence[CaseVerdict], + review: JudgeReview, + ) -> Optional[CaseVerdict]: + placements = (Placement.ORIGINAL, Placement.SWAPPED) if self.config.position_swap else (Placement.ORIGINAL, ) + values: Dict[str, Any] = {} + for placement in placements: + value = self._resolve_placement(definition, context, judge, repeat_id, case, placement, completed, review) + if value is None: + # A swapped pair is atomic. One rule fallback must not impersonate the missing + # opposite presentation. + if len(placements) > 1: + return None + fallback = definition.fallback_verdict(case, context) + if fallback is not None: + return fallback.model_copy( + update={ + 'status': ScoreStatus.FALLBACK, + 'metadata': fallback.metadata or dict(case.metadata) + } + ) + return None + values[placement.value] = value + if len(placements) == 1: + return CaseVerdict( + case_id=case.case_id, value=values[Placement.ORIGINAL.value], metadata=dict(case.metadata) + ) + return CaseVerdict( + case_id=case.case_id, + value=[values[Placement.ORIGINAL.value], values[Placement.SWAPPED.value]], + placements=values, + metadata=dict(case.metadata), + ) + + def _resolve_placement( + self, + definition: JudgeDefinition, + context: JudgeContext, + judge: Any, + repeat_id: int, + case: JudgeCase, + placement: Placement, + completed: Sequence[CaseVerdict], + review: JudgeReview, + ) -> Any: + """Make exactly one request. Transport retry belongs to the model implementation.""" + request = definition.build_request(case, placement, completed, context) + started = time.perf_counter() + try: + output = judge.generate(list(request.messages)) + except Exception as exc: # provider exceptions become inspectable failed attempts + review.attempts.append( + JudgeAttempt( + status=ScoreStatus.TRANSPORT_ERROR, + case_id=case.case_id, + judge_id=_judge_id(judge), + repeat_id=repeat_id, + placement=placement, + messages=list(request.messages), + error=f'{type(exc).__name__}: {exc}', + latency=time.perf_counter() - started, + ) + ) + return None + raw = output.completion + attempt = JudgeAttempt( + status=ScoreStatus.SUCCESS, + case_id=case.case_id, + judge_id=_judge_id(judge), + repeat_id=repeat_id, + placement=placement, + messages=list(request.messages), + model_output=output, + raw_response=raw, + latency=time.perf_counter() - started, + ) + result = case.output_contract.parse(raw) + if result.ok: + attempt.parsed_value = _jsonable(result.value) + review.attempts.append(attempt) + return result.value + attempt.status = ScoreStatus.PARSE_ERROR + attempt.error = result.error + review.attempts.append(attempt) + logger.warning(f'Judge {_judge_id(judge)} failed the output contract for case {case.case_id}; excluding it.') + return None + + def _aggregate(self, review: JudgeReview) -> None: + """Reduce repeats inside a judge, then combine equally weighted judge outcomes.""" + per_judge: Dict[str, List[Dict[str, float]]] = defaultdict(list) + pairwise_metrics: Dict[str, set[str]] = defaultdict(set) + for observation in review.valid_observations: + per_judge[observation.judge_id].append(observation.reduced.value) + if observation.reduced.outcome is not None: + pairwise_metrics[observation.judge_id].add(observation.reduced.outcome.metric_name) + if not per_judge: + if review.fallback_observations: + self._aggregate_rule_fallback(review) + return + review.status = ScoreStatus.EXCLUDED + review.error = 'no judge produced a usable verdict' + return + if len(per_judge) < self.config.min_valid_judges: + if review.fallback_observations: + self._aggregate_rule_fallback(review) + return + review.status = ScoreStatus.EXCLUDED + review.error = f'only {len(per_judge)} judge(s) produced valid verdicts; need {self.config.min_valid_judges}' + return + judge_values: Dict[str, Dict[str, float]] = {} + repeat_ties: Dict[str, List[str]] = {} + for judge_id, values in per_judge.items(): + judge_values[judge_id], ties = aggregate_repeat_values(values, self.config.aggregation) + ties = [metric_name for metric_name in ties if metric_name not in pairwise_metrics[judge_id]] + if ties: + repeat_ties[judge_id] = ties + eligible_metrics = { + metric_name + for values in judge_values.values() + for metric_name in values + if sum(metric_name in judge_value for judge_value in judge_values.values()) >= self.config.min_valid_judges + } + if not eligible_metrics: + review.status = ScoreStatus.EXCLUDED + review.error = 'no metric received enough valid judge verdicts' + return + excluded_metrics = sorted({metric_name + for values in judge_values.values() + for metric_name in values} - eligible_metrics) + judge_values = { + judge_id: { + metric_name: value + for metric_name, value in values.items() + if metric_name in eligible_metrics + } + for judge_id, values in judge_values.items() + } + combined, tie_broken, unresolved_ties = aggregate_judge_values( + judge_values, self.config.aggregation, _judge_id(self.judges[0]) + ) + if combined is None: + review.status = ScoreStatus.EXCLUDED + review.error = 'no metric could be aggregated after applying the majority_vote tie-break rule' + return + review.value = combined + pairwise, pairwise_tie_broken = aggregate_pairwise_outcomes( + review.valid_observations, _judge_id(self.judges[0]), self.config.min_valid_judges + ) + if pairwise is not None: + review.outcome = pairwise + review.value[pairwise.metric_name] = pairwise.score + tie_broken = tie_broken or pairwise_tie_broken + primary_observation = next( + ( + observation for observation in review.valid_observations + if observation.judge_id == _judge_id(self.judges[0]) + ), + review.valid_observations[0], + ) + review.metadata = dict(primary_observation.reduced.metadata) + review.observation_metadata = [{ + 'judge_id': observation.judge_id, + 'repeat_id': observation.repeat_id, + 'status': observation.status.value, + 'metadata': dict(observation.reduced.metadata), + } for observation in review.usable_observations] + review.disagreement = judge_disagreement(per_judge, review.valid_observations) + if excluded_metrics: + review.metadata['metrics_without_quorum'] = excluded_metrics + if unresolved_ties: + review.metadata['metrics_without_primary_tiebreak'] = unresolved_ties + if repeat_ties: + review.metadata['repeat_tie_broken_by_first_observation'] = repeat_ties + expected = len(self.judges) * self.config.repeats + if tie_broken: + review.metadata['tie_broken_by_primary'] = True + review.status = ScoreStatus.DEGRADED + elif len(review.valid_observations) != expected or unresolved_ties or repeat_ties: + review.status = ScoreStatus.DEGRADED + + def _aggregate_rule_fallback(self, review: JudgeReview) -> None: + """Use one deterministic official fallback without treating it as a Judge vote.""" + primary = _judge_id(self.judges[0]) + fallback = next( + (observation for observation in review.fallback_observations if observation.judge_id == primary), + review.fallback_observations[0], + ) + review.value = dict(fallback.reduced.value) + review.outcome = fallback.reduced.outcome + review.metadata = dict(fallback.reduced.metadata) + review.observation_metadata = [{ + 'judge_id': observation.judge_id, + 'repeat_id': observation.repeat_id, + 'status': observation.status.value, + 'metadata': dict(observation.reduced.metadata), + } for observation in review.fallback_observations] + review.status = ScoreStatus.DEGRADED + + def build_score( + self, + definition: JudgeDefinition, + review: JudgeReview, + context: JudgeContext, + *, + metadata: Optional[Dict[str, Any]] = None, + ) -> Score: + """Build the final score and persist all judge I/O in the review payload.""" + score = Score( + extracted_prediction=context.filtered_prediction, + prediction=context.original_prediction, + value=dict(review.value), + main_score_name=definition.main_score_name, + metadata=dict(metadata or {}), + ) + if definition.finalize is not None: + score = definition.finalize(score, review, context) + score.status = review.status + if not review.status.is_usable: + score.value = {} + score.judge_summary = JudgeSummary( + status=review.status, + scored=int(review.status.is_usable), + total=1, + coverage=float(review.status.is_usable), + judge_models=[_judge_id(judge) for judge in self.judges], + valid_observations=len(review.valid_observations), + total_observations=len(review.observations), + failures=review.failure_counts, + disagreement=review.disagreement, + error=review.error, + ) + score.metadata = dict(score.metadata or {}) + score.metadata['judge_attempts'] = [attempt.model_dump(exclude_none=True) for attempt in review.attempts] + score.metadata['judge_observation_metadata'] = review.observation_metadata + return score + + +def _judge_id(judge: Any) -> str: + return str(getattr(judge, 'judge_id', None) or getattr(judge, 'model_id', 'unknown-judge')) + + +def _jsonable(value: Any) -> Any: + return value.model_dump() if isinstance(value, BaseModel) else value + + +__all__: Sequence[str] = ('JudgeExecutor', 'JudgeExecutorConfig') diff --git a/evalscope/api/judge/summary.py b/evalscope/api/judge/summary.py new file mode 100644 index 000000000..99d6d259e --- /dev/null +++ b/evalscope/api/judge/summary.py @@ -0,0 +1,88 @@ +"""Aggregate per-sample judge summaries into run-level summaries.""" +from collections import defaultdict +from typing import Dict, Iterable, List, Optional + +from evalscope.api.metric import JudgeSummary, SampleScore +from evalscope.constants import ScoreStatus + + +def summarize_judge_runs(score_groups: Iterable[List[SampleScore]]) -> Optional[JudgeSummary]: + """Combine per-sample judge summaries without treating unavailable samples as zero scores.""" + summaries = [ + score.score.judge_summary + for scores in score_groups + for score in scores + if score.score.judge_summary is not None + ] + if not summaries: + return None + total = len(summaries) + scored = sum(summary.status.is_usable for summary in summaries) + failures: Dict[str, int] = defaultdict(int) + models = [] + for summary in summaries: + for name, count in summary.failures.items(): + failures[name] += count + models.extend(model for model in summary.judge_models if model not in models) + status = ( + ScoreStatus.EXCLUDED if not scored else ScoreStatus.DEGRADED if scored != total + or any(summary.status is not ScoreStatus.SUCCESS for summary in summaries) else ScoreStatus.SUCCESS + ) + return JudgeSummary( + status=status, + scored=scored, + total=total, + coverage=scored / total, + judge_models=models, + valid_observations=sum(summary.valid_observations for summary in summaries), + total_observations=sum(summary.total_observations for summary in summaries), + failures=dict(failures), + disagreement=summarize_judge_disagreement(summaries), + ) + + +def summarize_judge_disagreement(summaries: List[JudgeSummary]) -> Dict[str, object]: + """Aggregate comparable disagreement diagnostics without pretending sample variances are global variance.""" + numeric: Dict[str, Dict[str, List[float]]] = defaultdict(lambda: defaultdict(list)) + categorical: Dict[str, Dict[str, List[float]]] = defaultdict(lambda: defaultdict(list)) + position_consistency: List[float] = [] + swap_flip_count = 0 + for summary in summaries: + detail = summary.disagreement + for name, values in detail.get('numeric', {}).get('all_observations', {}).items(): + for key in ('std', 'range'): + if key in values: + numeric[name][key].append(float(values[key])) + for name, values in detail.get('categorical', {}).items(): + for key in ('agreement_ratio', 'vote_entropy'): + if key in values: + categorical[name][key].append(float(values[key])) + if detail.get('position_consistency') is not None: + position_consistency.append(float(detail['position_consistency'])) + swap_flip_count += int(detail.get('swap_flip_count', 0)) + + return { + 'numeric': { + name: { + 'mean_std': sum(values['std']) / len(values['std']) if values['std'] else 0.0, + 'max_range': max(values['range']) if values['range'] else 0.0, + 'samples': max(len(values['std']), len(values['range'])), + } + for name, values in numeric.items() + }, + 'categorical': { + name: { + 'mean_agreement_ratio': sum(values['agreement_ratio']) + / len(values['agreement_ratio']) if values['agreement_ratio'] else 0.0, + 'mean_vote_entropy': sum(values['vote_entropy']) + / len(values['vote_entropy']) if values['vote_entropy'] else 0.0, + 'samples': max(len(values['agreement_ratio']), len(values['vote_entropy'])), + } + for name, values in categorical.items() + }, + 'position_consistency': { + 'mean': sum(position_consistency) / len(position_consistency) if position_consistency else None, + 'samples': len(position_consistency), + 'swap_flip_count': swap_flip_count, + }, + } diff --git a/evalscope/api/judge/types.py b/evalscope/api/judge/types.py new file mode 100644 index 000000000..472b22c7a --- /dev/null +++ b/evalscope/api/judge/types.py @@ -0,0 +1,207 @@ +"""Contracts for one judge review: what is asked, what came back, and what the adapter must do.""" +from enum import Enum +from pydantic import BaseModel, ConfigDict, Field +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Sequence + +from evalscope.api.evaluator import TaskState +from evalscope.api.messages import ChatMessage +from evalscope.api.model import ModelOutput +from evalscope.constants import ScoreStatus +from .contracts import OutputContract + +if TYPE_CHECKING: + from evalscope.api.metric import Score + + +class Placement(str, Enum): + """Which side of a pairwise comparison a request presents first.""" + + ORIGINAL = 'original' + SWAPPED = 'swapped' + + +class JudgeContext(BaseModel): + """Everything the adapter's judge hooks may read about one sample.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + task_state: TaskState + original_prediction: str + filtered_prediction: str + reference: str + + +class JudgeCase(BaseModel): + """One thing the judge is asked about a sample: a rubric, a claim, a criterion, a cell.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + case_id: str + """Stable identifier within the sample.""" + + output_contract: OutputContract + """Declares the verdict shape and is the only thing allowed to parse the response.""" + + metadata: Dict[str, Any] = Field(default_factory=dict) + """Adapter-defined data carried through the request callback and onto the verdict.""" + + +class JudgeRequest(BaseModel): + """What to send the judge for one case. The adapter decides the content; the executor owns + the identity (judge, repeat, placement).""" + + messages: List[ChatMessage] + + +class JudgeAttempt(BaseModel): + """One request/response round trip, including the ones that failed.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + status: ScoreStatus + case_id: str + judge_id: str + repeat_id: int = 0 + placement: Placement = Placement.ORIGINAL + messages: List[ChatMessage] = Field(default_factory=list) + model_output: Optional[ModelOutput] = None + raw_response: Optional[str] = None + parsed_value: Any = None + error: Optional[str] = None + latency: Optional[float] = None + + +class CaseVerdict(BaseModel): + """The usable outcome of one case, after any retry and after both placements.""" + + case_id: str + value: Any + status: ScoreStatus = ScoreStatus.SUCCESS + placements: Dict[str, Any] = Field(default_factory=dict) + """Per-placement parsed values, present when the case was judged on both sides.""" + + metadata: Dict[str, Any] = Field(default_factory=dict) + """Copied from the case, so a reduce step reads its context instead of parsing ``case_id``.""" + + +class PairwisePlacementOutcome(BaseModel): + """One presentation order's candidate-oriented pairwise result.""" + + result: Literal['win', 'tie', 'loss'] + strength: Literal['weak', 'strong'] = 'weak' + + +class PairwiseOutcome(BaseModel): + """A candidate-oriented pairwise result, independent of presentation order. + + ``placements`` retains the separately aggregated official games. ``result`` is the semantic + vote used for the candidate-oriented summary and is never reconstructed from raw labels. + """ + + metric_name: str + result: Literal['win', 'tie', 'loss'] + strength: Literal['weak', 'strong'] = 'weak' + placements: Dict[str, PairwisePlacementOutcome] = Field(default_factory=dict) + + @property + def score(self) -> float: + if self.result == 'tie': + return 0.5 + if self.result == 'win': + return 1.0 if self.strength == 'strong' else 0.75 + return 0.0 if self.strength == 'strong' else 0.25 + + +class ReducedVerdict(BaseModel): + """One observation's worth of verdicts folded into per-metric values by the adapter.""" + + value: Dict[str, float] = Field(default_factory=dict) + outcome: Optional[PairwiseOutcome] = None + """Optional typed non-numeric outcome used by a benchmark's finalizer. + + This is deliberately separate from ``metadata``: it participates in the same repeat and + cross-judge aggregation as ``value``. Pairwise benchmarks use it to derive battle records + without letting a first observation's diagnostic metadata decide the final score. + """ + metadata: Dict[str, Any] = Field(default_factory=dict) + position_results: Dict[str, Literal['win', 'tie', 'loss']] = Field(default_factory=dict) + """Placement-level results used only for position-bias statistics. + + Numeric pairwise benchmarks such as AIR-Bench keep their official scalar reducer instead of + using :class:`PairwiseOutcome`, but still expose this typed signal for swap consistency. + """ + + +class JudgeObservation(BaseModel): + """One (judge, repeat) pass over all of a sample's cases.""" + + judge_id: str + repeat_id: int = 0 + status: ScoreStatus = ScoreStatus.SUCCESS + case_verdicts: List[CaseVerdict] = Field(default_factory=list) + reduced: Optional[ReducedVerdict] = None + error: Optional[str] = None + + @property + def is_valid(self) -> bool: + return self.status is ScoreStatus.SUCCESS and self.reduced is not None and bool(self.reduced.value) + + @property + def is_fallback(self) -> bool: + return self.status is ScoreStatus.FALLBACK and self.reduced is not None and bool(self.reduced.value) + + +class JudgeReview(BaseModel): + """Everything the judge produced for one sample.""" + + status: ScoreStatus = ScoreStatus.SUCCESS + observations: List[JudgeObservation] = Field(default_factory=list) + attempts: List[JudgeAttempt] = Field(default_factory=list) + value: Dict[str, float] = Field(default_factory=dict) + """Aggregated per-metric values; empty when no observation was usable.""" + + metadata: Dict[str, Any] = Field(default_factory=dict) + """Display metadata from the primary judge's first valid observation only.""" + + observation_metadata: List[Dict[str, Any]] = Field(default_factory=list) + """Per-observation reducer metadata retained for audit, never for scoring.""" + + failure_counts: Dict[str, int] = Field(default_factory=dict) + """Attempt counts keyed by :class:`ScoreStatus` value.""" + + error: Optional[str] = None + + outcome: Optional[PairwiseOutcome] = None + """Aggregated typed benchmark outcome, if the adapter declared one.""" + + disagreement: Dict[str, Any] = Field(default_factory=dict) + """Per-sample repeat, cross-judge, and position-consistency statistics.""" + + @property + def valid_observations(self) -> List[JudgeObservation]: + return [obs for obs in self.observations if obs.is_valid] + + @property + def fallback_observations(self) -> List[JudgeObservation]: + """Rule-derived observations, retained for official fallbacks but never Judge quorum.""" + return [obs for obs in self.observations if obs.is_fallback] + + @property + def usable_observations(self) -> List[JudgeObservation]: + """All observations with a reducer output, including official rule fallbacks.""" + return self.valid_observations + self.fallback_observations + + +__all__: Sequence[str] = ( + 'CaseVerdict', + 'JudgeAttempt', + 'JudgeCase', + 'JudgeContext', + 'JudgeObservation', + 'JudgeRequest', + 'JudgeReview', + 'Placement', + 'PairwiseOutcome', + 'PairwisePlacementOutcome', + 'ReducedVerdict', +) diff --git a/evalscope/api/messages/chat_message.py b/evalscope/api/messages/chat_message.py index 2ce2412c7..6eb8e9a0b 100644 --- a/evalscope/api/messages/chat_message.py +++ b/evalscope/api/messages/chat_message.py @@ -13,7 +13,7 @@ class ChatMessageBase(BaseModel): """Base class for chat messages.""" id: Optional[str] = Field(default=None) - """Unique identifer for message.""" + """Unique identifier for message.""" content: Union[str, List[Content]] """Content (simple string or list of content objects)""" diff --git a/evalscope/api/metric/__init__.py b/evalscope/api/metric/__init__.py index 13315601b..2d223d1de 100644 --- a/evalscope/api/metric/__init__.py +++ b/evalscope/api/metric/__init__.py @@ -1,5 +1,5 @@ from .metric import Metric, SingletonMetric, T2IMetric -from .scorer import Aggregator, AggScore, SampleScore, Score, Value +from .scorer import Aggregator, AggScore, JudgeSummary, SampleScore, Score, Value from .semantics import ( MetricDirection, MetricDisplayKind, diff --git a/evalscope/api/metric/scorer.py b/evalscope/api/metric/scorer.py index a3e82f274..216e1a5ef 100644 --- a/evalscope/api/metric/scorer.py +++ b/evalscope/api/metric/scorer.py @@ -3,10 +3,45 @@ from typing import Any, Callable, Dict, List, Optional, Union from evalscope.api.metric.semantics import MetricIdentity +from evalscope.constants import ScoreStatus Value = Dict[str, Union[int, float, bool]] +class JudgeSummary(BaseModel): + """First-class summary of a judge session, for reports and offline inspection.""" + + status: ScoreStatus = Field(default=ScoreStatus.SUCCESS) + """Whether scoring completed, degraded, or was unavailable.""" + + scored: int = Field(default=0) + """Samples with a usable score in this summary's scope (one for a sample summary).""" + + total: int = Field(default=1) + """Samples considered in this summary's scope.""" + + coverage: float = Field(default=0.0) + """``scored / total``; unavailable samples are reported as zero coverage, not score zero.""" + + judge_models: List[str] = Field(default_factory=list) + """Judge model ids that produced this score.""" + + valid_observations: int = Field(default=0) + """Observations that yielded a usable verdict.""" + + total_observations: int = Field(default=0) + """Observations attempted, including invalid ones.""" + + failures: Dict[str, int] = Field(default_factory=dict) + """Failure counts keyed by :class:`ScoreStatus` value.""" + + disagreement: Dict[str, Any] = Field(default_factory=dict) + """Typed disagreement statistics; distinct from execution degradation.""" + + error: Optional[str] = Field(default=None) + """Human-readable reason the score is unavailable, if any.""" + + class Score(BaseModel): """Score generated by a scorer.""" @@ -14,6 +49,16 @@ class Score(BaseModel): """Score value as a dictionary. Key is the score name, value is the score value. The first key is considered the main score by default.""" + status: ScoreStatus = Field(default=ScoreStatus.SUCCESS) + """Whether this score is usable. A non-usable status means the affected metric keys are + omitted from :attr:`value` and the sample drops out of aggregation instead of scoring 0.""" + + judge_summary: Optional[JudgeSummary] = Field( + default=None, + validation_alias=AliasChoices('judge_summary', 'judge_detail'), + ) + """Judge execution summary, populated only when an LLM judge produced this score.""" + extracted_prediction: Optional[str] = Field(default=None) """Answer extracted from model output (optional)""" @@ -33,6 +78,15 @@ class Score(BaseModel): report-level declaration; this field does not assign report metric roles. """ + @property + def judge_detail(self) -> Optional[JudgeSummary]: + """Deprecated Python compatibility alias for :attr:`judge_summary`.""" + return self.judge_summary + + @judge_detail.setter + def judge_detail(self, value: Optional[JudgeSummary]) -> None: + self.judge_summary = value + @property def main_value(self) -> Union[int, float, bool]: """Main score value.""" @@ -74,6 +128,9 @@ class SampleScore(BaseModel): group_id: Optional[Union[str, int]] = Field(default=None) """A group id for the sample, used for grouping k repeated samples.""" + generation_index: Optional[int] = Field(default=None) + """Planned zero-based generation position inside ``group_id``; never compact missing attempts.""" + sample_metadata: Optional[Dict[str, Any]] = Field(default=None) """Metadata from the sample""" diff --git a/evalscope/api/metric/semantics.py b/evalscope/api/metric/semantics.py index 565c2898d..b773394b4 100644 --- a/evalscope/api/metric/semantics.py +++ b/evalscope/api/metric/semantics.py @@ -224,6 +224,9 @@ class MetricSemantics(BaseModel): metric_name: str """Display name of the metric. May differ from the final report metric name.""" + display_name: Optional[str] = Field(default=None) + """Optional display override for a declared diagnostic metric label.""" + kind: MetricKind """Intrinsic classification. Report-level primary selection lives on ``Report``.""" @@ -277,6 +280,12 @@ def _check_kind_direction_display(self) -> Self: f"kind='diagnostic' requires direction='none', got '{self.direction.value}'" ) + if self.display_name is not None and self.kind is not MetricKind.DIAGNOSTIC: + raise ValueError( + f"semantic_id='{self.semantic_id}', metric_name='{self.metric_name}': " + 'display_name is only valid for diagnostic metrics' + ) + # Percent display needs an explicit range and multiplier. if self.display_kind == MetricDisplayKind.PERCENT: missing = [ diff --git a/evalscope/api/mixin/llm_judge_mixin.py b/evalscope/api/mixin/llm_judge_mixin.py index cd5a6200f..62bcdbf95 100644 --- a/evalscope/api/mixin/llm_judge_mixin.py +++ b/evalscope/api/mixin/llm_judge_mixin.py @@ -1,181 +1,342 @@ -from typing import TYPE_CHECKING, Any, Dict, List, Optional +import threading +from functools import lru_cache +from pydantic import BaseModel, Field, create_model +from typing import TYPE_CHECKING, Any, List, Literal, Optional, Tuple, Type from evalscope.api.evaluator import TaskState from evalscope.api.metric import Score -from evalscope.constants import JudgeStrategy +from evalscope.constants import JudgeScoreType, JudgeStrategy, ScoreStatus, ScoringPolicy from evalscope.metrics import LLMJudge from evalscope.utils.argument_utils import get_secret_value +from evalscope.utils.deprecation_utils import deprecated_warning from evalscope.utils.logger import get_logger if TYPE_CHECKING: from evalscope.api.benchmark import BenchmarkMeta - from evalscope.config import TaskConfig + from evalscope.api.judge import JudgeContext, JudgeDefinition, JudgeExecutor + from evalscope.config import JudgeConfig, TaskConfig logger = get_logger() +class _RatingVerdict(BaseModel): + """The ``numeric`` judge contract: a reference-free 0-1 rating.""" + + reasoning: str = '' + score: float = Field(ge=0.0, le=1.0) + + +@lru_cache(maxsize=None) +def _correctness_model(labels: Tuple[str, ...]) -> Type[BaseModel]: + """Build the JSON schema for the generic pattern-judge verdict.""" + return create_model('CorrectnessVerdict', reasoning=(str, ''), verdict=(Literal[labels], ...)) + + class LLMJudgeMixin: """ Mixin class for LLM Judge functionality. """ - llm_judge_default = False - """Whether JudgeStrategy.AUTO should use LLM judge by default.""" + scoring_policy: ScoringPolicy = ScoringPolicy.RULE_DEFAULT + """Declares what this benchmark's rule and judge paths can do. See :class:`ScoringPolicy`.""" + + llm_judge_default: Optional[bool] = None + """[Deprecated] Superseded by :attr:`scoring_policy`. Will be removed in v2.0.0.""" + + def __init_subclass__(cls, **kwargs: Any) -> None: + """Map a legacy ``llm_judge_default`` declaration onto :attr:`scoring_policy`. + + ``True`` maps to ``JUDGE_DEFAULT`` rather than ``JUDGE_ONLY`` so a third-party adapter + keeps rule scoring permitted. + """ + super().__init_subclass__(**kwargs) + legacy = cls.__dict__.get('llm_judge_default') + if legacy is None or 'scoring_policy' in cls.__dict__: + return + deprecated_warning( + logger, f'{cls.__name__} declares `llm_judge_default`, which is deprecated and will be removed in ' + 'v2.0.0. Declare `scoring_policy` instead.' + ) + cls.scoring_policy = ScoringPolicy.JUDGE_DEFAULT if legacy else ScoringPolicy.RULE_DEFAULT def __init__(self, benchmark_meta: 'BenchmarkMeta', task_config: Optional['TaskConfig'] = None): self._benchmark_meta = benchmark_meta self._task_config = task_config - self._llm_judge: Optional[LLMJudge] = None - """LLM judge instance""" + self._llm_judges: Optional[List[LLMJudge]] = None + self._judge_executor: Optional['JudgeExecutor'] = None + self._judge_executor_lock = threading.Lock() super().__init__() + @property + def llm_judges(self) -> List[LLMJudge]: + """Every configured judge model, lazily built.""" + if self._llm_judges is None and self.use_llm_judge: + self._llm_judges = self.init_llm_judges() + return self._llm_judges or [] + @property def llm_judge(self) -> Optional[LLMJudge]: - """Get LLM judge instance with lazy initialization.""" - if self._llm_judge is None and self.use_llm_judge: - self._llm_judge = self.init_llm_judge() - return self._llm_judge + """The primary judge: the first one when several are configured.""" + judges = self.llm_judges + return judges[0] if judges else None @llm_judge.setter def llm_judge(self, value: Optional[LLMJudge]): - """Set LLM judge instance.""" - self._llm_judge = value + """Replace the configured judges with a single one.""" + self._llm_judges = [value] if value is not None else None + # The executor is memoized around its judges, so it must not outlive them. + self._judge_executor = None @property def judge_strategy(self) -> str: """Get the judge strategy from the task configuration.""" - return self._task_config.judge_strategy + return self._judge_config.strategy + + @property + def _judge_config(self) -> 'JudgeConfig': + """Return the evaluation judge configuration, with a clear error for metadata-only adapters.""" + if self._task_config is None: + raise RuntimeError('LLM judge scoring requires a TaskConfig.') + return self._task_config.judge @property def use_llm_judge(self) -> bool: """Check if LLM judge is enabled.""" if self.judge_strategy == JudgeStrategy.RULE: return False - elif self.judge_strategy == JudgeStrategy.LLM: - return True - elif self.judge_strategy == JudgeStrategy.LLM_RECALL: + elif self.judge_strategy in (JudgeStrategy.LLM, JudgeStrategy.LLM_RECALL): return True elif self.judge_strategy == JudgeStrategy.AUTO: - return self.llm_judge_default + return self.scoring_policy.judge_by_default else: logger.warning(f'Unknown judge strategy: {self.judge_strategy}. Defaulting to False.') return False - def init_llm_judge(self) -> Optional[LLMJudge]: + def validate_judge_strategy(self) -> None: + """Reject a judge configuration this benchmark cannot honour, before any model call. + + Without these checks the run fails at scoring time -- after the samples have been + generated -- or silently reports an all-zero result. """ - Initialize the LLM judge for the benchmark. + if self._task_config is None: + return + strategy = self.judge_strategy + name = self._benchmark_meta.name + if strategy in (JudgeStrategy.RULE, JudgeStrategy.LLM_RECALL) and not self.scoring_policy.rule_supported: + raise ValueError( + f"Benchmark '{name}' has no usable rule-based scoring, so judge_strategy='{strategy}' " + 'cannot produce a meaningful score. ' + "Use judge.strategy='auto' or 'llm' with judge.models." + ) + if self.use_llm_judge and not self._judge_config.models: + raise ValueError( + f"Benchmark '{name}' scores with an LLM judge under judge_strategy='{strategy}', " + 'so judge.models must be provided.' + ) + + def _judge_specs(self) -> List[dict]: + """Return typed judge model specifications as model-construction dictionaries.""" + config = self._judge_config + contract = config.contract.model_dump(exclude_none=True) + return [{ + **spec.model_dump(exclude={'judge_id'}, exclude_none=True), + **contract, + } for spec in config.models] + + def init_llm_judges(self) -> List[LLMJudge]: + """Build every configured judge model. Returns: - Optional[LLMJudge]: The initialized LLM judge instance or None + List[LLMJudge]: One judge per entry in ``judge.models``, empty for rule scoring. """ - if self.judge_strategy == JudgeStrategy.RULE: - return None - else: - if not self._task_config.judge_model_args: - raise ValueError( - 'LLM judge model arguments must be provided for LLM-based judge strategies. ' - 'Please check your task configuration.' + return [] + config = self._judge_config + if not config.models: + raise ValueError( + 'LLM judge model arguments must be provided for LLM-based judge strategies. ' + 'Please check your task configuration.' + ) + judges = [LLMJudge(**get_secret_value(spec)) for spec in self._judge_specs()] + for judge, spec in zip(judges, config.models): + judge.judge_id = spec.judge_id + return judges + + # ------------------------------------------------------------------ + # Declarative judge path (evalscope.api.judge) + # ------------------------------------------------------------------ + + supports_position_swap: bool = False + """Whether this benchmark has a meaningful pairwise placement dimension.""" + + official_position_swap: bool = False + """Whether the benchmark's official protocol judges both placements by default.""" + + judge_metric_name: str = 'acc' + """Metric key used by the generic single-verdict judge definition.""" + + @property + def judge_executor(self) -> 'JudgeExecutor': + """Lazily build the one executor that owns every judge call for this benchmark.""" + from evalscope.api.judge import JudgeExecutor, JudgeExecutorConfig + + if self._judge_executor is None: + with self._judge_executor_lock: + if self._judge_executor is None: + self._judge_executor = JudgeExecutor( + self.llm_judges, + JudgeExecutorConfig( + repeats=self._judge_config.repeats, + position_swap=self._resolved_position_swap(), + aggregation=self._judge_config.aggregation, + min_valid_judges=self._judge_config.min_valid_judges, + ), + ) + return self._judge_executor + + def _resolved_position_swap(self) -> bool: + """Resolve the user override against the benchmark's official position policy.""" + configured = self._judge_config.position_swap + if configured == 'auto': + return bool(self.official_position_swap) + if not self.supports_position_swap: + if configured == 'on': + logger.warning( + f"Benchmark '{self._benchmark_meta.name}' has no pairwise position dimension; " + "ignoring judge.position_swap='on'." ) - judge_model_args = get_secret_value(self._task_config.judge_model_args) - return LLMJudge(**judge_model_args) + return False + return configured == 'on' - def maybe_llm_match_score( + def score_with_judge_contracts( self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState, - rule_based_score: Optional[Score] = None, ) -> Score: - """ - Compute the match score between the original and filtered predictions against the reference. - - Args: - original_prediction: The original prediction output from the model. - filtered_prediction: The filtered prediction output from the model. - reference: The ground truth reference output. - task_state: The current task state. - original_score: Optional original score to be used for comparison. + """Run this sample through the executor. Adapters do not override this.""" + from evalscope.api.judge import JudgeContext - Returns: - Score: The computed match score. - """ - # If LLM judge is not used, return the rule-based score directly - if not self.use_llm_judge: - return rule_based_score - - # For LLM_RECALL, if rule-based score is already perfect, skip LLM judge - if float(rule_based_score.main_value) > 0.99: - return rule_based_score - - # Compute LLM judge score - llm_score = self.llm_match_score( + context = JudgeContext( + task_state=task_state, original_prediction=original_prediction, filtered_prediction=filtered_prediction, reference=reference, - task_state=task_state, + ) + definition = self.judge_definition(context) + if definition.immediate_score is not None: + return definition.immediate_score.model_copy( + update={ + 'metadata': { + **(definition.immediate_score.metadata or {}), + 'judge_skipped': True, + 'judge_skip_reason': definition.skip_reason, + } + } + ) + executor = self.judge_executor + review = executor.execute(definition, context) + return executor.build_score( + definition, + review, + context, + metadata={ + 'source': 'llm_judge', + 'judge_strategy': self.judge_strategy, + 'model': self._judge_id(), + 'non_official_position_swap': self._task_config is not None + and self._resolved_position_swap() != self.official_position_swap, + **review.metadata, + }, ) - # For LLM RECALL, merge the scores - return self._merge_scores(rule_based_score, llm_score) - - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - """Compute the LLM match score. - - Args: - original_prediction (str): The original prediction output from the model. - filtered_prediction (str): The filtered prediction output from the model. - reference (str): The ground truth reference output. - task_state (TaskState): The current task state. + def judge_definition(self, context: 'JudgeContext') -> 'JudgeDefinition': + """Declare this sample's judge review. - Returns: - Score: The computed match score. + The generic single-verdict contract supports both numeric and pattern judges. Adapters + override it only for benchmark-owned labels, metrics, or multi-case workflows. """ - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - ) + from evalscope.api.judge import JudgeDefinition - question = task_state.input_text - - # Request judge and obtain score - prompt = self.llm_judge.build_prompt(pred=original_prediction, gold=reference, question=question) - judge_response = self.llm_judge.judge(prompt) - judge_score = self.llm_judge.get_score(judge_response) + judge = self._primary_judge() + prompt = judge.build_prompt( + pred=context.original_prediction, + gold=context.reference, + question=context.task_state.input_text, + ) + if judge.score_type == JudgeScoreType.NUMERIC: + return JudgeDefinition.numeric( + prompt=prompt, + schema_model=_RatingVerdict, + metric_name=self.judge_metric_name, + system_prompt=judge.system_prompt, + ) + return JudgeDefinition.labels( + prompt=prompt, + schema_model=_correctness_model(tuple(sorted(judge.score_mapping))), + scores={ + label: { + self.judge_metric_name: score + } + for label, score in judge.score_mapping.items() + }, + system_prompt=judge.system_prompt, + main_score_name=self.judge_metric_name, + ) - score.value = {'acc': judge_score} - score.explanation = f'LLM judge: {judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id + def _primary_judge(self) -> LLMJudge: + """Return the first configured judge after lazy construction.""" + judge = self.llm_judge + if judge is None: + raise RuntimeError('LLM judge scoring requires at least one configured judge model.') + return judge + + def _judge_id(self) -> str: + """Return the primary judge identifier stored in a sample score.""" + judge = self._primary_judge() + return str(getattr(judge, 'judge_id', None) or judge.model_id) + + def fallback_to_rule_score(self, rule_based_score: Score, judge_score: Score) -> Score: + """Retain rule evidence when a ``JUDGE_DEFAULT`` review is unavailable.""" + fallback = rule_based_score.model_copy(deep=True) + fallback.status = ScoreStatus.DEGRADED + fallback.judge_summary = judge_score.judge_summary.model_copy( + update={'status': ScoreStatus.DEGRADED} + ) if judge_score.judge_summary is not None else None + fallback.metadata = { + **(fallback.metadata or {}), + **(judge_score.metadata or {}), + 'judge_unavailable': judge_score.status.value, } - - return score + return fallback def _merge_scores(self, rule_based_score: Score, llm_score: Score) -> Score: - """ - Merge rule-based score with LLM judge score for LLM_RECALL strategy. - - Args: - rule_based_score: The original rule-based score - llm_score: The LLM judge score + """Merge the rule score with the judge score for the LLM_RECALL strategy. - Returns: - Score: The merged score + ``llm_recall`` exists to recover rule-based misses, so the judge can only raise the + score: the result is ``max(rule, judge)``. A failed judge must not erase rule evidence. """ - # Update the main value with LLM judge result - rule_based_score.main_value = llm_score.main_value + if not llm_score.status.is_usable or not llm_score.value: + # The rule score stands, so the sample is still scored -- it just fell back. + rule_based_score.status = ScoreStatus.FALLBACK + rule_based_score.judge_summary = llm_score.judge_summary.model_copy( + update={'status': ScoreStatus.FALLBACK} + ) if llm_score.judge_summary is not None else None + rule_based_score.metadata = { + **(rule_based_score.metadata or {}), + 'judge_unavailable': llm_score.status.value, + } + return rule_based_score + + rule_value = float(rule_based_score.main_value or 0.0) + judge_value = float(llm_score.main_value or 0.0) + rule_based_score.main_value = max(rule_value, judge_value) rule_based_score.explanation = llm_score.explanation - rule_based_score.metadata = llm_score.metadata + rule_based_score.metadata = {**(rule_based_score.metadata or {}), **(llm_score.metadata or {})} + rule_based_score.status = llm_score.status + rule_based_score.judge_summary = llm_score.judge_summary return rule_based_score diff --git a/evalscope/api/registry.py b/evalscope/api/registry.py index a22e50f3c..ccac935ea 100644 --- a/evalscope/api/registry.py +++ b/evalscope/api/registry.py @@ -106,8 +106,6 @@ def get_benchmark(name: str, config: Optional['TaskConfig'] = None) -> 'DataAdap Args: name (str): The name of the benchmark. config (Optional['TaskConfig']): The task configuration. - dataset_args (Optional[dict]): The dataset-specific arguments. - """ # copy to avoid modifying the original metadata metadata = copy.deepcopy(BENCHMARK_REGISTRY.get(name)) @@ -117,7 +115,7 @@ def get_benchmark(name: str, config: Optional['TaskConfig'] = None) -> 'DataAdap # Update metadata with dataset-specific configuration if config is not None: metadata._update(config.dataset_args.get(name, {})) - # Return the data adapter initialized with the benchmark metadata + # Construction must be side-effect free: callers also use this helper for metadata and docs. data_adapter_cls = metadata.data_adapter return data_adapter_cls(benchmark_meta=metadata, task_config=config) diff --git a/evalscope/api/tool/tool_info.py b/evalscope/api/tool/tool_info.py index 7ad5701e2..9202d5c5b 100644 --- a/evalscope/api/tool/tool_info.py +++ b/evalscope/api/tool/tool_info.py @@ -118,7 +118,7 @@ def parse_tool_info(func: Callable[..., Any]) -> ToolInfo: if param_name in type_hints: tool_param = json_schema(type_hints[param_name]) # as a fallback try to parse it from the docstring - # (this is minimally necessary for backwards compatiblity + # (this is minimally necessary for backwards compatibility # with tools gen1 type parsing, which only used docstrings) elif 'docstring_type' in docstring_info: json_type = python_type_to_json_type(docstring_info['docstring_type']) diff --git a/evalscope/api/tool/utils.py b/evalscope/api/tool/utils.py index d538381b8..b13145813 100644 --- a/evalscope/api/tool/utils.py +++ b/evalscope/api/tool/utils.py @@ -13,8 +13,8 @@ def parse_tool_call(id: str, function: str, arguments: str, tools: Optional[List """Parse a tool call from a JSON payload. Note that this function doesn't know about internal tool names so the caller - should ammend the returned `ToolCall` by mapping the parsed `function` field from - an internal name to an tool name and fixing up the `ToolCall` object + should amend the returned `ToolCall` by mapping the parsed `function` field from + an internal name to a tool name and fixing up the `ToolCall` object as required to reflect this change. """ error: Optional[str] = None diff --git a/evalscope/arguments.py b/evalscope/arguments.py index db1bd96ea..ba8158ddc 100644 --- a/evalscope/arguments.py +++ b/evalscope/arguments.py @@ -3,7 +3,7 @@ import ast import json -from evalscope.constants import EvalBackend, EvalType, JudgeStrategy, ModelTask +from evalscope.constants import EvalBackend, EvalType, ModelTask class ParseStrArgsAction(argparse.Action): @@ -98,10 +98,10 @@ def add_argument(parser: argparse.ArgumentParser): parser.add_argument('--timeout', type=float, default=None, help='[Deprecated] Use --generation-config timeout=... instead. Will be removed in v2.0.0.') # noqa: E501 parser.add_argument('--stream', action='store_true', default=None, help='[Deprecated] Use --generation-config stream=True instead. Will be removed in v2.0.0.') # noqa: E501 - # LLMJudge arguments - parser.add_argument('--judge-strategy', type=str, default=JudgeStrategy.AUTO, help='The judge strategy.') - parser.add_argument('--judge-model-args', type=json.loads, default='{}', help='The judge model args, should be a json string.') # noqa: E501 - parser.add_argument('--judge-worker-num', type=int, default=1, help='The number of workers for the judge model.') + # Native judge arguments + parser.add_argument('--judge', type=json.loads, default=None, help='The typed judge configuration as a JSON object.') + parser.add_argument('--judge-strategy', type=str, default=None, help='[Deprecated] Use --judge JSON instead.') + parser.add_argument('--judge-model-args', type=json.loads, default=None, help='[Deprecated] Use --judge JSON instead.') parser.add_argument('--analysis-report', action='store_true', default=False, help='Generate analysis report for the evaluation results using judge model.') # noqa: E501 parser.add_argument('--collect-perf', action=argparse.BooleanOptionalAction, default=True, help='Collect per-request performance metrics (latency, TTFT, token usage) during evaluation. TTFT requires streaming (--generation-config stream=True). Use --no-collect-perf to disable.') # noqa: E501 diff --git a/evalscope/benchmarks/_meta/air_bench_chat.json b/evalscope/benchmarks/_meta/air_bench_chat.json index e74036f0c..49bdf9d02 100644 --- a/evalscope/benchmarks/_meta/air_bench_chat.json +++ b/evalscope/benchmarks/_meta/air_bench_chat.json @@ -12,11 +12,6 @@ "judge_score", "win_rate" ], - "primary_metric": { - "name": "judge_score", - "aggregation": null, - "dimensions": {} - }, "few_shot_num": 0, "eval_split": "test", "train_split": "", @@ -30,7 +25,7 @@ "speech_and_sound_QA", "speech_and_music_QA" ], - "description": "\n## Overview\n\nAIR-Bench Chat is the generative half of [AIR-Bench](https://arxiv.org/abs/2402.07729) (Audio InstRuction Benchmark, ACL 2024 main conference) — the first instruction-following benchmark for large audio-language models (LALMs), covering **human speech, natural sounds and music**. It contains roughly 2k open-ended audio QA pairs covering speech, sound, music and mixed-audio scenes; responses are graded by a GPT-4 judge against a reference answer.\n\n## Task Description\n\n- **Task Type**: Open-ended audio question answering.\n- **Input**: An audio clip plus a free-form question.\n- **Output**: A textual answer evaluated against the reference response.\n\n## Categories (8 tasks → 5 reported categories)\n\nThe 8 Chat tasks are aggregated by the official `cal_score.py` into five categories:\n\n- `speech`: `speech_QA`, `speech_dialogue_QA`\n- `sound`: `sound_QA`, `sound_generation_QA`\n- `music`: `music_QA`, `music_generation_analysis_QA`\n- `speech_and_sound`: `speech_and_sound_QA`\n- `speech_and_music`: `speech_and_music_QA`\n\nThe paper's **Mixed-audio = mean(speech_and_sound, speech_and_music)**.\n\n## Dataset Access\n\n- The dataset is hosted on ModelScope: [`evalscope/AIR-Bench-Dataset`](https://modelscope.cn/datasets/evalscope/AIR-Bench-Dataset). It uses an *audiofolder + JSON metadata* layout. evalscope downloads it lazily via `modelscope.dataset_snapshot_download` on first run; the full release is ~49 GB, so it is recommended to limit which tasks are pulled via `extra_params`.\n- If the dataset is already on disk, pass `dataset_args={'air_bench_chat': {'local_path': '/path/to/AIR-Bench-Dataset'}}`; the local root should contain `Chat/`.\n\n## Evaluation Protocol\n\n- The judge LLM (default: GPT-4) receives the question, the textual audio description (`meta_info` from the dataset), the reference answer (`answer_gt`), and the model's response. It outputs a single line with two integer scores in `[1, 10]`.\n- To remove position bias, every sample is judged twice with the order of reference and prediction swapped, then averaged. This mirrors `cal_score.py` in the official repository — disable it via `extra_params={'do_swap': False}` to halve judge cost.\n- Reported metric `judge_score` is the model's mean judge score; `win_rate` records how often the model strictly beats the reference.\n\n```{warning}\nThe official leaderboard uses `gpt-4-0125-preview` as the judge model. If that exact snapshot is unavailable, use an available GPT-4-class judge; absolute scores can drift versus the published numbers because the judge model changed.\n```\n\n## Implementation Notes\n\n- The judge model is selected via `--judge-model-args`; ensure the model id supports long contexts (`meta_info` may exceed 4k tokens for dialogue tasks).\n- Set `extra_params={'tasks': [...]}` to evaluate only specific Chat task names — useful for partial runs.\n", + "description": "\n## Overview\n\nAIR-Bench Chat is the generative half of [AIR-Bench](https://arxiv.org/abs/2402.07729) (Audio InstRuction Benchmark, ACL 2024 main conference) — the first instruction-following benchmark for large audio-language models (LALMs), covering **human speech, natural sounds and music**. It contains roughly 2k open-ended audio QA pairs covering speech, sound, music and mixed-audio scenes; responses are graded by a GPT-4 judge against a reference answer.\n\n## Task Description\n\n- **Task Type**: Open-ended audio question answering.\n- **Input**: An audio clip plus a free-form question.\n- **Output**: A textual answer evaluated against the reference response.\n- **Modalities**: Audio (human speech, natural sounds, music) + text.\n\n## Key Features\n\n- ~2k open-ended audio QA pairs across speech, sound, music and mixed-audio scenes; the generative half of AIR-Bench (ACL 2024).\n- 8 Chat tasks aggregated by the official `cal_score.py` into 5 reported categories: `speech` (`speech_QA`, `speech_dialogue_QA`), `sound` (`sound_QA`, `sound_generation_QA`), `music` (`music_QA`, `music_generation_analysis_QA`), `speech_and_sound` (`speech_and_sound_QA`), `speech_and_music` (`speech_and_music_QA`). The paper's Mixed-audio = mean(speech_and_sound, speech_and_music).\n- Position bias is removed by judging every sample twice with reference/prediction order swapped, then averaging (disable via `extra_params={'do_swap': False}` to halve judge cost).\n- Hosted on ModelScope ([`evalscope/AIR-Bench-Dataset`](https://modelscope.cn/datasets/evalscope/AIR-Bench-Dataset)) in an audiofolder + JSON layout; the full release is ~49 GB, so limit tasks via `extra_params={'tasks': [...]}` for partial runs.\n\n## Evaluation Notes\n\n- Metrics: `judge_score` is the model's mean judge score; `win_rate` records how often the model strictly beats the reference.\n- The judge LLM receives the question, the textual audio description (`meta_info`), the reference answer (`answer_gt`), and the model's response, and outputs two integer scores in `[1, 10]`. Use a judge that supports long contexts, since `meta_info` may exceed 4k tokens for dialogue tasks.\n- The official leaderboard uses `gpt-4-0125-preview`. If that exact snapshot is unavailable, use an available GPT-4-class judge; absolute scores can drift versus the published numbers because the judge model changed.\n- If the dataset is already on disk, pass `dataset_args={'air_bench_chat': {'local_path': '/path/to/AIR-Bench-Dataset'}}`; the local root should contain `Chat/`.\n", "prompt_template": "{question}", "system_prompt": "", "few_shot_prompt_template": "", @@ -48,7 +43,12 @@ } }, "sandbox_config": {}, - "category": "vlm" + "category": "vlm", + "primary_metric": { + "name": "judge_score", + "aggregation": null, + "dimensions": {} + } }, "statistics": { "total_samples": 2200, @@ -332,11 +332,11 @@ "truncated": false }, "readme": { - "en": "# AIR-Bench-Chat\n\n\n## Overview\n\nAIR-Bench Chat is the generative half of [AIR-Bench](https://arxiv.org/abs/2402.07729) (Audio InstRuction Benchmark, ACL 2024 main conference) — the first instruction-following benchmark for large audio-language models (LALMs), covering **human speech, natural sounds and music**. It contains roughly 2k open-ended audio QA pairs covering speech, sound, music and mixed-audio scenes; responses are graded by a GPT-4 judge against a reference answer.\n\n## Task Description\n\n- **Task Type**: Open-ended audio question answering.\n- **Input**: An audio clip plus a free-form question.\n- **Output**: A textual answer evaluated against the reference response.\n\n## Categories (8 tasks → 5 reported categories)\n\nThe 8 Chat tasks are aggregated by the official `cal_score.py` into five categories:\n\n- `speech`: `speech_QA`, `speech_dialogue_QA`\n- `sound`: `sound_QA`, `sound_generation_QA`\n- `music`: `music_QA`, `music_generation_analysis_QA`\n- `speech_and_sound`: `speech_and_sound_QA`\n- `speech_and_music`: `speech_and_music_QA`\n\nThe paper's **Mixed-audio = mean(speech_and_sound, speech_and_music)**.\n\n## Dataset Access\n\n- The dataset is hosted on ModelScope: [`evalscope/AIR-Bench-Dataset`](https://modelscope.cn/datasets/evalscope/AIR-Bench-Dataset). It uses an *audiofolder + JSON metadata* layout. evalscope downloads it lazily via `modelscope.dataset_snapshot_download` on first run; the full release is ~49 GB, so it is recommended to limit which tasks are pulled via `extra_params`.\n- If the dataset is already on disk, pass `dataset_args={'air_bench_chat': {'local_path': '/path/to/AIR-Bench-Dataset'}}`; the local root should contain `Chat/`.\n\n## Evaluation Protocol\n\n- The judge LLM (default: GPT-4) receives the question, the textual audio description (`meta_info` from the dataset), the reference answer (`answer_gt`), and the model's response. It outputs a single line with two integer scores in `[1, 10]`.\n- To remove position bias, every sample is judged twice with the order of reference and prediction swapped, then averaged. This mirrors `cal_score.py` in the official repository — disable it via `extra_params={'do_swap': False}` to halve judge cost.\n- Reported metric `judge_score` is the model's mean judge score; `win_rate` records how often the model strictly beats the reference.\n\n```{warning}\nThe official leaderboard uses `gpt-4-0125-preview` as the judge model. If that exact snapshot is unavailable, use an available GPT-4-class judge; absolute scores can drift versus the published numbers because the judge model changed.\n```\n\n## Implementation Notes\n\n- The judge model is selected via `--judge-model-args`; ensure the model id supports long contexts (`meta_info` may exceed 4k tokens for dialogue tasks).\n- Set `extra_params={'tasks': [...]}` to evaluate only specific Chat task names — useful for partial runs.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `air_bench_chat` |\n| **Dataset ID** | [evalscope/AIR-Bench](https://modelscope.cn/datasets/evalscope/AIR-Bench/summary) |\n| **Paper** | [Paper](https://aclanthology.org/2024.acl-long.109/) |\n| **Tags** | `Audio`, `InstructionFollowing`, `QA` |\n| **Metrics** | `judge_score`, `win_rate` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 2,200 |\n| Prompt Length (Mean) | 83.89 chars |\n| Prompt Length (Min/Max) | 17 / 423 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `speech_QA` | 400 | 64.33 | 23 | 148 |\n| `speech_dialogue_QA` | 400 | 77.03 | 29 | 206 |\n| `sound_QA` | 400 | 73.29 | 17 | 166 |\n| `sound_generation_QA` | 100 | 222.52 | 130 | 423 |\n| `music_QA` | 400 | 57.54 | 24 | 202 |\n| `music_generation_analysis_QA` | 100 | 267.52 | 148 | 395 |\n| `speech_and_sound_QA` | 200 | 63.98 | 25 | 127 |\n| `speech_and_music_QA` | 200 | 69.37 | 32 | 127 |\n\n**Audio Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Audio Files | 2,200 |\n| Audio per Sample | min: 1, max: 1, mean: 1 |\n| Formats | mp3, wav |\n\n\n## Sample Example\n\n**Subset**: `speech_QA`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"5781ee73\",\n \"content\": [\n {\n \"audio\": \"/root/.cache/modelscope/hub/datasets/evalscope/AIR-Bench-Dataset/Chat/speech_QA_iemocap/Ses01F_script01_1_M025.wav\",\n \"format\": \"wav\"\n },\n {\n \"text\": \"Who is the speaker addressing at the end of the speech?\"\n }\n ]\n }\n ],\n \"target\": \"The speaker is addressing Mom at the end of the speech.\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"speech_QA\",\n \"metadata\": {\n \"uniq_id\": 400,\n \"task_name\": \"speech_QA\",\n \"dataset_name\": \"iemocap\",\n \"category\": \"speech\",\n \"meta_info\": \"{'emotion': 'neutral', 'gender': 'male', 'transcription': \\\"And then we'll thrash it out with father. Okay Mom? Don't avoid me.\\\"}\",\n \"question\": \"Who is the speaker addressing at the end of the speech?\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `tasks` | `list` | `None` | Optional list of Chat task names to evaluate (subset of ['music_QA', 'music_generation_analysis_QA', 'sound_QA', 'sound_generation_QA', 'speech_QA', 'speech_and_music_QA', 'speech_and_sound_QA', 'speech_dialogue_QA']). Defaults to all tasks. |\n| `do_swap` | `bool` | `True` | When True (default), each sample is judged twice with the order of reference vs. prediction swapped, then scores are averaged. Disable to halve judge cost at the price of position bias. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets air_bench_chat \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['air_bench_chat'],\n dataset_args={\n 'air_bench_chat': {\n # subset_list: ['speech_QA', 'speech_dialogue_QA', 'sound_QA'] # optional, evaluate specific subsets\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# AIR-Bench-Chat\n\n\n## 概述\n\nAIR-Bench Chat 是 [AIR-Bench](https://arxiv.org/abs/2402.07729)（Audio InstRuction Benchmark，ACL 2024 主会）的生成式部分——这是首个面向大音频语言模型（LALMs）的指令遵循基准测试，涵盖**人类语音、自然声音和音乐**。该数据集包含约 2,000 个开放式音频问答对，覆盖语音、声音、音乐及混合音频场景；模型的回答由 GPT-4 作为裁判，对照参考答案进行评分。\n\n## 任务描述\n\n- **任务类型**：开放式音频问答。\n- **输入**：一段音频片段加一个自由形式的问题。\n- **输出**：一段文本答案，将根据参考答案进行评估。\n\n## 类别（8 项任务 → 5 个报告类别）\n\n官方 `cal_score.py` 脚本将 8 项 Chat 任务聚合为以下五个类别：\n\n- `speech`（语音）: `speech_QA`, `speech_dialogue_QA`\n- `sound`（声音）: `sound_QA`, `sound_generation_QA`\n- `music`（音乐）: `music_QA`, `music_generation_analysis_QA`\n- `speech_and_sound`（语音与声音）: `speech_and_sound_QA`\n- `speech_and_music`（语音与音乐）: `speech_and_music_QA`\n\n论文中的 **Mixed-audio（混合音频）= mean(speech_and_sound, speech_and_music)**。\n\n## 数据集获取\n\n- 该数据集托管在 ModelScope 上：[`evalscope/AIR-Bench-Dataset`](https://modelscope.cn/datasets/evalscope/AIR-Bench-Dataset)。采用 *audiofolder + JSON metadata* 的布局方式。evalscope 在首次运行时通过 `modelscope.dataset_snapshot_download` 按需下载；完整数据集约 49 GB，建议通过 `extra_params` 参数限制仅拉取所需任务。\n- 如果数据集已存在于本地磁盘，请传入 `dataset_args={'air_bench_chat': {'local_path': '/path/to/AIR-Bench-Dataset'}}`；本地根目录应包含 `Chat/` 文件夹。\n\n## 评估协议\n\n- 裁判 LLM（默认为 GPT-4）接收问题、音频的文本描述（来自数据集的 `meta_info`）、参考答案（`answer_gt`）以及模型的回答，并输出一行包含两个 `[1, 10]` 范围内的整数分数。\n- 为消除位置偏差，每个样本会被评估两次，分别交换参考答案与模型预测的顺序，然后取平均分。此做法与官方仓库中的 `cal_score.py` 一致；可通过设置 `extra_params={'do_swap': False}` 禁用此行为，以节省一半的裁判成本。\n- 报告指标 `judge_score` 表示模型的平均裁判分数；`win_rate` 记录模型严格优于参考答案的频率。\n\n```{warning}\n官方排行榜使用 `gpt-4-0125-preview` 作为裁判模型。如果该特定快照不可用，请使用其他可用的 GPT-4 级别裁判模型；由于裁判模型变更，绝对分数可能与已发表结果存在偏差。\n```\n\n## 实现说明\n\n- 通过 `--judge-model-args` 指定裁判模型；请确保所选模型支持长上下文（对话类任务的 `meta_info` 可能超过 4k tokens）。\n- 设置 `extra_params={'tasks': [...]}` 可仅评估指定的 Chat 任务名称——适用于部分运行场景。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `air_bench_chat` |\n| **数据集 ID** | [evalscope/AIR-Bench](https://modelscope.cn/datasets/evalscope/AIR-Bench/summary) |\n| **论文** | [Paper](https://aclanthology.org/2024.acl-long.109/) |\n| **标签** | `Audio`, `InstructionFollowing`, `QA` |\n| **指标** | `judge_score`, `win_rate` |\n| **默认示例数** | 0-shot |\n| **评估分割** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 2,200 |\n| 提示词长度（平均） | 83.89 字符 |\n| 提示词长度（最小/最大） | 17 / 423 字符 |\n\n**各子集统计信息：**\n\n| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 |\n|--------|---------|-------------|------------|------------|\n| `speech_QA` | 400 | 64.33 | 23 | 148 |\n| `speech_dialogue_QA` | 400 | 77.03 | 29 | 206 |\n| `sound_QA` | 400 | 73.29 | 17 | 166 |\n| `sound_generation_QA` | 100 | 222.52 | 130 | 423 |\n| `music_QA` | 400 | 57.54 | 24 | 202 |\n| `music_generation_analysis_QA` | 100 | 267.52 | 148 | 395 |\n| `speech_and_sound_QA` | 200 | 63.98 | 25 | 127 |\n| `speech_and_music_QA` | 200 | 69.37 | 32 | 127 |\n\n**音频统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 音频文件总数 | 2,200 |\n| 每样本音频数量 | 最小: 1, 最大: 1, 平均: 1 |\n| 格式 | mp3, wav |\n\n\n## 样例示例\n\n**子集**: `speech_QA`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"5781ee73\",\n \"content\": [\n {\n \"audio\": \"/root/.cache/modelscope/hub/datasets/evalscope/AIR-Bench-Dataset/Chat/speech_QA_iemocap/Ses01F_script01_1_M025.wav\",\n \"format\": \"wav\"\n },\n {\n \"text\": \"Who is the speaker addressing at the end of the speech?\"\n }\n ]\n }\n ],\n \"target\": \"The speaker is addressing Mom at the end of the speech.\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"speech_QA\",\n \"metadata\": {\n \"uniq_id\": 400,\n \"task_name\": \"speech_QA\",\n \"dataset_name\": \"iemocap\",\n \"category\": \"speech\",\n \"meta_info\": \"{'emotion': 'neutral', 'gender': 'male', 'transcription': \\\"And then we'll thrash it out with father. Okay Mom? Don't avoid me.\\\"}\",\n \"question\": \"Who is the speaker addressing at the end of the speech?\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `tasks` | `list` | `None` | 可选的 Chat 任务名称列表（子集包括 ['music_QA', 'music_generation_analysis_QA', 'sound_QA', 'sound_generation_QA', 'speech_QA', 'speech_and_music_QA', 'speech_and_sound_QA', 'speech_dialogue_QA']）。默认评估所有任务。 |\n| `do_swap` | `bool` | `True` | 若为 True（默认），每个样本会评判两次（交换参考答案与模型预测的顺序），然后取平均分。禁用可节省一半裁判成本，但会引入位置偏差。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets air_bench_chat \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['air_bench_chat'],\n dataset_args={\n 'air_bench_chat': {\n # subset_list: ['speech_QA', 'speech_dialogue_QA', 'sound_QA'] # 可选，仅评估指定子集\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "5d0c0e598d9cba020f532aaedbab4752", + "en": "# AIR-Bench-Chat\n\n\n## Overview\n\nAIR-Bench Chat is the generative half of [AIR-Bench](https://arxiv.org/abs/2402.07729) (Audio InstRuction Benchmark, ACL 2024 main conference) — the first instruction-following benchmark for large audio-language models (LALMs), covering **human speech, natural sounds and music**. It contains roughly 2k open-ended audio QA pairs covering speech, sound, music and mixed-audio scenes; responses are graded by a GPT-4 judge against a reference answer.\n\n## Task Description\n\n- **Task Type**: Open-ended audio question answering.\n- **Input**: An audio clip plus a free-form question.\n- **Output**: A textual answer evaluated against the reference response.\n- **Modalities**: Audio (human speech, natural sounds, music) + text.\n\n## Key Features\n\n- ~2k open-ended audio QA pairs across speech, sound, music and mixed-audio scenes; the generative half of AIR-Bench (ACL 2024).\n- 8 Chat tasks aggregated by the official `cal_score.py` into 5 reported categories: `speech` (`speech_QA`, `speech_dialogue_QA`), `sound` (`sound_QA`, `sound_generation_QA`), `music` (`music_QA`, `music_generation_analysis_QA`), `speech_and_sound` (`speech_and_sound_QA`), `speech_and_music` (`speech_and_music_QA`). The paper's Mixed-audio = mean(speech_and_sound, speech_and_music).\n- Position bias is removed by judging every sample twice with reference/prediction order swapped, then averaging (disable via `extra_params={'do_swap': False}` to halve judge cost).\n- Hosted on ModelScope ([`evalscope/AIR-Bench-Dataset`](https://modelscope.cn/datasets/evalscope/AIR-Bench-Dataset)) in an audiofolder + JSON layout; the full release is ~49 GB, so limit tasks via `extra_params={'tasks': [...]}` for partial runs.\n\n## Evaluation Notes\n\n- Metrics: `judge_score` is the model's mean judge score; `win_rate` records how often the model strictly beats the reference.\n- The judge LLM receives the question, the textual audio description (`meta_info`), the reference answer (`answer_gt`), and the model's response, and outputs two integer scores in `[1, 10]`. Use a judge that supports long contexts, since `meta_info` may exceed 4k tokens for dialogue tasks.\n- The official leaderboard uses `gpt-4-0125-preview`. If that exact snapshot is unavailable, use an available GPT-4-class judge; absolute scores can drift versus the published numbers because the judge model changed.\n- If the dataset is already on disk, pass `dataset_args={'air_bench_chat': {'local_path': '/path/to/AIR-Bench-Dataset'}}`; the local root should contain `Chat/`.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `air_bench_chat` |\n| **Dataset ID** | [evalscope/AIR-Bench](https://modelscope.cn/datasets/evalscope/AIR-Bench/summary) |\n| **Paper** | [Paper](https://aclanthology.org/2024.acl-long.109/) |\n| **Tags** | `Audio`, `InstructionFollowing`, `QA` |\n| **Metrics** | `judge_score`, `win_rate` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 2,200 |\n| Prompt Length (Mean) | 83.89 chars |\n| Prompt Length (Min/Max) | 17 / 423 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `speech_QA` | 400 | 64.33 | 23 | 148 |\n| `speech_dialogue_QA` | 400 | 77.03 | 29 | 206 |\n| `sound_QA` | 400 | 73.29 | 17 | 166 |\n| `sound_generation_QA` | 100 | 222.52 | 130 | 423 |\n| `music_QA` | 400 | 57.54 | 24 | 202 |\n| `music_generation_analysis_QA` | 100 | 267.52 | 148 | 395 |\n| `speech_and_sound_QA` | 200 | 63.98 | 25 | 127 |\n| `speech_and_music_QA` | 200 | 69.37 | 32 | 127 |\n\n**Audio Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Audio Files | 2,200 |\n| Audio per Sample | min: 1, max: 1, mean: 1 |\n| Formats | mp3, wav |\n\n\n## Sample Example\n\n**Subset**: `speech_QA`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"5781ee73\",\n \"content\": [\n {\n \"audio\": \"/root/.cache/modelscope/hub/datasets/evalscope/AIR-Bench-Dataset/Chat/speech_QA_iemocap/Ses01F_script01_1_M025.wav\",\n \"format\": \"wav\"\n },\n {\n \"text\": \"Who is the speaker addressing at the end of the speech?\"\n }\n ]\n }\n ],\n \"target\": \"The speaker is addressing Mom at the end of the speech.\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"speech_QA\",\n \"metadata\": {\n \"uniq_id\": 400,\n \"task_name\": \"speech_QA\",\n \"dataset_name\": \"iemocap\",\n \"category\": \"speech\",\n \"meta_info\": \"{'emotion': 'neutral', 'gender': 'male', 'transcription': \\\"And then we'll thrash it out with father. Okay Mom? Don't avoid me.\\\"}\",\n \"question\": \"Who is the speaker addressing at the end of the speech?\"\n }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `tasks` | `list` | `None` | Optional list of Chat task names to evaluate (subset of ['music_QA', 'music_generation_analysis_QA', 'sound_QA', 'sound_generation_QA', 'speech_QA', 'speech_and_music_QA', 'speech_and_sound_QA', 'speech_dialogue_QA']). Defaults to all tasks. |\n| `do_swap` | `bool` | `True` | When True (default), each sample is judged twice with the order of reference vs. prediction swapped, then scores are averaged. Disable to halve judge cost at the price of position bias. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets air_bench_chat \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['air_bench_chat'],\n dataset_args={\n 'air_bench_chat': {\n # subset_list: ['speech_QA', 'speech_dialogue_QA', 'sound_QA'] # optional, evaluate specific subsets\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# AIR-Bench-Chat\n\n\n## 概述\n\nAIR-Bench Chat 是 [AIR-Bench](https://arxiv.org/abs/2402.07729)（Audio InstRuction Benchmark，ACL 2024 主会）的生成式部分——这是首个面向大音频语言模型（LALMs）的指令遵循基准测试，涵盖**人类语音、自然声音和音乐**。该数据集包含约 2,000 个开放式音频问答对，覆盖语音、声音、音乐及混合音频场景；模型的回答由 GPT-4 评判器根据参考答案进行评分。\n\n## 任务描述\n\n- **任务类型**：开放式音频问答。\n- **输入**：一段音频片段加一个自由形式的问题。\n- **输出**：文本答案，与参考答案进行对比评估。\n- **模态**：音频（人类语音、自然声音、音乐）+ 文本。\n\n## 核心特性\n\n- 包含约 2k 个开放式音频问答对，覆盖语音、声音、音乐及混合音频场景；是 AIR-Bench（ACL 2024）的生成式部分。\n- 官方 `cal_score.py` 将 8 个 Chat 任务聚合为 5 个报告类别：`speech`（`speech_QA`, `speech_dialogue_QA`）、`sound`（`sound_QA`, `sound_generation_QA`）、`music`（`music_QA`, `music_generation_analysis_QA`）、`speech_and_sound`（`speech_and_sound_QA`）、`speech_and_music`（`speech_and_music_QA`）。论文中的 Mixed-audio = mean(speech_and_sound, speech_and_music)。\n- 通过将每个样本判断两次（交换参考答案与模型预测的顺序）并取平均值来消除位置偏差（可通过设置 `extra_params={'do_swap': False}` 禁用此机制，以节省一半评判成本）。\n- 数据托管于 ModelScope（[`evalscope/AIR-Bench-Dataset`](https://modelscope.cn/datasets/evalscope/AIR-Bench-Dataset)），采用 audiofolder + JSON 格式；完整数据集约 49 GB，可通过 `extra_params={'tasks': [...]}` 限制任务范围以进行部分运行。\n\n## 评估说明\n\n- **指标**：`judge_score` 表示模型的平均评判分数；`win_rate` 记录模型严格优于参考答案的频率。\n- 评判 LLM 接收问题、音频的文本描述（`meta_info`）、参考答案（`answer_gt`）和模型回答，并输出两个 `[1, 10]` 范围内的整数分数。由于对话类任务的 `meta_info` 可能超过 4k tokens，请使用支持长上下文的评判模型。\n- 官方排行榜使用 `gpt-4-0125-preview`。若该特定快照不可用，请使用其他可用的 GPT-4 级别评判模型；由于评判模型变更，绝对分数可能与已发表结果存在偏差。\n- 若数据集已下载至本地，可通过 `dataset_args={'air_bench_chat': {'local_path': '/path/to/AIR-Bench-Dataset'}}` 指定路径；本地根目录应包含 `Chat/` 文件夹。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `air_bench_chat` |\n| **数据集ID** | [evalscope/AIR-Bench](https://modelscope.cn/datasets/evalscope/AIR-Bench/summary) |\n| **论文** | [Paper](https://aclanthology.org/2024.acl-long.109/) |\n| **标签** | `Audio`, `InstructionFollowing`, `QA` |\n| **指标** | `judge_score`, `win_rate` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 2,200 |\n| 提示词长度（平均） | 83.89 字符 |\n| 提示词长度（最小/最大） | 17 / 423 字符 |\n\n**各子集统计信息：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `speech_QA` | 400 | 64.33 | 23 | 148 |\n| `speech_dialogue_QA` | 400 | 77.03 | 29 | 206 |\n| `sound_QA` | 400 | 73.29 | 17 | 166 |\n| `sound_generation_QA` | 100 | 222.52 | 130 | 423 |\n| `music_QA` | 400 | 57.54 | 24 | 202 |\n| `music_generation_analysis_QA` | 100 | 267.52 | 148 | 395 |\n| `speech_and_sound_QA` | 200 | 63.98 | 25 | 127 |\n| `speech_and_music_QA` | 200 | 69.37 | 32 | 127 |\n\n**音频统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 音频文件总数 | 2,200 |\n| 每样本音频数量 | 最小: 1, 最大: 1, 平均: 1 |\n| 格式 | mp3, wav |\n\n\n## 样例示例\n\n**子集**: `speech_QA`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"5781ee73\",\n \"content\": [\n {\n \"audio\": \"/root/.cache/modelscope/hub/datasets/evalscope/AIR-Bench-Dataset/Chat/speech_QA_iemocap/Ses01F_script01_1_M025.wav\",\n \"format\": \"wav\"\n },\n {\n \"text\": \"Who is the speaker addressing at the end of the speech?\"\n }\n ]\n }\n ],\n \"target\": \"The speaker is addressing Mom at the end of the speech.\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"speech_QA\",\n \"metadata\": {\n \"uniq_id\": 400,\n \"task_name\": \"speech_QA\",\n \"dataset_name\": \"iemocap\",\n \"category\": \"speech\",\n \"meta_info\": \"{'emotion': 'neutral', 'gender': 'male', 'transcription': \\\"And then we'll thrash it out with father. Okay Mom? Don't avoid me.\\\"}\",\n \"question\": \"Who is the speaker addressing at the end of the speech?\"\n }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `tasks` | `list` | `None` | 可选的 Chat 任务名称列表（子集包括 ['music_QA', 'music_generation_analysis_QA', 'sound_QA', 'sound_generation_QA', 'speech_QA', 'speech_and_music_QA', 'speech_and_sound_QA', 'speech_dialogue_QA']）。默认评估所有任务。 |\n| `do_swap` | `bool` | `True` | 若为 True（默认），每个样本会被评判两次（交换参考答案与模型预测的顺序），然后取平均分。禁用此选项可节省一半评判成本，但会引入位置偏差。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets air_bench_chat \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['air_bench_chat'],\n dataset_args={\n 'air_bench_chat': {\n # subset_list: ['speech_QA', 'speech_dialogue_QA', 'sound_QA'] # 可选，仅评估指定子集\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "7dfac17110d8426ac989eaf3cfdebf82", "needs_translation": false }, - "updated_at": "2026-08-12T15:08:01.351654", - "translation_updated_at": "2026-05-18T10:23:22" + "updated_at": "2026-08-18T21:23:00.927854", + "translation_updated_at": "2026-08-18T21:23:02" } diff --git a/evalscope/benchmarks/_meta/baby_vision.json b/evalscope/benchmarks/_meta/baby_vision.json index 839e123f5..d398f6bbf 100644 --- a/evalscope/benchmarks/_meta/baby_vision.json +++ b/evalscope/benchmarks/_meta/baby_vision.json @@ -20,7 +20,7 @@ "Visual Pattern Recognition", "Visual Tracking" ], - "description": "\n## Overview\n\nBabyVision is a visual perception benchmark that evaluates the fundamental visual abilities of\nmultimodal large language models through tasks inspired by infant and early childhood visual\ndevelopment. It focuses on fine-grained discrimination, spatial perception, visual pattern\nrecognition, and visual tracking.\n\n## Task Description\n\n- **Task Type**: Visual Perception (Choice + Fill-in-the-blank)\n- **Input**: Image + question\n- **Output**: Choice letter or free-form short answer\n- **Domains**: Fine-grained discrimination, spatial perception, visual pattern recognition, visual tracking\n\n## Key Features\n\n- 388 test samples across 4 major visual ability categories and 22 subtypes\n- Two answer types: choice (135 samples) and blank (253 samples)\n- Subtypes include: Find the different, Find the same, Count clusters, Maze,\n 3D cube unfold, Pattern completion, Paper folding, Rotation patterns, etc.\n- Tests low-level visual perception rather than high-level reasoning or knowledge\n- Includes Chain-of-Thought (CoT) reference for analysis\n\n## Evaluation Notes\n\n- Default evaluation uses the **train** split (388 samples, single split dataset)\n- Primary metric: **Accuracy** via LLM-as-judge\n- Subsets organized by `type` field (4 categories)\n- LLM judge evaluates both choice and blank answer types uniformly\n- Requires `judge_model_args` configuration for LLM judge\n", + "description": "\n## Overview\n\nBabyVision is a visual perception benchmark that evaluates the fundamental visual abilities of\nmultimodal large language models through tasks inspired by infant and early childhood visual\ndevelopment. It focuses on fine-grained discrimination, spatial perception, visual pattern\nrecognition, and visual tracking.\n\n## Task Description\n\n- **Task Type**: Visual Perception (Choice + Fill-in-the-blank)\n- **Input**: Image + question\n- **Output**: Choice letter or free-form short answer\n- **Domains**: Fine-grained discrimination, spatial perception, visual pattern recognition, visual tracking\n\n## Key Features\n\n- 388 test samples across 4 major visual ability categories and 22 subtypes\n- Two answer types: choice (135 samples) and blank (253 samples)\n- Subtypes include: Find the different, Find the same, Count clusters, Maze,\n 3D cube unfold, Pattern completion, Paper folding, Rotation patterns, etc.\n- Tests low-level visual perception rather than high-level reasoning or knowledge\n- Includes Chain-of-Thought (CoT) reference for analysis\n\n## Evaluation Notes\n\n- Default evaluation uses the **train** split (388 samples, single split dataset)\n- Primary metric: **Accuracy** via LLM-as-judge\n- Subsets organized by `type` field (4 categories)\n- LLM judge evaluates both choice and blank answer types uniformly\n- Requires an LLM judge configured through `judge.models`\n", "prompt_template": "", "system_prompt": "", "few_shot_prompt_template": "", @@ -30,232 +30,28 @@ "category": "vlm" }, "statistics": { - "total_samples": 388, - "subset_stats": [ - { - "name": "Fine-grained Discrimination", - "sample_count": 163, - "prompt_length_mean": 152.09, - "prompt_length_min": 33, - "prompt_length_max": 450, - "prompt_length_std": 78.17, - "target_length_mean": 6.91, - "multimodal": { - "has_images": true, - "has_audio": false, - "has_video": false, - "image": { - "count_total": 163, - "count_per_sample": { - "min": 1, - "max": 1, - "mean": 1 - }, - "resolutions": [ - "1094x390", - "1104x1027", - "1212x882", - "1256x1017", - "213x270", - "216x275", - "228x228", - "234x228", - "239x225", - "243x225" - ], - "resolution_range": { - "min": "228x228", - "max": "758x1810" - }, - "formats": [ - "jpeg", - "png" - ] - } - } - }, - { - "name": "Spatial Perception", - "sample_count": 91, - "prompt_length_mean": 157.18, - "prompt_length_min": 73, - "prompt_length_max": 370, - "prompt_length_std": 61.52, - "target_length_mean": 4.68, - "multimodal": { - "has_images": true, - "has_audio": false, - "has_video": false, - "image": { - "count_total": 91, - "count_per_sample": { - "min": 1, - "max": 1, - "mean": 1 - }, - "resolutions": [ - "1034x1152", - "1050x854", - "1071x816", - "1086x1184", - "1088x872", - "1092x916", - "1096x954", - "1100x1112", - "1179x1619", - "1254x537" - ], - "resolution_range": { - "min": "209x168", - "max": "1179x1619" - }, - "formats": [ - "jpeg", - "png" - ] - } - } - }, - { - "name": "Visual Pattern Recognition", - "sample_count": 51, - "prompt_length_mean": 178.92, - "prompt_length_min": 94, - "prompt_length_max": 319, - "prompt_length_std": 53.47, - "target_length_mean": 3.73, - "multimodal": { - "has_images": true, - "has_audio": false, - "has_video": false, - "image": { - "count_total": 51, - "count_per_sample": { - "min": 1, - "max": 1, - "mean": 1 - }, - "resolutions": [ - "1023x605", - "1224x1110", - "164x226", - "246x571", - "264x600", - "271x600", - "303x717", - "332x600", - "360x476", - "373x673" - ], - "resolution_range": { - "min": "164x226", - "max": "1224x1110" - }, - "formats": [ - "jpeg", - "png" - ] - } - } - }, - { - "name": "Visual Tracking", - "sample_count": 83, - "prompt_length_mean": 201.45, - "prompt_length_min": 97, - "prompt_length_max": 389, - "prompt_length_std": 64.88, - "target_length_mean": 9.33, - "multimodal": { - "has_images": true, - "has_audio": false, - "has_video": false, - "image": { - "count_total": 83, - "count_per_sample": { - "min": 1, - "max": 1, - "mean": 1 - }, - "resolutions": [ - "1012x984", - "1400x1406", - "1502x1211", - "1690x1466", - "1744x1080", - "174x144", - "174x147", - "180x144", - "189x144", - "189x147" - ], - "resolution_range": { - "min": "174x144", - "max": "2378x1448" - }, - "formats": [ - "jpeg", - "png", - "webp" - ] - } - } - } - ], + "total_samples": 0, + "subset_stats": [], "prompt_length": { - "mean": 167.37, - "min": 33, - "max": 450, - "std": 71.34 + "mean": 0, + "min": 0, + "max": 0, + "std": null }, - "target_length_mean": 6.49, - "computed_at": "2026-07-03T11:09:46.867162", - "multimodal": { - "has_images": true, - "has_audio": false, - "has_video": false, - "image": { - "count_total": 388, - "count_per_sample": { - "min": 1, - "max": 1, - "mean": 1 - }, - "resolutions": [ - "1012x984", - "1023x605", - "1034x1152", - "1050x854", - "1071x816", - "1086x1184", - "1088x872", - "1092x916", - "1094x390", - "1096x954" - ], - "resolution_range": { - "min": "174x144", - "max": "2378x1448" - }, - "formats": [ - "jpeg", - "png", - "webp" - ] - } - } + "target_length_mean": null, + "computed_at": "2026-08-20T10:39:29.882507" }, "sample_example": { "data": { "input": [ { - "id": "8b83904b", + "id": "1256c961", "content": [ { "image": "[BASE64_IMAGE: jpeg, ~77.0KB]" }, { - "text": "The image shows a total of 49 tiger patterns arranged in 7 rows and 7 columns. One of them is different from the others. Which row and column is it in? The answer format is (x,y). (For example, the answer for the 2nd row and 3rd column is (2,3))." + "text": "The image shows a total of 49 tiger patterns arranged in 7 rows and 7 columns. One of them is different from the others. Which row and column is it in? The answer format is (x,y). (For example, the answer for the 2nd row and 3rd column is (2,3)).\nThink about the question and give your final answer in \\boxed{Answer} format." } ] } @@ -276,11 +72,11 @@ "truncated": false }, "readme": { - "en": "# BabyVision\n\n\n## Overview\n\nBabyVision is a visual perception benchmark that evaluates the fundamental visual abilities of\nmultimodal large language models through tasks inspired by infant and early childhood visual\ndevelopment. It focuses on fine-grained discrimination, spatial perception, visual pattern\nrecognition, and visual tracking.\n\n## Task Description\n\n- **Task Type**: Visual Perception (Choice + Fill-in-the-blank)\n- **Input**: Image + question\n- **Output**: Choice letter or free-form short answer\n- **Domains**: Fine-grained discrimination, spatial perception, visual pattern recognition, visual tracking\n\n## Key Features\n\n- 388 test samples across 4 major visual ability categories and 22 subtypes\n- Two answer types: choice (135 samples) and blank (253 samples)\n- Subtypes include: Find the different, Find the same, Count clusters, Maze,\n 3D cube unfold, Pattern completion, Paper folding, Rotation patterns, etc.\n- Tests low-level visual perception rather than high-level reasoning or knowledge\n- Includes Chain-of-Thought (CoT) reference for analysis\n\n## Evaluation Notes\n\n- Default evaluation uses the **train** split (388 samples, single split dataset)\n- Primary metric: **Accuracy** via LLM-as-judge\n- Subsets organized by `type` field (4 categories)\n- LLM judge evaluates both choice and blank answer types uniformly\n- Requires `judge_model_args` configuration for LLM judge\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `baby_vision` |\n| **Dataset ID** | [evalscope/BabyVision](https://modelscope.cn/datasets/evalscope/BabyVision/summary) |\n| **Paper** | N/A |\n| **Tags** | `MultiModal`, `QA`, `Reasoning` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 388 |\n| Prompt Length (Mean) | 167.37 chars |\n| Prompt Length (Min/Max) | 33 / 450 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `Fine-grained Discrimination` | 163 | 152.09 | 33 | 450 |\n| `Spatial Perception` | 91 | 157.18 | 73 | 370 |\n| `Visual Pattern Recognition` | 51 | 178.92 | 94 | 319 |\n| `Visual Tracking` | 83 | 201.45 | 97 | 389 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 388 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 174x144 - 2378x1448 |\n| Formats | jpeg, png, webp |\n\n\n## Sample Example\n\n**Subset**: `Fine-grained Discrimination`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"8b83904b\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~77.0KB]\"\n },\n {\n \"text\": \"The image shows a total of 49 tiger patterns arranged in 7 rows and 7 columns. One of them is different from the others. Which row and column is it in? The answer format is (x,y). (For example, the answer for the 2nd row and 3rd column is (2,3)).\"\n }\n ]\n }\n ],\n \"target\": \"(4,7)\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"Fine-grained Discrimination\",\n \"metadata\": {\n \"taskId\": 445,\n \"type\": \"Fine-grained Discrimination\",\n \"subtype\": \"Find the different\",\n \"ansType\": \"blank\",\n \"coT\": \"The image shows 49 tiger patterns arranged in 7 rows and 7 columns.\\nNow, we need to find the coordinates of the one tiger pattern that is different from the other 48.\\nIt can be observed that the tiger in the fourth row and seventh column has no ears (the ears are located in the upper right corner of each tiger pattern), while the other 48 tigers have ears.\\nTherefore, the correct answer is (4,7).\"\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets baby_vision \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['baby_vision'],\n dataset_args={\n 'baby_vision': {\n # subset_list: ['Fine-grained Discrimination', 'Spatial Perception', 'Visual Pattern Recognition'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# BabyVision\n\n\n## 概述\n\nBabyVision 是一个视觉感知基准测试，通过受婴儿及幼儿视觉发展启发的任务，评估多模态大语言模型的基础视觉能力。该基准聚焦于细粒度辨别、空间感知、视觉模式识别和视觉追踪能力。\n\n## 任务描述\n\n- **任务类型**：视觉感知（选择题 + 填空题）\n- **输入**：图像 + 问题\n- **输出**：选项字母或自由格式的简短答案\n- **领域**：细粒度辨别、空间感知、视觉模式识别、视觉追踪\n\n## 主要特点\n\n- 包含 388 个测试样本，覆盖 4 大视觉能力类别和 22 个子类型\n- 两种答案类型：选择题（135 个样本）和填空题（253 个样本）\n- 子类型包括：找不同、找相同、数簇、迷宫、立方体展开图、图案补全、折纸、旋转图案等\n- 侧重于低层次视觉感知，而非高层次推理或知识\n- 提供思维链（Chain-of-Thought, CoT）参考用于分析\n\n## 评估说明\n\n- 默认评估使用 **train** 切分（388 个样本，单切分数据集）\n- 主要指标：通过 LLM-as-judge 计算的 **准确率（Accuracy）**\n- 子集按 `type` 字段组织（4 个类别）\n- LLM 评判器统一评估选择题和填空题两种答案类型\n- 需要配置 `judge_model_args` 以启用 LLM 评判器\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `baby_vision` |\n| **数据集ID** | [evalscope/BabyVision](https://modelscope.cn/datasets/evalscope/BabyVision/summary) |\n| **论文** | N/A |\n| **标签** | `MultiModal`, `QA`, `Reasoning` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估切分** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 388 |\n| 提示词长度（平均） | 167.37 字符 |\n| 提示词长度（最小/最大） | 33 / 450 字符 |\n\n**各子集统计信息：**\n\n| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 |\n|--------|---------|-------------|------------|------------|\n| `Fine-grained Discrimination` | 163 | 152.09 | 33 | 450 |\n| `Spatial Perception` | 91 | 157.18 | 73 | 370 |\n| `Visual Pattern Recognition` | 51 | 178.92 | 94 | 319 |\n| `Visual Tracking` | 83 | 201.45 | 97 | 389 |\n\n**图像统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 388 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 174x144 - 2378x1448 |\n| 格式 | jpeg, png, webp |\n\n\n## 样例示例\n\n**子集**: `Fine-grained Discrimination`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"8b83904b\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~77.0KB]\"\n },\n {\n \"text\": \"The image shows a total of 49 tiger patterns arranged in 7 rows and 7 columns. One of them is different from the others. Which row and column is it in? The answer format is (x,y). (For example, the answer for the 2nd row and 3rd column is (2,3)).\"\n }\n ]\n }\n ],\n \"target\": \"(4,7)\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"Fine-grained Discrimination\",\n \"metadata\": {\n \"taskId\": 445,\n \"type\": \"Fine-grained Discrimination\",\n \"subtype\": \"Find the different\",\n \"ansType\": \"blank\",\n \"coT\": \"The image shows 49 tiger patterns arranged in 7 rows and 7 columns.\\nNow, we need to find the coordinates of the one tiger pattern that is different from the other 48.\\nIt can be observed that the tiger in the fourth row and seventh column has no ears (the ears are located in the upper right corner of each tiger pattern), while the other 48 tigers have ears.\\nTherefore, the correct answer is (4,7).\"\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 通过命令行（CLI）\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets baby_vision \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 通过 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['baby_vision'],\n dataset_args={\n 'baby_vision': {\n # subset_list: ['Fine-grained Discrimination', 'Spatial Perception', 'Visual Pattern Recognition'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "48f684de9e61019c216ec3bbbe2d123b", + "en": "# BabyVision\n\n\n## Overview\n\nBabyVision is a visual perception benchmark that evaluates the fundamental visual abilities of\nmultimodal large language models through tasks inspired by infant and early childhood visual\ndevelopment. It focuses on fine-grained discrimination, spatial perception, visual pattern\nrecognition, and visual tracking.\n\n## Task Description\n\n- **Task Type**: Visual Perception (Choice + Fill-in-the-blank)\n- **Input**: Image + question\n- **Output**: Choice letter or free-form short answer\n- **Domains**: Fine-grained discrimination, spatial perception, visual pattern recognition, visual tracking\n\n## Key Features\n\n- 388 test samples across 4 major visual ability categories and 22 subtypes\n- Two answer types: choice (135 samples) and blank (253 samples)\n- Subtypes include: Find the different, Find the same, Count clusters, Maze,\n 3D cube unfold, Pattern completion, Paper folding, Rotation patterns, etc.\n- Tests low-level visual perception rather than high-level reasoning or knowledge\n- Includes Chain-of-Thought (CoT) reference for analysis\n\n## Evaluation Notes\n\n- Default evaluation uses the **train** split (388 samples, single split dataset)\n- Primary metric: **Accuracy** via LLM-as-judge\n- Subsets organized by `type` field (4 categories)\n- LLM judge evaluates both choice and blank answer types uniformly\n- Requires an LLM judge configured through `judge.models`\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `baby_vision` |\n| **Dataset ID** | [evalscope/BabyVision](https://modelscope.cn/datasets/evalscope/BabyVision/summary) |\n| **Paper** | N/A |\n| **Tags** | `MultiModal`, `QA`, `Reasoning` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n*Statistics not available.*\n\n## Sample Example\n\n**Subset**: `Fine-grained Discrimination`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"1256c961\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~77.0KB]\"\n },\n {\n \"text\": \"The image shows a total of 49 tiger patterns arranged in 7 rows and 7 columns. One of them is different from the others. Which row and column is it in? The answer format is (x,y). (For example, the answer for the 2nd row and 3rd column is (2,3)).\\nThink about the question and give your final answer in \\\\boxed{Answer} format.\"\n }\n ]\n }\n ],\n \"target\": \"(4,7)\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"Fine-grained Discrimination\",\n \"metadata\": {\n \"taskId\": 445,\n \"type\": \"Fine-grained Discrimination\",\n \"subtype\": \"Find the different\",\n \"ansType\": \"blank\",\n \"coT\": \"The image shows 49 tiger patterns arranged in 7 rows and 7 columns.\\nNow, we need to find the coordinates of the one tiger pattern that is different from the other 48.\\nIt can be observed that the tiger in the fourth row and seventh column has no ears (the ears are located in the upper right corner of each tiger pattern), while the other 48 tigers have ears.\\nTherefore, the correct answer is (4,7).\"\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets baby_vision \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['baby_vision'],\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# BabyVision\n\n\n## 概述\n\nBabyVision 是一个视觉感知基准测试，通过受婴儿及幼儿视觉发展启发的任务，评估多模态大语言模型的基础视觉能力。该基准聚焦于细粒度辨别、空间感知、视觉模式识别和视觉追踪。\n\n## 任务描述\n\n- **任务类型**：视觉感知（选择题 + 填空题）\n- **输入**：图像 + 问题\n- **输出**：选项字母或自由格式的简短答案\n- **领域**：细粒度辨别、空间感知、视觉模式识别、视觉追踪\n\n## 主要特点\n\n- 包含 388 个测试样本，覆盖 4 大视觉能力类别和 22 个子类型\n- 两种答案类型：选择题（135 个样本）和填空题（235 个样本）\n- 子类型包括：找不同、找相同、数簇、迷宫、立方体展开图、图案补全、折纸、旋转图案等\n- 测试低层次视觉感知能力，而非高层次推理或知识\n- 提供思维链（Chain-of-Thought, CoT）参考用于分析\n\n## 评估说明\n\n- 默认评估使用 **train** 切分（388 个样本，单切分数据集）\n- 主要指标：通过 LLM-as-judge 计算的 **准确率（Accuracy）**\n- 子集按 `type` 字段组织（共 4 个类别）\n- LLM 评判器统一评估选择题和填空题两种答案类型\n- 需通过 `judge.models` 配置 LLM 评判器\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `baby_vision` |\n| **数据集ID** | [evalscope/BabyVision](https://modelscope.cn/datasets/evalscope/BabyVision/summary) |\n| **论文** | N/A |\n| **标签** | `MultiModal`, `QA`, `Reasoning` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估切分** | `train` |\n\n\n## 数据统计\n\n*统计数据不可用。*\n\n## 样例示例\n\n**子集**: `Fine-grained Discrimination`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"1256c961\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~77.0KB]\"\n },\n {\n \"text\": \"The image shows a total of 49 tiger patterns arranged in 7 rows and 7 columns. One of them is different from the others. Which row and column is it in? The answer format is (x,y). (For example, the answer for the 2nd row and 3rd column is (2,3)).\\nThink about the question and give your final answer in \\\\boxed{Answer} format.\"\n }\n ]\n }\n ],\n \"target\": \"(4,7)\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"Fine-grained Discrimination\",\n \"metadata\": {\n \"taskId\": 445,\n \"type\": \"Fine-grained Discrimination\",\n \"subtype\": \"Find the different\",\n \"ansType\": \"blank\",\n \"coT\": \"The image shows 49 tiger patterns arranged in 7 rows and 7 columns.\\nNow, we need to find the coordinates of the one tiger pattern that is different from the other 48.\\nIt can be observed that the tiger in the fourth row and seventh column has no ears (the ears are located in the upper right corner of each tiger pattern), while the other 48 tigers have ears.\\nTherefore, the correct answer is (4,7).\"\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets baby_vision \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['baby_vision'],\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "9dafcc30189073e81e0f0a0f5a35a976", "needs_translation": false }, - "updated_at": "2026-08-11T15:30:53.578834", - "translation_updated_at": "2026-07-03T16:19:29" + "updated_at": "2026-08-20T10:40:16.049132", + "translation_updated_at": "2026-08-20T10:40:30" } diff --git a/evalscope/benchmarks/_meta/charxiv.json b/evalscope/benchmarks/_meta/charxiv.json index cdffe8e40..02291e8dd 100644 --- a/evalscope/benchmarks/_meta/charxiv.json +++ b/evalscope/benchmarks/_meta/charxiv.json @@ -18,7 +18,7 @@ "descriptive", "reasoning" ], - "description": "\n## Overview\n\nCharXiv is a comprehensive chart understanding benchmark from NeurIPS 2024 that evaluates multimodal\nlarge language models on realistic scientific charts from arXiv papers. It tests both low-level\nchart element perception (descriptive) and high-level reasoning about chart data.\n\n## Task Description\n\n- **Task Type**: Chart Understanding (Descriptive + Reasoning)\n- **Input**: Scientific chart image + question\n- **Output**: Free-form text answer\n- **Domains**: cs, physics, math, eess, q-bio, q-fin, stat, econ\n\n## Key Features\n\n- 2,323 real scientific charts from arXiv papers across 8 disciplines\n- Two question types:\n - **Descriptive** (4 per chart): Basic element identification (titles, axes, legends, trends, etc.)\n - **Reasoning** (1 per chart): Higher-order reasoning requiring data synthesis\n- 19 descriptive question templates covering information extraction, enumeration, pattern recognition, counting, and compositionality\n- 4 reasoning answer types: text-in-chart, text-in-general, number-in-chart, number-in-general\n- Validation set (1,000 charts) and test set (1,323 charts)\n- Evaluation via LLM judge following the official CharXiv grading protocol\n\n## Evaluation Notes\n\n- Default evaluation uses the **validation** split (1,000 charts, 5,000 questions)\n- Each chart yields 5 samples: 4 descriptive + 1 reasoning\n- Primary metric: **Accuracy** via LLM-as-judge\n- Subsets: `descriptive` and `reasoning` (also by category)\n- Requires `judge_model_args` configuration for LLM judge\n- [Paper](https://arxiv.org/abs/2406.18521) | [GitHub](https://github.com/princeton-nlp/CharXiv)\n", + "description": "\n## Overview\n\nCharXiv is a comprehensive chart understanding benchmark from NeurIPS 2024 that evaluates multimodal\nlarge language models on realistic scientific charts from arXiv papers. It tests both low-level\nchart element perception (descriptive) and high-level reasoning about chart data.\n\n## Task Description\n\n- **Task Type**: Chart Understanding (Descriptive + Reasoning)\n- **Input**: Scientific chart image + question\n- **Output**: Free-form text answer\n- **Domains**: cs, physics, math, eess, q-bio, q-fin, stat, econ\n\n## Key Features\n\n- 2,323 real scientific charts from arXiv papers across 8 disciplines\n- Two question types:\n - **Descriptive** (4 per chart): Basic element identification (titles, axes, legends, trends, etc.)\n - **Reasoning** (1 per chart): Higher-order reasoning requiring data synthesis\n- 19 descriptive question templates covering information extraction, enumeration, pattern recognition, counting, and compositionality\n- 4 reasoning answer types: text-in-chart, text-in-general, number-in-chart, number-in-general\n- Validation set (1,000 charts) and test set (1,323 charts)\n- Evaluation via LLM judge following the official CharXiv grading protocol\n\n## Evaluation Notes\n\n- Default evaluation uses the **validation** split (1,000 charts, 5,000 questions)\n- Each chart yields 5 samples: 4 descriptive + 1 reasoning\n- Primary metric: **Accuracy** via LLM-as-judge\n- Subsets: `descriptive` and `reasoning` (also by category)\n- Requires an LLM judge configured through `judge.models`\n- [Paper](https://arxiv.org/abs/2406.18521) | [GitHub](https://github.com/princeton-nlp/CharXiv)\n", "prompt_template": "", "system_prompt": "", "few_shot_prompt_template": "", @@ -28,137 +28,22 @@ "category": "vlm" }, "statistics": { - "total_samples": 5000, - "subset_stats": [ - { - "name": "descriptive", - "sample_count": 4000, - "prompt_length_mean": 261.51, - "prompt_length_min": 156, - "prompt_length_max": 432, - "prompt_length_std": 60.39, - "target_length_mean": 9.04, - "multimodal": { - "has_images": true, - "has_audio": false, - "has_video": false, - "image": { - "count_total": 4000, - "count_per_sample": { - "min": 1, - "max": 1, - "mean": 1 - }, - "resolutions": [ - "1001x1024", - "1002x1024", - "1010x1023", - "1011x1024", - "1014x1024", - "1015x1024", - "1016x1023", - "1019x1024", - "1021x1024", - "1023x1015" - ], - "resolution_range": { - "min": "1023x139", - "max": "1024x1024" - }, - "formats": [ - "jpeg" - ] - } - } - }, - { - "name": "reasoning", - "sample_count": 1000, - "prompt_length_mean": 335.14, - "prompt_length_min": 80, - "prompt_length_max": 687, - "prompt_length_std": 121.63, - "target_length_mean": 6.67, - "multimodal": { - "has_images": true, - "has_audio": false, - "has_video": false, - "image": { - "count_total": 1000, - "count_per_sample": { - "min": 1, - "max": 1, - "mean": 1 - }, - "resolutions": [ - "1001x1024", - "1002x1024", - "1010x1023", - "1011x1024", - "1014x1024", - "1015x1024", - "1016x1023", - "1019x1024", - "1021x1024", - "1023x1015" - ], - "resolution_range": { - "min": "1023x139", - "max": "1024x1024" - }, - "formats": [ - "jpeg" - ] - } - } - } - ], + "total_samples": 0, + "subset_stats": [], "prompt_length": { - "mean": 276.24, - "min": 80, - "max": 687, - "std": 82.1 + "mean": 0, + "min": 0, + "max": 0, + "std": null }, - "target_length_mean": 8.57, - "computed_at": "2026-07-03T11:09:48.413672", - "multimodal": { - "has_images": true, - "has_audio": false, - "has_video": false, - "image": { - "count_total": 5000, - "count_per_sample": { - "min": 1, - "max": 1, - "mean": 1 - }, - "resolutions": [ - "1001x1024", - "1002x1024", - "1010x1023", - "1011x1024", - "1014x1024", - "1015x1024", - "1016x1023", - "1019x1024", - "1021x1024", - "1023x1015" - ], - "resolution_range": { - "min": "1023x139", - "max": "1024x1024" - }, - "formats": [ - "jpeg" - ] - } - } + "target_length_mean": null, + "computed_at": "2026-08-20T10:39:29.881476" }, "sample_example": { "data": { "input": [ { - "id": "44ff5b8a", + "id": "dd588b4d", "content": [ { "image": "[BASE64_IMAGE: jpeg, ~70.0KB]" @@ -184,11 +69,11 @@ "truncated": false }, "readme": { - "en": "# CharXiv\n\n\n## Overview\n\nCharXiv is a comprehensive chart understanding benchmark from NeurIPS 2024 that evaluates multimodal\nlarge language models on realistic scientific charts from arXiv papers. It tests both low-level\nchart element perception (descriptive) and high-level reasoning about chart data.\n\n## Task Description\n\n- **Task Type**: Chart Understanding (Descriptive + Reasoning)\n- **Input**: Scientific chart image + question\n- **Output**: Free-form text answer\n- **Domains**: cs, physics, math, eess, q-bio, q-fin, stat, econ\n\n## Key Features\n\n- 2,323 real scientific charts from arXiv papers across 8 disciplines\n- Two question types:\n - **Descriptive** (4 per chart): Basic element identification (titles, axes, legends, trends, etc.)\n - **Reasoning** (1 per chart): Higher-order reasoning requiring data synthesis\n- 19 descriptive question templates covering information extraction, enumeration, pattern recognition, counting, and compositionality\n- 4 reasoning answer types: text-in-chart, text-in-general, number-in-chart, number-in-general\n- Validation set (1,000 charts) and test set (1,323 charts)\n- Evaluation via LLM judge following the official CharXiv grading protocol\n\n## Evaluation Notes\n\n- Default evaluation uses the **validation** split (1,000 charts, 5,000 questions)\n- Each chart yields 5 samples: 4 descriptive + 1 reasoning\n- Primary metric: **Accuracy** via LLM-as-judge\n- Subsets: `descriptive` and `reasoning` (also by category)\n- Requires `judge_model_args` configuration for LLM judge\n- [Paper](https://arxiv.org/abs/2406.18521) | [GitHub](https://github.com/princeton-nlp/CharXiv)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `charxiv` |\n| **Dataset ID** | [princeton-nlp/CharXiv](https://modelscope.cn/datasets/princeton-nlp/CharXiv/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2406.18521) |\n| **Tags** | `MultiModal`, `QA`, `Reasoning` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `validation` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 5,000 |\n| Prompt Length (Mean) | 276.24 chars |\n| Prompt Length (Min/Max) | 80 / 687 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `descriptive` | 4,000 | 261.51 | 156 | 432 |\n| `reasoning` | 1,000 | 335.14 | 80 | 687 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 5,000 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 1023x139 - 1024x1024 |\n| Formats | jpeg |\n\n\n## Sample Example\n\n**Subset**: `descriptive`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"44ff5b8a\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~70.0KB]\"\n },\n {\n \"text\": \"For the current plot, what is the spatially highest labeled tick on the y-axis?\\n* Your final answer should be the tick value on the y-axis that is explicitly written. Ignore units or scales that are written separately from the tick.\"\n }\n ]\n }\n ],\n \"target\": \"60\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"descriptive\",\n \"metadata\": {\n \"question_type\": \"descriptive\",\n \"question_id\": 7,\n \"category\": \"cs\",\n \"original_id\": \"2004.10956\"\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets charxiv \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['charxiv'],\n dataset_args={\n 'charxiv': {\n # subset_list: ['descriptive', 'reasoning'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# CharXiv\n\n\n## 概述\n\nCharXiv 是 NeurIPS 2024 提出的一个全面的图表理解基准测试，用于评估多模态大语言模型在来自 arXiv 论文的真实科学图表上的表现。该基准同时考察模型对图表元素的低层次感知能力（描述性任务）和对图表数据的高层次推理能力。\n\n## 任务描述\n\n- **任务类型**：图表理解（描述性 + 推理性）\n- **输入**：科学图表图像 + 问题\n- **输出**：自由格式文本答案\n- **领域**：cs、physics、math、eess、q-bio、q-fin、stat、econ\n\n## 主要特点\n\n- 包含来自 8 个学科 arXiv 论文的 2,323 张真实科学图表\n- 两种问题类型：\n - **描述性**（每张图表 4 个）：基础元素识别（标题、坐标轴、图例、趋势等）\n - **推理型**（每张图表 1 个）：需要综合数据进行高阶推理\n- 19 种描述性问题模板，涵盖信息提取、枚举、模式识别、计数和组合性任务\n- 4 种推理答案类型：图表内文本、通用文本、图表内数值、通用数值\n- 验证集（1,000 张图表）和测试集（1,323 张图表）\n- 依据官方 CharXiv 评分协议，通过 LLM 作为评判者进行评估\n\n## 评估说明\n\n- 默认评估使用 **验证集**（1,000 张图表，5,000 个问题）\n- 每张图表生成 5 个样本：4 个描述性 + 1 个推理型\n- 主要指标：通过 LLM-as-judge 计算的 **准确率（Accuracy）**\n- 子集划分：`descriptive` 和 `reasoning`（也可按类别细分）\n- 需配置 `judge_model_args` 以指定 LLM 评判模型\n- [论文](https://arxiv.org/abs/2406.18521) | [GitHub](https://github.com/princeton-nlp/CharXiv)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `charxiv` |\n| **数据集ID** | [princeton-nlp/CharXiv](https://modelscope.cn/datasets/princeton-nlp/CharXiv/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2406.18521) |\n| **标签** | `MultiModal`, `QA`, `Reasoning` |\n| **指标** | `accuracy` |\n| **默认示例数量** | 0-shot |\n| **评估划分** | `validation` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 5,000 |\n| 提示词长度（平均） | 276.24 字符 |\n| 提示词长度（最小/最大） | 80 / 687 字符 |\n\n**各子集统计数据：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `descriptive` | 4,000 | 261.51 | 156 | 432 |\n| `reasoning` | 1,000 | 335.14 | 80 | 687 |\n\n**图像统计数据：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 5,000 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 1023x139 - 1024x1024 |\n| 格式 | jpeg |\n\n\n## 样例示例\n\n**子集**: `descriptive`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"44ff5b8a\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~70.0KB]\"\n },\n {\n \"text\": \"For the current plot, what is the spatially highest labeled tick on the y-axis?\\n* Your final answer should be the tick value on the y-axis that is explicitly written. Ignore units or scales that are written separately from the tick.\"\n }\n ]\n }\n ],\n \"target\": \"60\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"descriptive\",\n \"metadata\": {\n \"question_type\": \"descriptive\",\n \"question_id\": 7,\n \"category\": \"cs\",\n \"original_id\": \"2004.10956\"\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets charxiv \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['charxiv'],\n dataset_args={\n 'charxiv': {\n # subset_list: ['descriptive', 'reasoning'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "176bf818eb92c85e398b63149e53ec06", + "en": "# CharXiv\n\n\n## Overview\n\nCharXiv is a comprehensive chart understanding benchmark from NeurIPS 2024 that evaluates multimodal\nlarge language models on realistic scientific charts from arXiv papers. It tests both low-level\nchart element perception (descriptive) and high-level reasoning about chart data.\n\n## Task Description\n\n- **Task Type**: Chart Understanding (Descriptive + Reasoning)\n- **Input**: Scientific chart image + question\n- **Output**: Free-form text answer\n- **Domains**: cs, physics, math, eess, q-bio, q-fin, stat, econ\n\n## Key Features\n\n- 2,323 real scientific charts from arXiv papers across 8 disciplines\n- Two question types:\n - **Descriptive** (4 per chart): Basic element identification (titles, axes, legends, trends, etc.)\n - **Reasoning** (1 per chart): Higher-order reasoning requiring data synthesis\n- 19 descriptive question templates covering information extraction, enumeration, pattern recognition, counting, and compositionality\n- 4 reasoning answer types: text-in-chart, text-in-general, number-in-chart, number-in-general\n- Validation set (1,000 charts) and test set (1,323 charts)\n- Evaluation via LLM judge following the official CharXiv grading protocol\n\n## Evaluation Notes\n\n- Default evaluation uses the **validation** split (1,000 charts, 5,000 questions)\n- Each chart yields 5 samples: 4 descriptive + 1 reasoning\n- Primary metric: **Accuracy** via LLM-as-judge\n- Subsets: `descriptive` and `reasoning` (also by category)\n- Requires an LLM judge configured through `judge.models`\n- [Paper](https://arxiv.org/abs/2406.18521) | [GitHub](https://github.com/princeton-nlp/CharXiv)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `charxiv` |\n| **Dataset ID** | [princeton-nlp/CharXiv](https://modelscope.cn/datasets/princeton-nlp/CharXiv/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2406.18521) |\n| **Tags** | `MultiModal`, `QA`, `Reasoning` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `validation` |\n\n\n## Data Statistics\n\n*Statistics not available.*\n\n## Sample Example\n\n**Subset**: `descriptive`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"dd588b4d\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~70.0KB]\"\n },\n {\n \"text\": \"For the current plot, what is the spatially highest labeled tick on the y-axis?\\n* Your final answer should be the tick value on the y-axis that is explicitly written. Ignore units or scales that are written separately from the tick.\"\n }\n ]\n }\n ],\n \"target\": \"60\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"descriptive\",\n \"metadata\": {\n \"question_type\": \"descriptive\",\n \"question_id\": 7,\n \"category\": \"cs\",\n \"original_id\": \"2004.10956\"\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets charxiv \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['charxiv'],\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# CharXiv\n\n\n## 概述\n\nCharXiv 是 NeurIPS 2024 提出的一个全面的图表理解基准测试，用于评估多模态大语言模型在来自 arXiv 论文的真实科学图表上的表现。该基准同时考察模型对图表元素的底层感知能力（描述性任务）和对图表数据的高层推理能力。\n\n## 任务描述\n\n- **任务类型**：图表理解（描述性 + 推理性）\n- **输入**：科学图表图像 + 问题\n- **输出**：自由格式文本答案\n- **领域**：cs、physics、math、eess、q-bio、q-fin、stat、econ\n\n## 主要特点\n\n- 包含来自 8 个学科 arXiv 论文的 2,323 张真实科学图表\n- 两种问题类型：\n - **描述性**（每张图表 4 个）：基础元素识别（标题、坐标轴、图例、趋势等）\n - **推理型**（每张图表 1 个）：需要数据综合的高阶推理\n- 19 种描述性问题模板，涵盖信息提取、枚举、模式识别、计数和组合性任务\n- 4 种推理答案类型：图表内文本、通用文本、图表内数值、通用数值\n- 验证集（1,000 张图表）和测试集（1,323 张图表）\n- 依据官方 CharXiv 评分协议，通过 LLM 作为评判器进行评估\n\n## 评估说明\n\n- 默认评估使用 **验证集**（1,000 张图表，5,000 个问题）\n- 每张图表生成 5 个样本：4 个描述性 + 1 个推理型\n- 主要指标：通过 LLM-as-judge 计算的 **准确率（Accuracy）**\n- 子集划分：`descriptive` 和 `reasoning`（也可按类别细分）\n- 需通过 `judge.models` 配置 LLM 评判器\n- [论文](https://arxiv.org/abs/2406.18521) | [GitHub](https://github.com/princeton-nlp/CharXiv)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `charxiv` |\n| **数据集ID** | [princeton-nlp/CharXiv](https://modelscope.cn/datasets/princeton-nlp/CharXiv/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2406.18521) |\n| **标签** | `MultiModal`, `QA`, `Reasoning` |\n| **指标** | `accuracy` |\n| **默认示例数量** | 0-shot |\n| **评估划分** | `validation` |\n\n\n## 数据统计\n\n*统计数据暂不可用。*\n\n## 样例示例\n\n**子集**: `descriptive`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"dd588b4d\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~70.0KB]\"\n },\n {\n \"text\": \"For the current plot, what is the spatially highest labeled tick on the y-axis?\\n* Your final answer should be the tick value on the y-axis that is explicitly written. Ignore units or scales that are written separately from the tick.\"\n }\n ]\n }\n ],\n \"target\": \"60\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"descriptive\",\n \"metadata\": {\n \"question_type\": \"descriptive\",\n \"question_id\": 7,\n \"category\": \"cs\",\n \"original_id\": \"2004.10956\"\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 通过 CLI 使用\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets charxiv \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 通过 Python 使用\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['charxiv'],\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "f6c394b74b44a3135892720b0ec1b468", "needs_translation": false }, - "updated_at": "2026-08-11T15:30:53.503441", - "translation_updated_at": "2026-07-03T16:19:29" + "updated_at": "2026-08-20T10:40:25.306713", + "translation_updated_at": "2026-08-20T10:41:18" } diff --git a/evalscope/benchmarks/_meta/count_qa.json b/evalscope/benchmarks/_meta/count_qa.json index 35f8f8938..5f5fbc296 100644 --- a/evalscope/benchmarks/_meta/count_qa.json +++ b/evalscope/benchmarks/_meta/count_qa.json @@ -18,7 +18,7 @@ "subset_list": [ "default" ], - "description": "\n## Overview\n\nCountQA probes object counting, a basic perceptual skill that multimodal models are largely\nunevaluated on. Its images were hand-captured in everyday environments and deliberately feature\nhigh object density, clutter and occlusion, so counting cannot be solved by detecting a handful of\nwell-separated objects.\n\n## Task Description\n\n- **Task Type**: Free-form Visual Question Answering (object counting)\n- **Input**: A real-world photograph + a counting question (e.g. \"How many jackets are there?\")\n- **Output**: A single integer\n- **Domain**: Everyday scenes — groceries, kitchenware, tools, clothing, office and outdoor objects\n\n## Key Features\n\n- 1,528 question-answer pairs over 1,001 images; an image may carry several questions\n- Ground-truth counts were annotated *in situ* during capture rather than post-hoc, and range from 0 to 400\n- Questions include compositional ones that require summing over several object types\n- Roughly half the images are cluttered rather than focused on a single subject (recorded as\n ``is_focused`` in each sample's metadata), and scene categories are recorded as ``categories``\n\n## Evaluation Notes\n\n- Default evaluation uses the **test** split as a single subset\n- Primary metric: **Accuracy** (`accuracy`) — Exact Match against the ground-truth integer\n- Secondary metric: **relaxed_acc** — the paper's Relaxed Accuracy, counting a prediction correct\n when it is within 5% of the ground truth\n- The paper's system prompt is used as-is; it constrains the reply to a bare integer\n- Answer parsing takes the reply if it is already an integer, otherwise its first integer — the\n rule the paper states for its rewriter LLM. A reply with no digit scores 0, so `max_tokens` must\n leave the model room to reach its answer; a model that narrates its count (\"row 1 has 3 ...\") is\n scored on the first number it mentions rather than on its stated total\n- Scoring is deterministic arithmetic and needs no LLM judge: keep `judge_strategy` at `rule` or\n `auto`, since `llm` replaces both metrics with a generic judge score. To read a different number\n out of a model that ignores the output format, prepend a per-run filter such as\n `filters={'regex': {'regex_pattern': '(\\d+)', 'group_select': -1}}` (last number) via\n `dataset_args` rather than editing the adapter\n- [Paper](https://arxiv.org/abs/2508.06585)\n", + "description": "\n## Overview\n\nCountQA probes object counting, a basic perceptual skill that multimodal models are largely\nunevaluated on. Its images were hand-captured in everyday environments and deliberately feature\nhigh object density, clutter and occlusion, so counting cannot be solved by detecting a handful of\nwell-separated objects.\n\n## Task Description\n\n- **Task Type**: Free-form Visual Question Answering (object counting)\n- **Input**: A real-world photograph + a counting question (e.g. \"How many jackets are there?\")\n- **Output**: A single integer\n- **Domain**: Everyday scenes — groceries, kitchenware, tools, clothing, office and outdoor objects\n\n## Key Features\n\n- 1,528 question-answer pairs over 1,001 images; an image may carry several questions\n- Ground-truth counts were annotated *in situ* during capture rather than post-hoc, and range from 0 to 400\n- Questions include compositional ones that require summing over several object types\n- Roughly half the images are cluttered rather than focused on a single subject (recorded as\n ``is_focused`` in each sample's metadata), and scene categories are recorded as ``categories``\n\n## Evaluation Notes\n\n- Default evaluation uses the **test** split as a single subset\n- Primary metric: **Accuracy** (`accuracy`) — Exact Match against the ground-truth integer\n- Secondary metric: **relaxed_acc** — the paper's Relaxed Accuracy, counting a prediction correct\n when it is within 5% of the ground truth\n- The paper's system prompt is used as-is; it constrains the reply to a bare integer\n- Answer parsing takes the reply if it is already an integer, otherwise its first integer — the\n rule the paper states for its rewriter LLM. A reply with no digit scores 0, so `max_tokens` must\n leave the model room to reach its answer; a model that narrates its count (\"row 1 has 3 ...\") is\n scored on the first number it mentions rather than on its stated total\n- Scoring is deterministic arithmetic and needs no LLM judge: keep `judge.strategy` at `rule` or\n `auto`, since `llm` replaces both metrics with a generic judge score. To read a different number\n out of a model that ignores the output format, prepend a per-run filter such as\n `filters={'regex': {'regex_pattern': '(\\d+)', 'group_select': -1}}` (last number) via\n `dataset_args` rather than editing the adapter\n- [Paper](https://arxiv.org/abs/2508.06585)\n", "prompt_template": "", "system_prompt": "You are a helpful assistant that counts the number of items in an image. The user will provide an image and ask a question about the number of a certain type of item in the image. If the user question is referring to multiple objects, it means that you need to provide a sum of the number of items. You will count the number of items and return the number as an integer. Your output should STRICTLY be a single integer and nothing else.", "few_shot_prompt_template": "", @@ -153,11 +153,11 @@ "truncated": false }, "readme": { - "en": "# CountQA\n\n\n## Overview\n\nCountQA probes object counting, a basic perceptual skill that multimodal models are largely\nunevaluated on. Its images were hand-captured in everyday environments and deliberately feature\nhigh object density, clutter and occlusion, so counting cannot be solved by detecting a handful of\nwell-separated objects.\n\n## Task Description\n\n- **Task Type**: Free-form Visual Question Answering (object counting)\n- **Input**: A real-world photograph + a counting question (e.g. \"How many jackets are there?\")\n- **Output**: A single integer\n- **Domain**: Everyday scenes — groceries, kitchenware, tools, clothing, office and outdoor objects\n\n## Key Features\n\n- 1,528 question-answer pairs over 1,001 images; an image may carry several questions\n- Ground-truth counts were annotated *in situ* during capture rather than post-hoc, and range from 0 to 400\n- Questions include compositional ones that require summing over several object types\n- Roughly half the images are cluttered rather than focused on a single subject (recorded as\n ``is_focused`` in each sample's metadata), and scene categories are recorded as ``categories``\n\n## Evaluation Notes\n\n- Default evaluation uses the **test** split as a single subset\n- Primary metric: **Accuracy** (`accuracy`) — Exact Match against the ground-truth integer\n- Secondary metric: **relaxed_acc** — the paper's Relaxed Accuracy, counting a prediction correct\n when it is within 5% of the ground truth\n- The paper's system prompt is used as-is; it constrains the reply to a bare integer\n- Answer parsing takes the reply if it is already an integer, otherwise its first integer — the\n rule the paper states for its rewriter LLM. A reply with no digit scores 0, so `max_tokens` must\n leave the model room to reach its answer; a model that narrates its count (\"row 1 has 3 ...\") is\n scored on the first number it mentions rather than on its stated total\n- Scoring is deterministic arithmetic and needs no LLM judge: keep `judge_strategy` at `rule` or\n `auto`, since `llm` replaces both metrics with a generic judge score. To read a different number\n out of a model that ignores the output format, prepend a per-run filter such as\n `filters={'regex': {'regex_pattern': '(\\d+)', 'group_select': -1}}` (last number) via\n `dataset_args` rather than editing the adapter\n- [Paper](https://arxiv.org/abs/2508.06585)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `count_qa` |\n| **Dataset ID** | [evalscope/CountQA](https://modelscope.cn/datasets/evalscope/CountQA/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2508.06585) |\n| **Tags** | `MultiModal`, `QA`, `Reasoning` |\n| **Metrics** | `accuracy`, `relaxed_acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 1,528 |\n| Prompt Length (Mean) | 467.52 chars |\n| Prompt Length (Min/Max) | 459 / 508 chars |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 1,528 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 1714x178 - 2482x2500 |\n| Formats | jpeg, png |\n\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"dd3eebdf\",\n \"content\": \"You are a helpful assistant that counts the number of items in an image. The user will provide an image and ask a question about the number of a certain type of item in the image. If the user question is referring to multiple objects, it means that you need to provide a sum of the number of items. You will count the number of items and return the number as an integer. Your output should STRICTLY be a single integer and nothing else.\"\n },\n {\n \"id\": \"ba0d838c\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~202.2KB]\"\n },\n {\n \"text\": \"How many tiles are on the wall with the shower?\"\n }\n ]\n }\n ],\n \"target\": \"18\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"is_focused\": false,\n \"categories\": [\n \"Outdoor, Vehicles & Structural\"\n ]\n }\n}\n```\n\n## Prompt Template\n\n**System Prompt:**\n```text\nYou are a helpful assistant that counts the number of items in an image. The user will provide an image and ask a question about the number of a certain type of item in the image. If the user question is referring to multiple objects, it means that you need to provide a sum of the number of items. You will count the number of items and return the number as an integer. Your output should STRICTLY be a single integer and nothing else.\n```\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets count_qa \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['count_qa'],\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# CountQA\n\n\n## 概述\n\nCountQA 探究物体计数这一基本感知能力，而多模态模型在此方面大多未经充分评估。该数据集的图像均在日常环境中手工拍摄，并刻意包含高密度物体、杂乱背景和遮挡情况，使得仅靠检测少量分离良好的物体无法完成计数任务。\n\n## 任务描述\n\n- **任务类型**：自由形式视觉问答（物体计数）\n- **输入**：一张真实世界照片 + 一个计数问题（例如：“有多少件夹克？”）\n- **输出**：一个整数\n- **领域**：日常场景——食品杂货、厨房用具、工具、衣物、办公及户外物品\n\n## 关键特性\n\n- 包含 1,528 个问答对，覆盖 1,001 张图像；每张图像可能对应多个问题\n- 真实计数值在拍摄过程中现场标注（而非事后标注），范围从 0 到 400\n- 问题包括组合型问题，需对多种物体类型分别计数后求和\n- 约一半图像属于杂乱场景而非聚焦于单一主体（每个样本元数据中标记为 ``is_focused``），场景类别记录在 ``categories`` 字段中\n\n## 评估说明\n\n- 默认评估使用 **test** 划分作为单一子集\n- 主要指标：**Accuracy** (`accuracy`) —— 与真实整数值完全匹配\n- 次要指标：**relaxed_acc** —— 论文中定义的宽松准确率，当预测值在真实值的 ±5% 范围内即视为正确\n- 使用论文中的系统提示词原样不变；该提示词强制模型仅输出纯整数\n- 答案解析规则：若回复本身是整数则直接采用；否则取其中第一个整数——此规则与论文中用于重写答案的大语言模型一致。若回复不含任何数字，则得分为 0，因此 `max_tokens` 必须为模型留出足够空间输出答案；若模型以叙述方式计数（如“第一行有 3 个……”），则以其提到的第一个数字评分，而非其最终陈述的总数\n- 评分基于确定性算术规则，无需大语言模型裁判：请保持 `judge_strategy` 为 `rule` 或 `auto`，因为 `llm` 会用通用裁判分数替代上述两个指标。若需从忽略输出格式的模型回复中提取不同数字，可通过 `dataset_args` 添加运行时过滤器（例如 `filters={'regex': {'regex_pattern': '(\\d+)', 'group_select': -1}}` 提取最后一个数字），而非修改适配器\n\n- [论文](https://arxiv.org/abs/2508.06585)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `count_qa` |\n| **数据集ID** | [evalscope/CountQA](https://modelscope.cn/datasets/evalscope/CountQA/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2508.06585) |\n| **标签** | `MultiModal`, `QA`, `Reasoning` |\n| **指标** | `accuracy`, `relaxed_acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 1,528 |\n| 提示词长度（平均） | 467.52 字符 |\n| 提示词长度（最小/最大） | 459 / 508 字符 |\n\n**图像统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 总图像数 | 1,528 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 1714x178 - 2482x2500 |\n| 格式 | jpeg, png |\n\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"dd3eebdf\",\n \"content\": \"You are a helpful assistant that counts the number of items in an image. The user will provide an image and ask a question about the number of a certain type of item in the image. If the user question is referring to multiple objects, it means that you need to provide a sum of the number of items. You will count the number of items and return the number as an integer. Your output should STRICTLY be a single integer and nothing else.\"\n },\n {\n \"id\": \"ba0d838c\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~202.2KB]\"\n },\n {\n \"text\": \"How many tiles are on the wall with the shower?\"\n }\n ]\n }\n ],\n \"target\": \"18\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"is_focused\": false,\n \"categories\": [\n \"Outdoor, Vehicles & Structural\"\n ]\n }\n}\n```\n\n## 提示模板\n\n**系统提示词：**\n```text\nYou are a helpful assistant that counts the number of items in an image. The user will provide an image and ask a question about the number of a certain type of item in the image. If the user question is referring to multiple objects, it means that you need to provide a sum of the number of items. You will count the number of items and return the number as an integer. Your output should STRICTLY be a single integer and nothing else.\n```\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets count_qa \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['count_qa'],\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "f16b2b7bb76d0afe8484cd24b41f7108", + "en": "# CountQA\n\n\n## Overview\n\nCountQA probes object counting, a basic perceptual skill that multimodal models are largely\nunevaluated on. Its images were hand-captured in everyday environments and deliberately feature\nhigh object density, clutter and occlusion, so counting cannot be solved by detecting a handful of\nwell-separated objects.\n\n## Task Description\n\n- **Task Type**: Free-form Visual Question Answering (object counting)\n- **Input**: A real-world photograph + a counting question (e.g. \"How many jackets are there?\")\n- **Output**: A single integer\n- **Domain**: Everyday scenes — groceries, kitchenware, tools, clothing, office and outdoor objects\n\n## Key Features\n\n- 1,528 question-answer pairs over 1,001 images; an image may carry several questions\n- Ground-truth counts were annotated *in situ* during capture rather than post-hoc, and range from 0 to 400\n- Questions include compositional ones that require summing over several object types\n- Roughly half the images are cluttered rather than focused on a single subject (recorded as\n ``is_focused`` in each sample's metadata), and scene categories are recorded as ``categories``\n\n## Evaluation Notes\n\n- Default evaluation uses the **test** split as a single subset\n- Primary metric: **Accuracy** (`accuracy`) — Exact Match against the ground-truth integer\n- Secondary metric: **relaxed_acc** — the paper's Relaxed Accuracy, counting a prediction correct\n when it is within 5% of the ground truth\n- The paper's system prompt is used as-is; it constrains the reply to a bare integer\n- Answer parsing takes the reply if it is already an integer, otherwise its first integer — the\n rule the paper states for its rewriter LLM. A reply with no digit scores 0, so `max_tokens` must\n leave the model room to reach its answer; a model that narrates its count (\"row 1 has 3 ...\") is\n scored on the first number it mentions rather than on its stated total\n- Scoring is deterministic arithmetic and needs no LLM judge: keep `judge.strategy` at `rule` or\n `auto`, since `llm` replaces both metrics with a generic judge score. To read a different number\n out of a model that ignores the output format, prepend a per-run filter such as\n `filters={'regex': {'regex_pattern': '(\\d+)', 'group_select': -1}}` (last number) via\n `dataset_args` rather than editing the adapter\n- [Paper](https://arxiv.org/abs/2508.06585)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `count_qa` |\n| **Dataset ID** | [evalscope/CountQA](https://modelscope.cn/datasets/evalscope/CountQA/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2508.06585) |\n| **Tags** | `MultiModal`, `QA`, `Reasoning` |\n| **Metrics** | `accuracy`, `relaxed_acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n*Statistics not available.*\n\n## Sample Example\n\n*Sample example not available.*\n\n## Prompt Template\n\n**System Prompt:**\n```text\nYou are a helpful assistant that counts the number of items in an image. The user will provide an image and ask a question about the number of a certain type of item in the image. If the user question is referring to multiple objects, it means that you need to provide a sum of the number of items. You will count the number of items and return the number as an integer. Your output should STRICTLY be a single integer and nothing else.\n```\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets count_qa \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['count_qa'],\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# CountQA\n\n\n## 概述\n\nCountQA 用于测试物体计数能力，这是一种基础的感知技能，而多模态模型在此方面基本未经过充分评估。该基准的数据集图像均在日常环境中手工拍摄，并刻意包含高密度物体、杂乱背景和遮挡情况，使得仅靠检测少量分离良好的物体无法完成计数任务。\n\n## 任务描述\n\n- **任务类型**：自由形式视觉问答（物体计数）\n- **输入**：一张真实世界照片 + 一个计数问题（例如：“有多少件夹克？”）\n- **输出**：一个整数\n- **领域**：日常场景——包括食品杂货、厨房用具、工具、衣物、办公用品及户外物品\n\n## 主要特点\n\n- 包含 1,528 个问答对，覆盖 1,001 张图像；每张图像可能对应多个问题\n- 真实计数值在拍摄过程中现场标注（而非事后标注），范围从 0 到 400\n- 问题中包含组合型问题，需对多种物体类型分别计数后求和\n- 约一半图像属于杂乱场景而非聚焦于单一主体（每个样本元数据中标记为 ``is_focused``），场景类别记录在 ``categories`` 字段中\n\n## 评估说明\n\n- 默认评估使用 **test** 划分作为单一子集\n- 主要指标：**Accuracy** (`accuracy`) —— 与真实整数值完全匹配\n- 次要指标：**relaxed_acc** —— 论文中定义的宽松准确率，当预测值在真实值的 ±5% 范围内即视为正确\n- 使用论文中的系统提示词原样不变；该提示强制模型仅输出一个纯整数\n- 答案解析规则：若回复本身是整数则直接采用；否则提取其中第一个整数——此规则与论文中重写器 LLM 所用规则一致。若回复中不含任何数字，则得分为 0，因此 `max_tokens` 必须为模型留出足够空间输出答案；若模型以叙述方式计数（如“第一行有 3 个……”），则以其提到的第一个数字评分，而非其最终陈述的总数\n- 评分基于确定性算术运算，无需 LLM 评判器：请保持 `judge.strategy` 为 `rule` 或 `auto`，因为若设为 `llm` 将会用通用评判分数替代上述两个指标。若需从忽略输出格式的模型回复中提取不同数字，可通过 `dataset_args` 添加运行时过滤器（例如 `filters={'regex': {'regex_pattern': '(\\d+)', 'group_select': -1}}` 提取最后一个数字），而非修改适配器\n\n- [论文](https://arxiv.org/abs/2508.06585)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `count_qa` |\n| **数据集ID** | [evalscope/CountQA](https://modelscope.cn/datasets/evalscope/CountQA/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2508.06585) |\n| **标签** | `MultiModal`, `QA`, `Reasoning` |\n| **指标** | `accuracy`, `relaxed_acc` |\n| **默认示例数量** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n*统计数据暂不可用。*\n\n## 样例示例\n\n*样例示例暂不可用。*\n\n## 提示模板\n\n**系统提示词：**\n```text\nYou are a helpful assistant that counts the number of items in an image. The user will provide an image and ask a question about the number of a certain type of item in the image. If the user question is referring to multiple objects, it means that you need to provide a sum of the number of items. You will count the number of items and return the number as an integer. Your output should STRICTLY be a single integer and nothing else.\n```\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets count_qa \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['count_qa'],\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "9048e150bfd13b414f6d3a32f1a176b7", "needs_translation": false }, - "updated_at": "2026-08-18T10:33:50.713107", - "translation_updated_at": "2026-08-18T10:33:55" + "updated_at": "2026-08-20T10:42:50.807926", + "translation_updated_at": "2026-08-20T10:45:37" } diff --git a/evalscope/benchmarks/_meta/hipho.json b/evalscope/benchmarks/_meta/hipho.json index 86c59bd81..271e93a66 100644 --- a/evalscope/benchmarks/_meta/hipho.json +++ b/evalscope/benchmarks/_meta/hipho.json @@ -31,7 +31,7 @@ "PanPhO_2024", "PanPhO_2025" ], - "description": "\n## Overview\n\nHiPhO is the first benchmark dedicated to high school physics Olympiads with human-aligned evaluation. It compiles\n13 recent Olympiad exams (2024-2025) spanning international and regional competitions, with mixed modalities that\nrange from text-only problems to diagram-based problems.\n\n## Task Description\n\n- **Task Type**: Free-form physics problem solving graded against official marking schemes\n- **Input**: A physics problem (constants sheet + context + question), optionally with figures\n- **Output**: A step-by-step solution ending with boxed final answers inside `...`\n- **Modalities**: Text-only and text+figure (illustration / variable / data figures)\n\n## Key Features\n\n- 403 problems across 14 exam papers (IPhO, APhO, EuPhO, NBPhO, PanPhO, PanMechanics, CPhO, F=MA), each exam\n exposed as its own subset.\n- English prompts are used for English exams and Chinese prompts for the Chinese exams (CPhO, PanMechanics),\n following the official language mapping.\n- Two grading regimes reproduced from the paper, dispatched per problem:\n - **Step-level** for problems shipping an official marking scheme: the LLM judge scores every marking criterion\n and the awarded points are summed.\n - **Answer-level** for problems without a marking scheme: boxed final answers are matched against the ground\n truth by a rule-based math check, with an LLM judge as fallback.\n\n## Evaluation Notes\n\n- Requires an LLM judge: run with `judge_strategy='llm'` (or `'auto'`, which enables the judge for this benchmark)\n and provide `judge_model_args`. `judge_strategy='rule'` is not supported.\n- Primary metric: `accuracy`, the per-problem awarded/attainable point ratio in `[0, 1]`, aggregated by mean per subset.\n For step-level problems the attainable maximum is the sum of the marking criteria; for problems with several\n official schemes (EuPhO, NBPhO) the highest-scoring scheme is used, matching the paper.\n- This reports the normalized exam score per exam. It does not compute the paper's gold/silver/bronze medal\n thresholds, which require the raw point totals and official cutoffs.\n- Solutions can be long and figure problems need vision input; give the evaluated model a generous\n `generation_config.max_tokens`. A solution truncated before its `` block yields no boxed answer and\n scores near zero for reasons unrelated to physics ability.\n- Figures are sent inline as base64 and the largest is ~1.5 MB; set `max_image_bytes` in `dataset_args` if the\n served model enforces a smaller per-image limit.\n- Resources: [Paper](https://arxiv.org/abs/2509.07894) | [GitHub](https://github.com/SciYu/HiPhO) |\n [Leaderboard](https://phyarena.github.io/)\n", + "description": "\n## Overview\n\nHiPhO is the first benchmark dedicated to high school physics Olympiads with human-aligned evaluation. It compiles\n13 recent Olympiad exams (2024-2025) spanning international and regional competitions, with mixed modalities that\nrange from text-only problems to diagram-based problems.\n\n## Task Description\n\n- **Task Type**: Free-form physics problem solving graded against official marking schemes\n- **Input**: A physics problem (constants sheet + context + question), optionally with figures\n- **Output**: A step-by-step solution ending with boxed final answers inside `...`\n- **Modalities**: Text-only and text+figure (illustration / variable / data figures)\n\n## Key Features\n\n- 403 problems across 14 exam papers (IPhO, APhO, EuPhO, NBPhO, PanPhO, PanMechanics, CPhO, F=MA), each exam\n exposed as its own subset.\n- English prompts are used for English exams and Chinese prompts for the Chinese exams (CPhO, PanMechanics),\n following the official language mapping.\n- Two grading regimes reproduced from the paper, dispatched per problem:\n - **Step-level** for problems shipping an official marking scheme: the LLM judge scores every marking criterion\n and the awarded points are summed.\n - **Answer-level** for problems without a marking scheme: boxed final answers are matched against the ground\n truth by a rule-based math check, with an LLM judge as fallback.\n\n## Evaluation Notes\n\n- Requires an LLM judge: set `judge.strategy='llm'` (or `'auto'`, which enables the judge for this benchmark)\n and provide `judge.models`. `judge.strategy='rule'` is not supported.\n- Primary metric: `accuracy`, the per-problem awarded/attainable point ratio in `[0, 1]`, aggregated by mean per subset.\n For step-level problems the attainable maximum is the sum of the marking criteria; for problems with several\n official schemes (EuPhO, NBPhO) the highest-scoring scheme is used, matching the paper.\n- This reports the normalized exam score per exam. It does not compute the paper's gold/silver/bronze medal\n thresholds, which require the raw point totals and official cutoffs.\n- Solutions can be long and figure problems need vision input; give the evaluated model a generous\n `generation_config.max_tokens`. A solution truncated before its `` block yields no boxed answer and\n scores near zero for reasons unrelated to physics ability.\n- Figures are sent inline as base64 and the largest is ~1.5 MB; set `max_image_bytes` in `dataset_args` if the\n served model enforces a smaller per-image limit.\n- Resources: [Paper](https://arxiv.org/abs/2509.07894) | [GitHub](https://github.com/SciYu/HiPhO) |\n [Leaderboard](https://phyarena.github.io/)\n", "prompt_template": "", "system_prompt": "", "few_shot_prompt_template": "", @@ -589,7 +589,7 @@ "std": 1917.32 }, "target_length_mean": null, - "computed_at": "2026-08-10T14:27:01.696954+08:00", + "computed_at": "2026-08-20T10:40:06.245919", "multimodal": { "has_images": true, "has_audio": false, @@ -627,7 +627,7 @@ "data": { "input": [ { - "id": "9ec40cee", + "id": "41868500", "content": [ { "text": "You are participating in a high school physics Olympiad exam.\nPlease read the following question carefully and provide a clear, step-by-step solution with full reasoning.\nInstructions:\n1. Use LaTeX to format all variables, equations, and calc ... [TRUNCATED 3334 chars] ... gamma} R^{\\delta}$ \nwhere $G$ is the gravitational constant, and $\\beta, \\gamma$ and $\\delta$ are constant exponents.\nQuestion (Answer only the question stated below):\nFind the values of exponents: (1) $\\beta$, (2) $\\gamma$, and (3) $\\delta$." @@ -668,11 +668,11 @@ "truncated": true }, "readme": { - "en": "# HiPhO\n\n\n## Overview\n\nHiPhO is the first benchmark dedicated to high school physics Olympiads with human-aligned evaluation. It compiles\n13 recent Olympiad exams (2024-2025) spanning international and regional competitions, with mixed modalities that\nrange from text-only problems to diagram-based problems.\n\n## Task Description\n\n- **Task Type**: Free-form physics problem solving graded against official marking schemes\n- **Input**: A physics problem (constants sheet + context + question), optionally with figures\n- **Output**: A step-by-step solution ending with boxed final answers inside `...`\n- **Modalities**: Text-only and text+figure (illustration / variable / data figures)\n\n## Key Features\n\n- 403 problems across 14 exam papers (IPhO, APhO, EuPhO, NBPhO, PanPhO, PanMechanics, CPhO, F=MA), each exam\n exposed as its own subset.\n- English prompts are used for English exams and Chinese prompts for the Chinese exams (CPhO, PanMechanics),\n following the official language mapping.\n- Two grading regimes reproduced from the paper, dispatched per problem:\n - **Step-level** for problems shipping an official marking scheme: the LLM judge scores every marking criterion\n and the awarded points are summed.\n - **Answer-level** for problems without a marking scheme: boxed final answers are matched against the ground\n truth by a rule-based math check, with an LLM judge as fallback.\n\n## Evaluation Notes\n\n- Requires an LLM judge: run with `judge_strategy='llm'` (or `'auto'`, which enables the judge for this benchmark)\n and provide `judge_model_args`. `judge_strategy='rule'` is not supported.\n- Primary metric: `accuracy`, the per-problem awarded/attainable point ratio in `[0, 1]`, aggregated by mean per subset.\n For step-level problems the attainable maximum is the sum of the marking criteria; for problems with several\n official schemes (EuPhO, NBPhO) the highest-scoring scheme is used, matching the paper.\n- This reports the normalized exam score per exam. It does not compute the paper's gold/silver/bronze medal\n thresholds, which require the raw point totals and official cutoffs.\n- Solutions can be long and figure problems need vision input; give the evaluated model a generous\n `generation_config.max_tokens`. A solution truncated before its `` block yields no boxed answer and\n scores near zero for reasons unrelated to physics ability.\n- Figures are sent inline as base64 and the largest is ~1.5 MB; set `max_image_bytes` in `dataset_args` if the\n served model enforces a smaller per-image limit.\n- Resources: [Paper](https://arxiv.org/abs/2509.07894) | [GitHub](https://github.com/SciYu/HiPhO) |\n [Leaderboard](https://phyarena.github.io/)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `hipho` |\n| **Dataset ID** | [evalscope/HiPhO](https://modelscope.cn/datasets/evalscope/HiPhO/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2509.07894) |\n| **Tags** | `Math`, `MultiModal`, `QA`, `Reasoning` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 403 |\n| Prompt Length (Mean) | 3020.35 chars |\n| Prompt Length (Min/Max) | 653 / 9336 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `APhO_2025` | 45 | 4624.02 | 2496 | 8787 |\n| `CPhO_2025` | 43 | 2041.81 | 960 | 3745 |\n| `EuPhO_2024` | 7 | 1924.29 | 1468 | 2051 |\n| `EuPhO_2025` | 6 | 1646.33 | 1422 | 1856 |\n| `F=MA_2024` | 25 | 1598.76 | 1279 | 1957 |\n| `F=MA_2025` | 25 | 1721.2 | 1395 | 2513 |\n| `IPhO_2024` | 37 | 4152.57 | 2201 | 6701 |\n| `IPhO_2025` | 39 | 6359.74 | 3362 | 9336 |\n| `NBPhO_2024` | 24 | 2305.25 | 1317 | 4486 |\n| `NBPhO_2025` | 20 | 2677.7 | 1359 | 4808 |\n| `PanMechanics_2024` | 29 | 878.55 | 653 | 1283 |\n| `PanMechanics_2025` | 23 | 874.87 | 667 | 1150 |\n| `PanPhO_2024` | 33 | 2820.55 | 1448 | 3880 |\n| `PanPhO_2025` | 47 | 3526.47 | 1561 | 6209 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 413 |\n| Images per Sample | min: 1, max: 5, mean: 1.5 |\n| Resolution Range | 456x60 - 3200x1645 |\n| Formats | png |\n\n\n## Sample Example\n\n**Subset**: `APhO_2025`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"9ec40cee\",\n \"content\": [\n {\n \"text\": \"You are participating in a high school physics Olympiad exam.\\nPlease read the following question carefully and provide a clear, step-by-step solution with full reasoning.\\nInstructions:\\n1. Use LaTeX to format all variables, equations, and calc ... [TRUNCATED 3334 chars] ... gamma} R^{\\\\delta}$ \\nwhere $G$ is the gravitational constant, and $\\\\beta, \\\\gamma$ and $\\\\delta$ are constant exponents.\\nQuestion (Answer only the question stated below):\\nFind the values of exponents: (1) $\\\\beta$, (2) $\\\\gamma$, and (3) $\\\\delta$.\"\n },\n {\n \"image\": \"[BASE64_IMAGE: png, ~101.8KB]\"\n }\n ]\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"APhO_2025\",\n \"metadata\": {\n \"id\": \"APhO_2025_1_A_1\",\n \"source\": \"APhO_2025\",\n \"question\": \"Find the values of exponents: (1) $\\\\beta$, (2) $\\\\gamma$, and (3) $\\\\delta$.\",\n \"answers\": [\n \"\\\\boxed{$\\\\beta = 2$}\",\n \"\\\\boxed{$\\\\gamma = -1$}\",\n \"\\\\boxed{$\\\\delta = 4$}\"\n ],\n \"marking\": [\n [\n \"Award 0.2 pt if the answer correctly expresses the dimension of $G$ as $[G] = L^3 M^{-1} T^{-2}$, where $L$ is the base dimensions length, $M$ is mass, and $T$ is time. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer correctly sets up the exponent equation $0 = 2 - \\\\beta$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer correctly sets up the exponent equation $0 = \\\\gamma + 1$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer correctly sets up the exponent equation $1 = \\\\delta - 3$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer obtains the correct value $\\\\beta = 2$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer obtains the correct value $\\\\gamma = -1$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer obtains the correct value $\\\\delta = 4$. Otherwise, award 0 pt.\"\n ]\n ]\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets hipho \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['hipho'],\n dataset_args={\n 'hipho': {\n # subset_list: ['APhO_2025', 'CPhO_2025', 'EuPhO_2024'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# HiPhO\n\n\n## 概述\n\nHiPhO 是首个专注于高中物理奥林匹克竞赛并采用人类对齐评估的基准测试。它汇集了13场近期（2024-2025年）的国际及区域性奥赛试题，涵盖多种模态，包括纯文本题目和基于图表的题目。\n\n## 任务描述\n\n- **任务类型**：自由形式的物理问题求解，依据官方评分标准进行评分\n- **输入**：一道物理题（包含常数表、上下文和问题），可选附带图表\n- **输出**：分步解答，最终答案用 `...` 包裹并放入 `\\boxed{}` 中\n- **模态**：纯文本 和 文本+图表（示意图 / 变量图 / 数据图）\n\n## 核心特性\n\n- 共计403道题目，来自14套试卷（IPhO、APhO、EuPhO、NBPhO、PanPhO、PanMechanics、CPhO、F=MA），每套试卷作为一个独立子集。\n- 英语试卷使用英文提示，中文试卷（CPhO、PanMechanics）使用中文提示，遵循官方语言对应关系。\n- 复现论文中的两种评分机制，按题目分别应用：\n - **步骤级评分**：适用于提供官方评分细则的题目，LLM 评委对每个评分项打分，总分为各项得分之和。\n - **答案级评分**：适用于无评分细则的题目，通过基于规则的数学检查匹配 `\\boxed{}` 中的最终答案；若规则检查失败，则回退至 LLM 评委判断。\n\n## 评估说明\n\n- 需要 LLM 评委：运行时需设置 `judge_strategy='llm'`（或 `'auto'`，该选项会为此基准自动启用评委），并提供 `judge_model_args`。不支持 `judge_strategy='rule'`。\n- 主要指标：`accuracy`，即每道题得分与满分之比（范围 `[0, 1]`），按子集取平均值聚合。\n - 对于步骤级题目，满分为所有评分项分数之和；\n - 对于拥有多个官方评分方案的题目（如 EuPhO、NBPhO），采用得分最高的方案，与论文一致。\n- 报告的是每场考试的标准化得分，**不计算**论文中金/银/铜牌的分数线，因为这需要原始总分和官方截止分数。\n- 解答可能很长，且图表题需要视觉输入；请为被测模型设置较大的 `generation_config.max_tokens`。若解答在 `` 块前被截断，则无法提取答案，得分接近零，但这与物理能力无关。\n- 图表以内联 base64 形式发送，最大约 1.5 MB；若所用模型对单图有更小限制，请在 `dataset_args` 中设置 `max_image_bytes`。\n- 资源链接：[论文](https://arxiv.org/abs/2509.07894) | [GitHub](https://github.com/SciYu/HiPhO) | [排行榜](https://phyarena.github.io/)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `hipho` |\n| **数据集ID** | [evalscope/HiPhO](https://modelscope.cn/datasets/evalscope/HiPhO/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2509.07894) |\n| **标签** | `Math`, `MultiModal`, `QA`, `Reasoning` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估分割** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 403 |\n| 提示词长度（平均） | 3020.35 字符 |\n| 提示词长度（最小/最大） | 653 / 9336 字符 |\n\n**各子集统计：**\n\n| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 |\n|--------|---------|-------------|------------|------------|\n| `APhO_2025` | 45 | 4624.02 | 2496 | 8787 |\n| `CPhO_2025` | 43 | 2041.81 | 960 | 3745 |\n| `EuPhO_2024` | 7 | 1924.29 | 1468 | 2051 |\n| `EuPhO_2025` | 6 | 1646.33 | 1422 | 1856 |\n| `F=MA_2024` | 25 | 1598.76 | 1279 | 1957 |\n| `F=MA_2025` | 25 | 1721.2 | 1395 | 2513 |\n| `IPhO_2024` | 37 | 4152.57 | 2201 | 6701 |\n| `IPhO_2025` | 39 | 6359.74 | 3362 | 9336 |\n| `NBPhO_2024` | 24 | 2305.25 | 1317 | 4486 |\n| `NBPhO_2025` | 20 | 2677.7 | 1359 | 4808 |\n| `PanMechanics_2024` | 29 | 878.55 | 653 | 1283 |\n| `PanMechanics_2025` | 23 | 874.87 | 667 | 1150 |\n| `PanPhO_2024` | 33 | 2820.55 | 1448 | 3880 |\n| `PanPhO_2025` | 47 | 3526.47 | 1561 | 6209 |\n\n**图像统计：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 413 |\n| 每样本图像数 | 最小: 1, 最大: 5, 平均: 1.5 |\n| 分辨率范围 | 456x60 - 3200x1645 |\n| 格式 | png |\n\n\n## 样例示例\n\n**子集**: `APhO_2025`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"9ec40cee\",\n \"content\": [\n {\n \"text\": \"You are participating in a high school physics Olympiad exam.\\nPlease read the following question carefully and provide a clear, step-by-step solution with full reasoning.\\nInstructions:\\n1. Use LaTeX to format all variables, equations, and calc ... [TRUNCATED 3334 chars] ... gamma} R^{\\\\delta}$ \\nwhere $G$ is the gravitational constant, and $\\\\beta, \\\\gamma$ and $\\\\delta$ are constant exponents.\\nQuestion (Answer only the question stated below):\\nFind the values of exponents: (1) $\\\\beta$, (2) $\\\\gamma$, and (3) $\\\\delta$.\"\n },\n {\n \"image\": \"[BASE64_IMAGE: png, ~101.8KB]\"\n }\n ]\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"APhO_2025\",\n \"metadata\": {\n \"id\": \"APhO_2025_1_A_1\",\n \"source\": \"APhO_2025\",\n \"question\": \"Find the values of exponents: (1) $\\\\beta$, (2) $\\\\gamma$, and (3) $\\\\delta$.\",\n \"answers\": [\n \"\\\\boxed{$\\\\beta = 2$}\",\n \"\\\\boxed{$\\\\gamma = -1$}\",\n \"\\\\boxed{$\\\\delta = 4$}\"\n ],\n \"marking\": [\n [\n \"Award 0.2 pt if the answer correctly expresses the dimension of $G$ as $[G] = L^3 M^{-1} T^{-2}$, where $L$ is the base dimensions length, $M$ is mass, and $T$ is time. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer correctly sets up the exponent equation $0 = 2 - \\\\beta$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer correctly sets up the exponent equation $0 = \\\\gamma + 1$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer correctly sets up the exponent equation $1 = \\\\delta - 3$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer obtains the correct value $\\\\beta = 2$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer obtains the correct value $\\\\gamma = -1$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer obtains the correct value $\\\\delta = 4$. Otherwise, award 0 pt.\"\n ]\n ]\n }\n}\n```\n\n*注：部分内容因展示需要已被截断。*\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets hipho \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['hipho'],\n dataset_args={\n 'hipho': {\n # subset_list: ['APhO_2025', 'CPhO_2025', 'EuPhO_2024'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "69b57eb1d1aeac862fa00d6e99881008", + "en": "# HiPhO\n\n\n## Overview\n\nHiPhO is the first benchmark dedicated to high school physics Olympiads with human-aligned evaluation. It compiles\n13 recent Olympiad exams (2024-2025) spanning international and regional competitions, with mixed modalities that\nrange from text-only problems to diagram-based problems.\n\n## Task Description\n\n- **Task Type**: Free-form physics problem solving graded against official marking schemes\n- **Input**: A physics problem (constants sheet + context + question), optionally with figures\n- **Output**: A step-by-step solution ending with boxed final answers inside `...`\n- **Modalities**: Text-only and text+figure (illustration / variable / data figures)\n\n## Key Features\n\n- 403 problems across 14 exam papers (IPhO, APhO, EuPhO, NBPhO, PanPhO, PanMechanics, CPhO, F=MA), each exam\n exposed as its own subset.\n- English prompts are used for English exams and Chinese prompts for the Chinese exams (CPhO, PanMechanics),\n following the official language mapping.\n- Two grading regimes reproduced from the paper, dispatched per problem:\n - **Step-level** for problems shipping an official marking scheme: the LLM judge scores every marking criterion\n and the awarded points are summed.\n - **Answer-level** for problems without a marking scheme: boxed final answers are matched against the ground\n truth by a rule-based math check, with an LLM judge as fallback.\n\n## Evaluation Notes\n\n- Requires an LLM judge: set `judge.strategy='llm'` (or `'auto'`, which enables the judge for this benchmark)\n and provide `judge.models`. `judge.strategy='rule'` is not supported.\n- Primary metric: `accuracy`, the per-problem awarded/attainable point ratio in `[0, 1]`, aggregated by mean per subset.\n For step-level problems the attainable maximum is the sum of the marking criteria; for problems with several\n official schemes (EuPhO, NBPhO) the highest-scoring scheme is used, matching the paper.\n- This reports the normalized exam score per exam. It does not compute the paper's gold/silver/bronze medal\n thresholds, which require the raw point totals and official cutoffs.\n- Solutions can be long and figure problems need vision input; give the evaluated model a generous\n `generation_config.max_tokens`. A solution truncated before its `` block yields no boxed answer and\n scores near zero for reasons unrelated to physics ability.\n- Figures are sent inline as base64 and the largest is ~1.5 MB; set `max_image_bytes` in `dataset_args` if the\n served model enforces a smaller per-image limit.\n- Resources: [Paper](https://arxiv.org/abs/2509.07894) | [GitHub](https://github.com/SciYu/HiPhO) |\n [Leaderboard](https://phyarena.github.io/)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `hipho` |\n| **Dataset ID** | [evalscope/HiPhO](https://modelscope.cn/datasets/evalscope/HiPhO/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2509.07894) |\n| **Tags** | `Math`, `MultiModal`, `QA`, `Reasoning` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 403 |\n| Prompt Length (Mean) | 3020.35 chars |\n| Prompt Length (Min/Max) | 653 / 9336 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `APhO_2025` | 45 | 4624.02 | 2496 | 8787 |\n| `CPhO_2025` | 43 | 2041.81 | 960 | 3745 |\n| `EuPhO_2024` | 7 | 1924.29 | 1468 | 2051 |\n| `EuPhO_2025` | 6 | 1646.33 | 1422 | 1856 |\n| `F=MA_2024` | 25 | 1598.76 | 1279 | 1957 |\n| `F=MA_2025` | 25 | 1721.2 | 1395 | 2513 |\n| `IPhO_2024` | 37 | 4152.57 | 2201 | 6701 |\n| `IPhO_2025` | 39 | 6359.74 | 3362 | 9336 |\n| `NBPhO_2024` | 24 | 2305.25 | 1317 | 4486 |\n| `NBPhO_2025` | 20 | 2677.7 | 1359 | 4808 |\n| `PanMechanics_2024` | 29 | 878.55 | 653 | 1283 |\n| `PanMechanics_2025` | 23 | 874.87 | 667 | 1150 |\n| `PanPhO_2024` | 33 | 2820.55 | 1448 | 3880 |\n| `PanPhO_2025` | 47 | 3526.47 | 1561 | 6209 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 413 |\n| Images per Sample | min: 1, max: 5, mean: 1.5 |\n| Resolution Range | 456x60 - 3200x1645 |\n| Formats | png |\n\n\n## Sample Example\n\n**Subset**: `APhO_2025`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"41868500\",\n \"content\": [\n {\n \"text\": \"You are participating in a high school physics Olympiad exam.\\nPlease read the following question carefully and provide a clear, step-by-step solution with full reasoning.\\nInstructions:\\n1. Use LaTeX to format all variables, equations, and calc ... [TRUNCATED 3334 chars] ... gamma} R^{\\\\delta}$ \\nwhere $G$ is the gravitational constant, and $\\\\beta, \\\\gamma$ and $\\\\delta$ are constant exponents.\\nQuestion (Answer only the question stated below):\\nFind the values of exponents: (1) $\\\\beta$, (2) $\\\\gamma$, and (3) $\\\\delta$.\"\n },\n {\n \"image\": \"[BASE64_IMAGE: png, ~101.8KB]\"\n }\n ]\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"APhO_2025\",\n \"metadata\": {\n \"id\": \"APhO_2025_1_A_1\",\n \"source\": \"APhO_2025\",\n \"question\": \"Find the values of exponents: (1) $\\\\beta$, (2) $\\\\gamma$, and (3) $\\\\delta$.\",\n \"answers\": [\n \"\\\\boxed{$\\\\beta = 2$}\",\n \"\\\\boxed{$\\\\gamma = -1$}\",\n \"\\\\boxed{$\\\\delta = 4$}\"\n ],\n \"marking\": [\n [\n \"Award 0.2 pt if the answer correctly expresses the dimension of $G$ as $[G] = L^3 M^{-1} T^{-2}$, where $L$ is the base dimensions length, $M$ is mass, and $T$ is time. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer correctly sets up the exponent equation $0 = 2 - \\\\beta$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer correctly sets up the exponent equation $0 = \\\\gamma + 1$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer correctly sets up the exponent equation $1 = \\\\delta - 3$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer obtains the correct value $\\\\beta = 2$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer obtains the correct value $\\\\gamma = -1$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer obtains the correct value $\\\\delta = 4$. Otherwise, award 0 pt.\"\n ]\n ]\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets hipho \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['hipho'],\n dataset_args={\n 'hipho': {\n # subset_list: ['APhO_2025', 'CPhO_2025', 'EuPhO_2024'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# HiPhO\n\n\n## 概述\n\nHiPhO 是首个专注于高中物理奥林匹克竞赛并采用人类对齐评估的基准测试。它汇集了 13 场近期（2024–2025 年）的国际及区域性奥赛试题，涵盖多种模态，包括纯文本题目和基于图表的题目。\n\n## 任务描述\n\n- **任务类型**：自由形式的物理问题求解，依据官方评分标准进行评分\n- **输入**：一道物理题（包含常数表、上下文和问题），可选附带图表\n- **输出**：逐步推导的解答，最终答案用 `...` 包裹并以 `\\boxed{}` 标注\n- **模态**：纯文本，以及文本+图表（示意图 / 变量图 / 数据图）\n\n## 主要特性\n\n- 共计 403 道题目，来自 14 套试卷（IPhO、APhO、EuPhO、NBPhO、PanPhO、PanMechanics、CPhO、F=MA），每套试卷作为一个独立子集。\n- 英文试卷使用英文提示，中文试卷（CPhO、PanMechanics）使用中文提示，遵循官方语言对应关系。\n- 复现论文中的两种评分机制，按题目分别应用：\n - **步骤级评分**：适用于提供官方评分标准的题目，LLM 评委会对每个评分项打分，并将得分累加。\n - **答案级评分**：适用于无评分标准的题目，通过基于规则的数学检查匹配 `\\boxed{}` 中的最终答案；若规则检查失败，则由 LLM 评委会作为后备方案。\n\n## 评估说明\n\n- 需要 LLM 评委会：设置 `judge.strategy='llm'`（或 `'auto'`，该选项会为此基准自动启用评委会），并提供 `judge.models`。不支持 `judge.strategy='rule'`。\n- 主要指标：`accuracy`，即每道题得分与满分之比（范围 `[0, 1]`），按子集取平均值聚合。\n - 对于步骤级题目，满分为各评分项分数之和；\n - 对于拥有多个官方评分方案的题目（如 EuPhO、NBPhO），采用得分最高的方案，与论文一致。\n- 本基准报告的是每场考试的标准化得分，**不计算**论文中金/银/铜牌的分数线，后者需要原始总分和官方截断值。\n- 解答可能很长，且图表题需要视觉输入；请为被测模型设置较大的 `generation_config.max_tokens`。若解答在 `` 块前被截断，则无法提取最终答案，导致得分接近零，这与模型的物理能力无关。\n- 图表以内联 base64 形式发送，最大约 1.5 MB；若所用模型对单图有更小的限制，请在 `dataset_args` 中设置 `max_image_bytes`。\n- 资源链接：[论文](https://arxiv.org/abs/2509.07894) | [GitHub](https://github.com/SciYu/HiPhO) | [排行榜](https://phyarena.github.io/)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `hipho` |\n| **数据集ID** | [evalscope/HiPhO](https://modelscope.cn/datasets/evalscope/HiPhO/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2509.07894) |\n| **标签** | `Math`, `MultiModal`, `QA`, `Reasoning` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估分割** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 403 |\n| 提示词长度（平均） | 3020.35 字符 |\n| 提示词长度（最小/最大） | 653 / 9336 字符 |\n\n**各子集统计：**\n\n| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 |\n|--------|---------|-------------|------------|------------|\n| `APhO_2025` | 45 | 4624.02 | 2496 | 8787 |\n| `CPhO_2025` | 43 | 2041.81 | 960 | 3745 |\n| `EuPhO_2024` | 7 | 1924.29 | 1468 | 2051 |\n| `EuPhO_2025` | 6 | 1646.33 | 1422 | 1856 |\n| `F=MA_2024` | 25 | 1598.76 | 1279 | 1957 |\n| `F=MA_2025` | 25 | 1721.2 | 1395 | 2513 |\n| `IPhO_2024` | 37 | 4152.57 | 2201 | 6701 |\n| `IPhO_2025` | 39 | 6359.74 | 3362 | 9336 |\n| `NBPhO_2024` | 24 | 2305.25 | 1317 | 4486 |\n| `NBPhO_2025` | 20 | 2677.7 | 1359 | 4808 |\n| `PanMechanics_2024` | 29 | 878.55 | 653 | 1283 |\n| `PanMechanics_2025` | 23 | 874.87 | 667 | 1150 |\n| `PanPhO_2024` | 33 | 2820.55 | 1448 | 3880 |\n| `PanPhO_2025` | 47 | 3526.47 | 1561 | 6209 |\n\n**图像统计：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 413 |\n| 每样本图像数 | 最小: 1, 最大: 5, 平均: 1.5 |\n| 分辨率范围 | 456x60 - 3200x1645 |\n| 格式 | png |\n\n\n## 样例示例\n\n**子集**: `APhO_2025`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"41868500\",\n \"content\": [\n {\n \"text\": \"You are participating in a high school physics Olympiad exam.\\nPlease read the following question carefully and provide a clear, step-by-step solution with full reasoning.\\nInstructions:\\n1. Use LaTeX to format all variables, equations, and calc ... [TRUNCATED 3334 chars] ... gamma} R^{\\\\delta}$ \\nwhere $G$ is the gravitational constant, and $\\\\beta, \\\\gamma$ and $\\\\delta$ are constant exponents.\\nQuestion (Answer only the question stated below):\\nFind the values of exponents: (1) $\\\\beta$, (2) $\\\\gamma$, and (3) $\\\\delta$.\"\n },\n {\n \"image\": \"[BASE64_IMAGE: png, ~101.8KB]\"\n }\n ]\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"APhO_2025\",\n \"metadata\": {\n \"id\": \"APhO_2025_1_A_1\",\n \"source\": \"APhO_2025\",\n \"question\": \"Find the values of exponents: (1) $\\\\beta$, (2) $\\\\gamma$, and (3) $\\\\delta$.\",\n \"answers\": [\n \"\\\\boxed{$\\\\beta = 2$}\",\n \"\\\\boxed{$\\\\gamma = -1$}\",\n \"\\\\boxed{$\\\\delta = 4$}\"\n ],\n \"marking\": [\n [\n \"Award 0.2 pt if the answer correctly expresses the dimension of $G$ as $[G] = L^3 M^{-1} T^{-2}$, where $L$ is the base dimensions length, $M$ is mass, and $T$ is time. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer correctly sets up the exponent equation $0 = 2 - \\\\beta$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer correctly sets up the exponent equation $0 = \\\\gamma + 1$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer correctly sets up the exponent equation $1 = \\\\delta - 3$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer obtains the correct value $\\\\beta = 2$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer obtains the correct value $\\\\gamma = -1$. Otherwise, award 0 pt.\",\n \"Award 0.1 pt if the answer obtains the correct value $\\\\delta = 4$. Otherwise, award 0 pt.\"\n ]\n ]\n }\n}\n```\n\n*注：部分内容为显示目的已截断。*\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets hipho \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['hipho'],\n dataset_args={\n 'hipho': {\n # subset_list: ['APhO_2025', 'CPhO_2025', 'EuPhO_2024'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "088bfcc1c0c4050403ee30e8e67c38de", "needs_translation": false }, - "updated_at": "2026-08-11T16:15:42.664229", - "translation_updated_at": "2026-08-10T14:27:06+08:00" + "updated_at": "2026-08-20T10:40:09.263339", + "translation_updated_at": "2026-08-20T10:40:30" } diff --git a/evalscope/benchmarks/_meta/job_bench.json b/evalscope/benchmarks/_meta/job_bench.json index 982aa7062..fb7628cb6 100644 --- a/evalscope/benchmarks/_meta/job_bench.json +++ b/evalscope/benchmarks/_meta/job_bench.json @@ -13,18 +13,13 @@ "pass_rate", "judge_score" ], - "primary_metric": { - "name": "normalized_score", - "aggregation": null, - "dimensions": {} - }, "few_shot_num": 0, "eval_split": "main", "train_split": "", "subset_list": [ "default" ], - "description": "\n## Overview\n\nJobBench evaluates agentic systems on realistic professional work tasks that require reading reference files, producing\ndeliverables, and reconciling multi-source information. This adapter uses the ModelScope dataset\n`evalscope/job-bench`.\n\n## Task Description\n\n- **Task Type**: Agentic professional work / deliverable generation\n- **Input**: Workplace-style task prompt with optional reference files under `reference_files/`\n- **Output**: Final deliverable files written to `jobbench_output/`\n- **Dataset**: ModelScope `evalscope/job-bench`\n- **Metric**: Weighted LLM-judge rubric score (`normalized_score`)\n\n## Evaluation Notes\n\n- The default evaluation split is `main`.\n- Configure `judge_model_args` for rubric scoring.\n- `normalized_score` is the primary weighted score (raw total / max score); `pass_rate` is the unweighted\n proportion of fully passed rubrics; `judge_score` is the raw sum of passed rubric weights, reported as a diagnostic.\n- Docker runs use `python:3.11-slim-bookworm` by default. For formal evaluation, provide an image with the Office,\n PDF, and spreadsheet tools required by the tasks.\n", + "description": "\n## Overview\n\nJobBench evaluates agentic systems on realistic professional work tasks that require reading reference files, producing\ndeliverables, and reconciling multi-source information. This adapter uses the ModelScope dataset\n`evalscope/job-bench`.\n\n## Task Description\n\n- **Task Type**: Agentic professional work / deliverable generation\n- **Input**: Workplace-style task prompt with optional reference files under `reference_files/`\n- **Output**: Final deliverable files written to `jobbench_output/`\n- **Dataset**: ModelScope `evalscope/job-bench`\n- **Metric**: Weighted LLM-judge rubric score (`normalized_score`)\n\n## Evaluation Notes\n\n- The default evaluation split is `main`.\n- Configure `judge.models` for rubric scoring.\n- `normalized_score` is the primary weighted score (raw total / max score); `pass_rate` is the unweighted\n proportion of fully passed rubrics; `judge_score` is the raw sum of passed rubric weights, reported as a diagnostic.\n- Docker runs use `python:3.11-slim-bookworm` by default. For formal evaluation, provide an image with the Office,\n PDF, and spreadsheet tools required by the tasks.\n", "prompt_template": "{question}", "system_prompt": "", "few_shot_prompt_template": "", @@ -32,6 +27,11 @@ "extra_params": {}, "sandbox_config": {}, "category": "agent", + "primary_metric": { + "name": "normalized_score", + "aggregation": null, + "dimensions": {} + }, "agent_config": { "strategy": "function_calling", "max_steps": 250 @@ -57,13 +57,13 @@ "std": 582.11 }, "target_length_mean": null, - "computed_at": "2026-07-23T16:40:07.716638" + "computed_at": "2026-08-20T10:40:26.058100" }, "sample_example": { "data": { "input": [ { - "id": "203fd1aa", + "id": "9cdb8ee0", "content": "You are preparing the Statistical Analysis Plan for Phase III trial XR-2847 evaluating cardiovascular disease prevention. The sponsor requires validation of statistical assumptions against empirical data and regulatory alignment before protoc ... [TRUNCATED 2026 chars] ... erables.\n\nWrite every final deliverable file under `jobbench_output`. Do not put intermediate scratch files there.\nYour final message may summarize what you produced, but files requested by the task must be actual files in\n`jobbench_output`.\n" } ], @@ -124,11 +124,11 @@ "truncated": true }, "readme": { - "en": "# JobBench\n\n\n## Overview\n\nJobBench evaluates agentic systems on realistic professional work tasks that require reading reference files, producing\ndeliverables, and reconciling multi-source information. This adapter uses the ModelScope dataset\n`evalscope/job-bench`.\n\n## Task Description\n\n- **Task Type**: Agentic professional work / deliverable generation\n- **Input**: Workplace-style task prompt with optional reference files under `reference_files/`\n- **Output**: Final deliverable files written to `jobbench_output/`\n- **Dataset**: ModelScope `evalscope/job-bench`\n- **Metric**: Weighted LLM-judge rubric score (`normalized_score`)\n\n## Evaluation Notes\n\n- The default evaluation split is `main`.\n- Configure `judge_model_args` for rubric scoring.\n- `normalized_score` is the primary weighted score (raw total / max score); `pass_rate` is the unweighted\n proportion of fully passed rubrics; `judge_score` is the raw sum of passed rubric weights, reported as a diagnostic.\n- Docker runs use `python:3.11-slim-bookworm` by default. For formal evaluation, provide an image with the Office,\n PDF, and spreadsheet tools required by the tasks.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `job_bench` |\n| **Dataset ID** | [evalscope/job-bench](https://modelscope.cn/datasets/evalscope/job-bench/summary) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `Knowledge`, `MultiTurn` |\n| **Metrics** | `normalized_score`, `pass_rate`, `judge_score` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `main` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 65 |\n| Prompt Length (Mean) | 3344.32 chars |\n| Prompt Length (Min/Max) | 2000 / 4920 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"203fd1aa\",\n \"content\": \"You are preparing the Statistical Analysis Plan for Phase III trial XR-2847 evaluating cardiovascular disease prevention. The sponsor requires validation of statistical assumptions against empirical data and regulatory alignment before protoc ... [TRUNCATED 2026 chars] ... erables.\\n\\nWrite every final deliverable file under `jobbench_output`. Do not put intermediate scratch files there.\\nYour final message may summarize what you produced, but files requested by the task must be actual files in\\n`jobbench_output`.\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n },\n {\n \"name\": \"python_exec\",\n \"description\": \"Execute Python source code inside the sandbox environment. Returns stdout and stderr output.\",\n \"parameters\": {\n \"properties\": {\n \"code\": {\n \"type\": \"string\",\n \"description\": \"Python source code to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"code\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"task_id\": \"biostatisticians__task1\",\n \"reference_files\": [\n \"dataset/biostatisticians/task1/task_folder/Clinical_Study_Proposal_CVD_Prevention.csv\",\n \"dataset/biostatisticians/task1/task_folder/framingham.csv\"\n ],\n \"rubric_json\": \"{\\n \\\"rubrics\\\": [\\n {\\n \\\"rubric\\\": \\\"Does the analysis calculate the observed 10-year CHD event rate in the eligible population as 20.2% (236/1166) and identify this as higher than the assumed 15% control group rate?\\\",\\n \\\"weight\\\": 10,\\n ... [TRUNCATED 4642 chars] ... ge criterion and by the sysBP criterion separately\\\",\\n \\\"The flow shows the final eligible population count (1,166)\\\",\\n \\\"The flow enables verification of the filtering process and assessment of generalizability\\\"\\n ]\\n }\\n ]\\n}\"\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets job_bench \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":250}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['job_bench'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=250,\n ),\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# JobBench\n\n\n## 概述\n\nJobBench 评估智能体系统在真实专业工作场景中的表现，这些任务需要阅读参考文件、生成交付成果，并整合多源信息。此适配器使用 ModelScope 数据集 `evalscope/job-bench`。\n\n## 任务描述\n\n- **任务类型**：智能体专业工作 / 交付成果生成\n- **输入**：职场风格的任务提示，可选包含 `reference_files/` 目录下的参考文件\n- **输出**：最终交付成果文件写入 `jobbench_output/` 目录\n- **数据集**：ModelScope `evalscope/job-bench`\n- **评估指标**：加权 LLM 评分标准得分（`normalized_score`）\n\n## 评估说明\n\n- 默认评估划分（split）为 `main`。\n- 配置 `judge_model_args` 用于评分标准打分。\n- `normalized_score` 是主要的加权得分（原始总分 / 满分）；`pass_rate` 是完全通过评分项的未加权比例；`judge_score` 是通过评分项权重的原始总和，仅作为诊断指标报告。\n- Docker 运行默认使用 `python:3.11-slim-bookworm` 镜像。正式评估时，请提供包含任务所需 Office、PDF 和电子表格工具的镜像。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `job_bench` |\n| **数据集ID** | [evalscope/job-bench](https://modelscope.cn/datasets/evalscope/job-bench/summary) |\n| **论文** | 无 |\n| **标签** | `Agent`, `Knowledge`, `MultiTurn` |\n| **指标** | `normalized_score`, `pass_rate`, `judge_score` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `main` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 65 |\n| 提示词长度（平均） | 3344.32 字符 |\n| 提示词长度（最小/最大） | 2000 / 4920 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"203fd1aa\",\n \"content\": \"You are preparing the Statistical Analysis Plan for Phase III trial XR-2847 evaluating cardiovascular disease prevention. The sponsor requires validation of statistical assumptions against empirical data and regulatory alignment before protoc ... [TRUNCATED 2026 chars] ... erables.\\n\\nWrite every final deliverable file under `jobbench_output`. Do not put intermediate scratch files there.\\nYour final message may summarize what you produced, but files requested by the task must be actual files in\\n`jobbench_output`.\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n },\n {\n \"name\": \"python_exec\",\n \"description\": \"Execute Python source code inside the sandbox environment. Returns stdout and stderr output.\",\n \"parameters\": {\n \"properties\": {\n \"code\": {\n \"type\": \"string\",\n \"description\": \"Python source code to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"code\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"task_id\": \"biostatisticians__task1\",\n \"reference_files\": [\n \"dataset/biostatisticians/task1/task_folder/Clinical_Study_Proposal_CVD_Prevention.csv\",\n \"dataset/biostatisticians/task1/task_folder/framingham.csv\"\n ],\n \"rubric_json\": \"{\\n \\\"rubrics\\\": [\\n {\\n \\\"rubric\\\": \\\"Does the analysis calculate the observed 10-year CHD event rate in the eligible population as 20.2% (236/1166) and identify this as higher than the assumed 15% control group rate?\\\",\\n \\\"weight\\\": 10,\\n ... [TRUNCATED 4642 chars] ... ge criterion and by the sysBP criterion separately\\\",\\n \\\"The flow shows the final eligible population count (1,166)\\\",\\n \\\"The flow enables verification of the filtering process and assessment of generalizability\\\"\\n ]\\n }\\n ]\\n}\"\n }\n}\n```\n\n*注：部分内容因展示需要已被截断。*\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets job_bench \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":250}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['job_bench'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=250,\n ),\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "39fd7f8640151f390d80cda8fb5e66f1", + "en": "# JobBench\n\n\n## Overview\n\nJobBench evaluates agentic systems on realistic professional work tasks that require reading reference files, producing\ndeliverables, and reconciling multi-source information. This adapter uses the ModelScope dataset\n`evalscope/job-bench`.\n\n## Task Description\n\n- **Task Type**: Agentic professional work / deliverable generation\n- **Input**: Workplace-style task prompt with optional reference files under `reference_files/`\n- **Output**: Final deliverable files written to `jobbench_output/`\n- **Dataset**: ModelScope `evalscope/job-bench`\n- **Metric**: Weighted LLM-judge rubric score (`normalized_score`)\n\n## Evaluation Notes\n\n- The default evaluation split is `main`.\n- Configure `judge.models` for rubric scoring.\n- `normalized_score` is the primary weighted score (raw total / max score); `pass_rate` is the unweighted\n proportion of fully passed rubrics; `judge_score` is the raw sum of passed rubric weights, reported as a diagnostic.\n- Docker runs use `python:3.11-slim-bookworm` by default. For formal evaluation, provide an image with the Office,\n PDF, and spreadsheet tools required by the tasks.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `job_bench` |\n| **Dataset ID** | [evalscope/job-bench](https://modelscope.cn/datasets/evalscope/job-bench/summary) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `Knowledge`, `MultiTurn` |\n| **Metrics** | `normalized_score`, `pass_rate`, `judge_score` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `main` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 65 |\n| Prompt Length (Mean) | 3344.32 chars |\n| Prompt Length (Min/Max) | 2000 / 4920 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"9cdb8ee0\",\n \"content\": \"You are preparing the Statistical Analysis Plan for Phase III trial XR-2847 evaluating cardiovascular disease prevention. The sponsor requires validation of statistical assumptions against empirical data and regulatory alignment before protoc ... [TRUNCATED 2026 chars] ... erables.\\n\\nWrite every final deliverable file under `jobbench_output`. Do not put intermediate scratch files there.\\nYour final message may summarize what you produced, but files requested by the task must be actual files in\\n`jobbench_output`.\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n },\n {\n \"name\": \"python_exec\",\n \"description\": \"Execute Python source code inside the sandbox environment. Returns stdout and stderr output.\",\n \"parameters\": {\n \"properties\": {\n \"code\": {\n \"type\": \"string\",\n \"description\": \"Python source code to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"code\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"task_id\": \"biostatisticians__task1\",\n \"reference_files\": [\n \"dataset/biostatisticians/task1/task_folder/Clinical_Study_Proposal_CVD_Prevention.csv\",\n \"dataset/biostatisticians/task1/task_folder/framingham.csv\"\n ],\n \"rubric_json\": \"{\\n \\\"rubrics\\\": [\\n {\\n \\\"rubric\\\": \\\"Does the analysis calculate the observed 10-year CHD event rate in the eligible population as 20.2% (236/1166) and identify this as higher than the assumed 15% control group rate?\\\",\\n \\\"weight\\\": 10,\\n ... [TRUNCATED 4642 chars] ... ge criterion and by the sysBP criterion separately\\\",\\n \\\"The flow shows the final eligible population count (1,166)\\\",\\n \\\"The flow enables verification of the filtering process and assessment of generalizability\\\"\\n ]\\n }\\n ]\\n}\"\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets job_bench \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":250}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['job_bench'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=250,\n ),\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# JobBench\n\n\n## 概述\n\nJobBench 评估智能体系统在真实专业工作场景中的表现，这些任务要求阅读参考文件、生成交付物，并整合多源信息。本适配器使用 ModelScope 数据集 `evalscope/job-bench`。\n\n## 任务描述\n\n- **任务类型**：智能体专业工作 / 交付物生成\n- **输入**：职场风格的任务提示，可选包含 `reference_files/` 目录下的参考文件\n- **输出**：最终交付物文件写入 `jobbench_output/` 目录\n- **数据集**：ModelScope `evalscope/job-bench`\n- **评估指标**：加权 LLM 评分标准得分（`normalized_score`）\n\n## 评估说明\n\n- 默认评估划分是 `main`。\n- 配置 `judge.models` 用于评分标准打分。\n- `normalized_score` 是主要的加权得分（原始总分 / 满分）；`pass_rate` 是完全通过评分项的未加权比例；`judge_score` 是通过评分项权重的原始总和，仅作为诊断指标报告。\n- Docker 运行默认使用 `python:3.11-slim-bookworm` 镜像。正式评估时，请提供包含任务所需 Office、PDF 和电子表格工具的镜像。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `job_bench` |\n| **数据集ID** | [evalscope/job-bench](https://modelscope.cn/datasets/evalscope/job-bench/summary) |\n| **论文** | 无 |\n| **标签** | `Agent`, `Knowledge`, `MultiTurn` |\n| **指标** | `normalized_score`, `pass_rate`, `judge_score` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `main` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 65 |\n| 提示词长度（平均） | 3344.32 字符 |\n| 提示词长度（最小/最大） | 2000 / 4920 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"9cdb8ee0\",\n \"content\": \"You are preparing the Statistical Analysis Plan for Phase III trial XR-2847 evaluating cardiovascular disease prevention. The sponsor requires validation of statistical assumptions against empirical data and regulatory alignment before protoc ... [TRUNCATED 2026 chars] ... erables.\\n\\nWrite every final deliverable file under `jobbench_output`. Do not put intermediate scratch files there.\\nYour final message may summarize what you produced, but files requested by the task must be actual files in\\n`jobbench_output`.\\n\"\n }\n ],\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n },\n {\n \"name\": \"python_exec\",\n \"description\": \"Execute Python source code inside the sandbox environment. Returns stdout and stderr output.\",\n \"parameters\": {\n \"properties\": {\n \"code\": {\n \"type\": \"string\",\n \"description\": \"Python source code to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"code\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"task_id\": \"biostatisticians__task1\",\n \"reference_files\": [\n \"dataset/biostatisticians/task1/task_folder/Clinical_Study_Proposal_CVD_Prevention.csv\",\n \"dataset/biostatisticians/task1/task_folder/framingham.csv\"\n ],\n \"rubric_json\": \"{\\n \\\"rubrics\\\": [\\n {\\n \\\"rubric\\\": \\\"Does the analysis calculate the observed 10-year CHD event rate in the eligible population as 20.2% (236/1166) and identify this as higher than the assumed 15% control group rate?\\\",\\n \\\"weight\\\": 10,\\n ... [TRUNCATED 4642 chars] ... ge criterion and by the sysBP criterion separately\\\",\\n \\\"The flow shows the final eligible population count (1,166)\\\",\\n \\\"The flow enables verification of the filtering process and assessment of generalizability\\\"\\n ]\\n }\\n ]\\n}\"\n }\n}\n```\n\n*注：部分内容因展示需要已被截断。*\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets job_bench \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":250}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['job_bench'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=250,\n ),\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "af9750a6e55ce1ed7ef0704c2f8d1a0e", "needs_translation": false }, - "updated_at": "2026-08-12T15:08:01.352270", - "translation_updated_at": "2026-07-23T16:40:09" + "updated_at": "2026-08-20T10:40:26.066466", + "translation_updated_at": "2026-08-20T10:41:18" } diff --git a/evalscope/benchmarks/_meta/mia_bench.json b/evalscope/benchmarks/_meta/mia_bench.json index 804c970c2..213402f9c 100644 --- a/evalscope/benchmarks/_meta/mia_bench.json +++ b/evalscope/benchmarks/_meta/mia_bench.json @@ -17,7 +17,7 @@ "subset_list": [ "default" ], - "description": "\n## Overview\n\nMIA-Bench is a multimodal instruction-following benchmark designed to evaluate vision-language models on their ability to follow complex, compositional instructions grounded in images. Each sample contains an image paired with a multi-component instruction, and model responses are scored by an LLM judge per component.\n\n## Task Description\n\n- **Task Type**: Multimodal Instruction Following\n- **Input**: Image + multi-component instruction\n- **Output**: Free-form response following all instruction components\n- **Domains**: Visual understanding, instruction following, language generation\n\n## Key Features\n\n- 400 test samples with diverse instruction types (basic to advanced)\n- Each instruction decomposes into 1–5 graded components with weighted scores\n- Component types include: describe, length_limit, linguistics, format, etc.\n- LLM-as-judge scoring: judge evaluates each component independently and gives a weighted total score (0–10 range, normalized to 0–1)\n- No predefined reference answers; scoring is fully judge-based\n\n## Evaluation Notes\n\n- Default evaluation uses the **test** split (400 samples)\n- Primary metric: **judge_score** (mean of per-sample normalized 0–1 total scores)\n- Requires a capable LLM judge (e.g., GPT-4o, Qwen-Max) configured via `judge_model_args`\n- Judge strategy should be set to `JudgeStrategy.LLM`\n", + "description": "\n## Overview\n\nMIA-Bench is a multimodal instruction-following benchmark designed to evaluate vision-language models on their ability to follow complex, compositional instructions grounded in images. Each sample contains an image paired with a multi-component instruction, and model responses are scored by an LLM judge per component.\n\n## Task Description\n\n- **Task Type**: Multimodal Instruction Following\n- **Input**: Image + multi-component instruction\n- **Output**: Free-form response following all instruction components\n- **Domains**: Visual understanding, instruction following, language generation\n\n## Key Features\n\n- 400 test samples with diverse instruction types (basic to advanced)\n- Each instruction decomposes into 1–5 graded components with weighted scores\n- Component types include: describe, length_limit, linguistics, format, etc.\n- LLM-as-judge scoring: judge evaluates each component independently and gives a weighted total score (0–10 range, normalized to 0–1)\n- No predefined reference answers; scoring is fully judge-based\n\n## Evaluation Notes\n\n- Default evaluation uses the **test** split (400 samples)\n- Primary metric: **judge_score** (mean of per-sample normalized 0–1 total scores)\n- Requires a capable LLM judge (e.g., GPT-4o, Qwen-Max) configured through `judge.models`\n- Judge strategy should be set to `judge.strategy='llm'`\n", "prompt_template": "", "system_prompt": "", "few_shot_prompt_template": "", @@ -80,7 +80,7 @@ "std": 49.72 }, "target_length_mean": null, - "computed_at": "2026-03-26T16:11:40.171943", + "computed_at": "2026-08-20T10:42:41.928967", "multimodal": { "has_images": true, "has_audio": false, @@ -120,7 +120,7 @@ "data": { "input": [ { - "id": "9156e81c", + "id": "c09c1ebf", "content": [ { "image": "[BASE64_IMAGE: jpeg, ~195.9KB]" @@ -159,11 +159,11 @@ "truncated": false }, "readme": { - "en": "# MIA-Bench\n\n\n## Overview\n\nMIA-Bench is a multimodal instruction-following benchmark designed to evaluate vision-language models on their ability to follow complex, compositional instructions grounded in images. Each sample contains an image paired with a multi-component instruction, and model responses are scored by an LLM judge per component.\n\n## Task Description\n\n- **Task Type**: Multimodal Instruction Following\n- **Input**: Image + multi-component instruction\n- **Output**: Free-form response following all instruction components\n- **Domains**: Visual understanding, instruction following, language generation\n\n## Key Features\n\n- 400 test samples with diverse instruction types (basic to advanced)\n- Each instruction decomposes into 1–5 graded components with weighted scores\n- Component types include: describe, length_limit, linguistics, format, etc.\n- LLM-as-judge scoring: judge evaluates each component independently and gives a weighted total score (0–10 range, normalized to 0–1)\n- No predefined reference answers; scoring is fully judge-based\n\n## Evaluation Notes\n\n- Default evaluation uses the **test** split (400 samples)\n- Primary metric: **judge_score** (mean of per-sample normalized 0–1 total scores)\n- Requires a capable LLM judge (e.g., GPT-4o, Qwen-Max) configured via `judge_model_args`\n- Judge strategy should be set to `JudgeStrategy.LLM`\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `mia_bench` |\n| **Dataset ID** | [lmms-lab/MIA-Bench](https://modelscope.cn/datasets/lmms-lab/MIA-Bench/summary) |\n| **Paper** | N/A |\n| **Tags** | `InstructionFollowing`, `MultiModal`, `QA` |\n| **Metrics** | `judge_score` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 400 |\n| Prompt Length (Mean) | 137.81 chars |\n| Prompt Length (Min/Max) | 34 / 327 chars |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 400 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 135x240 - 3264x4928 |\n| Formats | jpeg, mpo, webp |\n\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"9156e81c\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~195.9KB]\"\n },\n {\n \"text\": \"Explain the activity taking place in the image using exactly two sentences, including one metaphor.\"\n }\n ]\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"instruction\": \"Explain the activity taking place in the image using exactly two sentences, including one metaphor.\",\n \"type\": \"advanced\",\n \"num_of_component\": 3,\n \"components\": [\n \"Explain the activity taking place in the image\",\n \"using exactly two sentences\",\n \"including one metaphor\"\n ],\n \"component_weight\": [\n 4,\n 3,\n 3\n ],\n \"component_type\": [\n \"describe\",\n \"length_limit\",\n \"linguistics\"\n ]\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets mia_bench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['mia_bench'],\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# MIA-Bench\n\n\n## 概述\n\nMIA-Bench 是一个专为评估视觉-语言模型而设计的多模态指令遵循基准测试，重点考察模型在图像基础上遵循复杂、组合式指令的能力。每个样本包含一张图像和一条多组件指令，模型的回答将由大语言模型（LLM）评委按组件分别评分。\n\n## 任务描述\n\n- **任务类型**：多模态指令遵循\n- **输入**：图像 + 多组件指令\n- **输出**：自由形式的回答，需满足所有指令组件的要求\n- **领域**：视觉理解、指令遵循、语言生成\n\n## 主要特点\n\n- 包含 400 个测试样本，涵盖从基础到高级的多样化指令类型\n- 每条指令可分解为 1–5 个评分组件，并配有加权分数\n- 组件类型包括：描述（describe）、长度限制（length_limit）、语言学要求（linguistics）、格式（format）等\n- 采用 LLM 作为评委进行评分：评委独立评估每个组件，并给出加权总分（0–10 分范围，归一化至 0–1）\n- 不提供预定义的标准答案；评分完全依赖评委判断\n\n## 评估说明\n\n- 默认使用 **test** 划分（400 个样本）进行评估\n- 主要指标：**judge_score**（各样本归一化总分的平均值，范围 0–1）\n- 需配置一个能力强的 LLM 评委（例如 GPT-4o、Qwen-Max），通过 `judge_model_args` 参数指定\n- 评委策略应设置为 `JudgeStrategy.LLM`\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `mia_bench` |\n| **数据集ID** | [lmms-lab/MIA-Bench](https://modelscope.cn/datasets/lmms-lab/MIA-Bench/summary) |\n| **论文** | N/A |\n| **标签** | `InstructionFollowing`, `MultiModal`, `QA` |\n| **指标** | `judge_score` |\n| **默认示例数量** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 400 |\n| 提示词长度（平均） | 137.81 字符 |\n| 提示词长度（最小/最大） | 34 / 327 字符 |\n\n**图像统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 总图像数 | 400 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 135x240 - 3264x4928 |\n| 格式 | jpeg, mpo, webp |\n\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"9156e81c\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~195.9KB]\"\n },\n {\n \"text\": \"Explain the activity taking place in the image using exactly two sentences, including one metaphor.\"\n }\n ]\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"instruction\": \"Explain the activity taking place in the image using exactly two sentences, including one metaphor.\",\n \"type\": \"advanced\",\n \"num_of_component\": 3,\n \"components\": [\n \"Explain the activity taking place in the image\",\n \"using exactly two sentences\",\n \"including one metaphor\"\n ],\n \"component_weight\": [\n 4,\n 3,\n 3\n ],\n \"component_type\": [\n \"describe\",\n \"length_limit\",\n \"linguistics\"\n ]\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets mia_bench \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['mia_bench'],\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "1afa27eb2a58092f13b4349d059792d0", + "en": "# MIA-Bench\n\n\n## Overview\n\nMIA-Bench is a multimodal instruction-following benchmark designed to evaluate vision-language models on their ability to follow complex, compositional instructions grounded in images. Each sample contains an image paired with a multi-component instruction, and model responses are scored by an LLM judge per component.\n\n## Task Description\n\n- **Task Type**: Multimodal Instruction Following\n- **Input**: Image + multi-component instruction\n- **Output**: Free-form response following all instruction components\n- **Domains**: Visual understanding, instruction following, language generation\n\n## Key Features\n\n- 400 test samples with diverse instruction types (basic to advanced)\n- Each instruction decomposes into 1–5 graded components with weighted scores\n- Component types include: describe, length_limit, linguistics, format, etc.\n- LLM-as-judge scoring: judge evaluates each component independently and gives a weighted total score (0–10 range, normalized to 0–1)\n- No predefined reference answers; scoring is fully judge-based\n\n## Evaluation Notes\n\n- Default evaluation uses the **test** split (400 samples)\n- Primary metric: **judge_score** (mean of per-sample normalized 0–1 total scores)\n- Requires a capable LLM judge (e.g., GPT-4o, Qwen-Max) configured through `judge.models`\n- Judge strategy should be set to `judge.strategy='llm'`\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `mia_bench` |\n| **Dataset ID** | [lmms-lab/MIA-Bench](https://modelscope.cn/datasets/lmms-lab/MIA-Bench/summary) |\n| **Paper** | N/A |\n| **Tags** | `InstructionFollowing`, `MultiModal`, `QA` |\n| **Metrics** | `judge_score` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 400 |\n| Prompt Length (Mean) | 137.81 chars |\n| Prompt Length (Min/Max) | 34 / 327 chars |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 400 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 135x240 - 3264x4928 |\n| Formats | jpeg, mpo, webp |\n\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"c09c1ebf\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~195.9KB]\"\n },\n {\n \"text\": \"Explain the activity taking place in the image using exactly two sentences, including one metaphor.\"\n }\n ]\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"instruction\": \"Explain the activity taking place in the image using exactly two sentences, including one metaphor.\",\n \"type\": \"advanced\",\n \"num_of_component\": 3,\n \"components\": [\n \"Explain the activity taking place in the image\",\n \"using exactly two sentences\",\n \"including one metaphor\"\n ],\n \"component_weight\": [\n 4,\n 3,\n 3\n ],\n \"component_type\": [\n \"describe\",\n \"length_limit\",\n \"linguistics\"\n ]\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets mia_bench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['mia_bench'],\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# MIA-Bench\n\n\n## 概述\n\nMIA-Bench 是一个**多模态指令遵循基准测试**，旨在评估视觉-语言模型在图像基础上遵循复杂、组合式指令的能力。每个样本包含一张图像和一个多组件指令，模型的响应由大语言模型（LLM）评委按组件分别打分。\n\n## 任务描述\n\n- **任务类型**：多模态指令遵循\n- **输入**：图像 + 多组件指令\n- **输出**：自由形式的回答，需满足所有指令组件的要求\n- **领域**：视觉理解、指令遵循、语言生成\n\n## 核心特性\n\n- 包含 400 个测试样本，涵盖从基础到高级的多样化指令类型\n- 每条指令可分解为 1–5 个评分组件，并配有加权分数\n- 组件类型包括：describe（描述）、length_limit（长度限制）、linguistics（语言学要求）、format（格式）等\n- 采用 LLM 作为评委进行评分：评委独立评估每个组件，并给出加权总分（0–10 分范围，归一化至 0–1）\n- **无预设参考答案**；评分完全依赖评委判断\n\n## 评估说明\n\n- 默认使用 **test** 划分（400 个样本）进行评估\n- 主要指标：**judge_score**（各样本归一化总分 [0–1] 的平均值）\n- 需配置一个能力强的 LLM 评委（例如 GPT-4o、Qwen-Max），通过 `judge.models` 设置\n- 评委策略应设为 `judge.strategy='llm'`\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `mia_bench` |\n| **数据集ID** | [lmms-lab/MIA-Bench](https://modelscope.cn/datasets/lmms-lab/MIA-Bench/summary) |\n| **论文** | N/A |\n| **标签** | `InstructionFollowing`, `MultiModal`, `QA` |\n| **指标** | `judge_score` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 400 |\n| 提示词长度（平均） | 137.81 字符 |\n| 提示词长度（最小/最大） | 34 / 327 字符 |\n\n**图像统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 400 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 135x240 - 3264x4928 |\n| 格式 | jpeg, mpo, webp |\n\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"c09c1ebf\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~195.9KB]\"\n },\n {\n \"text\": \"Explain the activity taking place in the image using exactly two sentences, including one metaphor.\"\n }\n ]\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"instruction\": \"Explain the activity taking place in the image using exactly two sentences, including one metaphor.\",\n \"type\": \"advanced\",\n \"num_of_component\": 3,\n \"components\": [\n \"Explain the activity taking place in the image\",\n \"using exactly two sentences\",\n \"including one metaphor\"\n ],\n \"component_weight\": [\n 4,\n 3,\n 3\n ],\n \"component_type\": [\n \"describe\",\n \"length_limit\",\n \"linguistics\"\n ]\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets mia_bench \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['mia_bench'],\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "6558bda5fcfd3b7343331bc9dcfa7738", "needs_translation": false }, - "updated_at": "2026-08-12T15:08:01.351011", - "translation_updated_at": "2026-03-26T16:11:48Z" + "updated_at": "2026-08-20T10:42:42.303089", + "translation_updated_at": "2026-08-20T10:46:09" } diff --git a/evalscope/benchmarks/_meta/perception_bench.json b/evalscope/benchmarks/_meta/perception_bench.json index 1c750f365..0a59d0c0e 100644 --- a/evalscope/benchmarks/_meta/perception_bench.json +++ b/evalscope/benchmarks/_meta/perception_bench.json @@ -25,7 +25,7 @@ "ocr_error", "hallucination" ], - "description": "\n## Overview\n\nPerceptionBench is a benchmark from Moonshot AI that evaluates the atomic visual perception\ncapabilities of multimodal large language models. It is built bottom-up: the earliest failure\npoints of frontier MLLMs on 42 existing benchmarks were diagnosed to derive an error taxonomy\nwhose perception branch defines ten atomic perceptual capabilities. Each question isolates a\nsingle capability, so difficulty stems from perception rather than reasoning or knowledge.\n\n## Task Description\n\n- **Task Type**: Visual Perception (open-ended question answering)\n- **Input**: One or more images interleaved with a question\n- **Output**: Free-form short answer with a uniquely determined reference\n- **Domain**: Atomic visual perception across ten capabilities\n\n## Key Features\n\n- 3,000 verified questions covering ten atomic perceptual capabilities\n- 1,800 questions (60%) are atomic sub-questions decomposed from attributed failures on source\n benchmarks; 1,200 (40%) are newly authored on supplemented images\n- Subsets follow the ten `error_category` labels: visual relation, counting, attribute,\n depth & 3D perception, localization, comparison, fine-grained recognition, contextual\n integration, OCR, and perception-related hallucination\n- Multi-image questions are supported: images are interleaved into the question via\n `<|image_N|>` placeholders\n- Samples carrying a `hint` (coordinate convention or image dimensions) pass it as a system\n message, matching the official message builder\n\n## Evaluation Notes\n\n- Default evaluation uses the **train** split (3,000 samples, single split dataset)\n- Primary metric: **Accuracy**, reported overall and per capability\n- Scoring follows the official protocol: an LLM judge grades the free-form answer against the\n reference with the teacher-grading prompt and returns a strict 0/1 verdict per item\n (`[reason]` / `[judge] True|False`); the paper uses GPT-oss-120B, whose agreement with human\n judgment is 99.7% on a 300-sample audit\n- Empty or failed generations are scored 0 without invoking the judge\n- Requires `judge_model_args` configuration for the LLM judge\n- The dataset embeds images as base64 data URIs (~1.6 GB download on first use)\n", + "description": "\n## Overview\n\nPerceptionBench is a benchmark from Moonshot AI that evaluates the atomic visual perception\ncapabilities of multimodal large language models. It is built bottom-up: the earliest failure\npoints of frontier MLLMs on 42 existing benchmarks were diagnosed to derive an error taxonomy\nwhose perception branch defines ten atomic perceptual capabilities. Each question isolates a\nsingle capability, so difficulty stems from perception rather than reasoning or knowledge.\n\n## Task Description\n\n- **Task Type**: Visual Perception (open-ended question answering)\n- **Input**: One or more images interleaved with a question\n- **Output**: Free-form short answer with a uniquely determined reference\n- **Domain**: Atomic visual perception across ten capabilities\n\n## Key Features\n\n- 3,000 verified questions covering ten atomic perceptual capabilities\n- 1,800 questions (60%) are atomic sub-questions decomposed from attributed failures on source\n benchmarks; 1,200 (40%) are newly authored on supplemented images\n- Subsets follow the ten `error_category` labels: visual relation, counting, attribute,\n depth & 3D perception, localization, comparison, fine-grained recognition, contextual\n integration, OCR, and perception-related hallucination\n- Multi-image questions are supported: images are interleaved into the question via\n `<|image_N|>` placeholders\n- Samples carrying a `hint` (coordinate convention or image dimensions) pass it as a system\n message, matching the official message builder\n\n## Evaluation Notes\n\n- Default evaluation uses the **train** split (3,000 samples, single split dataset)\n- Primary metric: **Accuracy**, reported overall and per capability\n- Scoring follows the official protocol: an LLM judge grades the free-form answer against the\n reference with the teacher-grading prompt and returns a strict 0/1 verdict per item\n (`[reason]` / `[judge] True|False`); the paper uses GPT-oss-120B, whose agreement with human\n judgment is 99.7% on a 300-sample audit\n- Empty or failed generations are scored 0 without invoking the judge\n- Requires an LLM judge configured through `judge.models`\n- The dataset embeds images as base64 data URIs (~1.6 GB download on first use)\n", "prompt_template": "", "system_prompt": "", "few_shot_prompt_template": "", @@ -537,11 +537,11 @@ "truncated": false }, "readme": { - "en": "# PerceptionBench\n\n\n## Overview\n\nPerceptionBench is a benchmark from Moonshot AI that evaluates the atomic visual perception\ncapabilities of multimodal large language models. It is built bottom-up: the earliest failure\npoints of frontier MLLMs on 42 existing benchmarks were diagnosed to derive an error taxonomy\nwhose perception branch defines ten atomic perceptual capabilities. Each question isolates a\nsingle capability, so difficulty stems from perception rather than reasoning or knowledge.\n\n## Task Description\n\n- **Task Type**: Visual Perception (open-ended question answering)\n- **Input**: One or more images interleaved with a question\n- **Output**: Free-form short answer with a uniquely determined reference\n- **Domain**: Atomic visual perception across ten capabilities\n\n## Key Features\n\n- 3,000 verified questions covering ten atomic perceptual capabilities\n- 1,800 questions (60%) are atomic sub-questions decomposed from attributed failures on source\n benchmarks; 1,200 (40%) are newly authored on supplemented images\n- Subsets follow the ten `error_category` labels: visual relation, counting, attribute,\n depth & 3D perception, localization, comparison, fine-grained recognition, contextual\n integration, OCR, and perception-related hallucination\n- Multi-image questions are supported: images are interleaved into the question via\n `<|image_N|>` placeholders\n- Samples carrying a `hint` (coordinate convention or image dimensions) pass it as a system\n message, matching the official message builder\n\n## Evaluation Notes\n\n- Default evaluation uses the **train** split (3,000 samples, single split dataset)\n- Primary metric: **Accuracy**, reported overall and per capability\n- Scoring follows the official protocol: an LLM judge grades the free-form answer against the\n reference with the teacher-grading prompt and returns a strict 0/1 verdict per item\n (`[reason]` / `[judge] True|False`); the paper uses GPT-oss-120B, whose agreement with human\n judgment is 99.7% on a 300-sample audit\n- Empty or failed generations are scored 0 without invoking the judge\n- Requires `judge_model_args` configuration for the LLM judge\n- The dataset embeds images as base64 data URIs (~1.6 GB download on first use)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `perception_bench` |\n| **Dataset ID** | [moonshotai/PerceptionBench](https://modelscope.cn/datasets/moonshotai/PerceptionBench/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2607.24957) |\n| **Tags** | `MultiModal`, `QA` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 3,000 |\n| Prompt Length (Mean) | 233.87 chars |\n| Prompt Length (Min/Max) | 29 / 1076 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `visual_relation_error` | 330 | 275.62 | 43 | 876 |\n| `visual_counting_error` | 330 | 161.11 | 37 | 831 |\n| `visual_attribute_error` | 330 | 225.58 | 34 | 1006 |\n| `depth_3d_perception_error` | 330 | 278.5 | 60 | 976 |\n| `visual_localization_error` | 330 | 284.79 | 62 | 1076 |\n| `visual_comparison_error` | 279 | 270.14 | 39 | 801 |\n| `fine_grained_recognition_error` | 290 | 225.91 | 44 | 917 |\n| `context_integration_error` | 255 | 277.04 | 58 | 845 |\n| `ocr_error` | 255 | 175.39 | 29 | 934 |\n| `hallucination` | 271 | 150.94 | 42 | 515 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 3,567 |\n| Images per Sample | min: 1, max: 8, mean: 1.19 |\n| Resolution Range | 101x64 - 5712x4953 |\n| Formats | jpeg, png, webp |\n\n\n## Sample Example\n\n**Subset**: `visual_relation_error`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"28bf28ec\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: png, ~97.9KB]\"\n },\n {\n \"text\": \"How many arrows does the dashed box intersect with? Just answer with the number.\"\n }\n ]\n }\n ],\n \"target\": \"4\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"visual_relation_error\",\n \"metadata\": {\n \"index\": 5,\n \"problem\": \"<|image_1|>How many arrows does the dashed box intersect with? Just answer with the number.\",\n \"error_category\": \"visual_relation_error\",\n \"source_bmk\": \"NA\",\n \"source_idx\": null\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets perception_bench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['perception_bench'],\n dataset_args={\n 'perception_bench': {\n # subset_list: ['visual_relation_error', 'visual_counting_error', 'visual_attribute_error'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# PerceptionBench\n\n\n## 概述\n\nPerceptionBench 是由 Moonshot AI 提出的一项基准测试，用于评估多模态大语言模型（MLLM）的原子级视觉感知能力。该基准采用自底向上的构建方式：通过对前沿 MLLM 在 42 个现有基准上最早出现的失败点进行诊断，归纳出一个错误分类体系，其中感知分支定义了十种原子级感知能力。每个问题仅聚焦于单一能力，因此其难度来源于感知本身，而非推理或知识。\n\n## 任务描述\n\n- **任务类型**：视觉感知（开放式问答）\n- **输入**：一张或多张图像与一个问题交错排列\n- **输出**：自由格式的简短答案，且具有唯一确定的参考答案\n- **领域**：涵盖十种原子级视觉感知能力\n\n## 主要特性\n\n- 包含 3,000 个经过验证的问题，覆盖十种原子级感知能力\n- 其中 1,800 个问题（60%）是从源基准中归因失败案例分解出的原子子问题；1,200 个问题（40%）是在补充图像上全新编写的问题\n- 子集对应十种 `error_category` 标签：视觉关系、计数、属性、深度与 3D 感知、定位、比较、细粒度识别、上下文整合、OCR 和感知相关幻觉\n- 支持多图像问题：图像通过 `<|image_N|>` 占位符嵌入到问题中\n- 对于带有 `hint`（坐标约定或图像尺寸）的样本，会以系统消息形式传递，与官方消息构建器保持一致\n\n## 评估说明\n\n- 默认评估使用 **train** 切分（3,000 个样本，单切分数据集）\n- 主要指标：**准确率（Accuracy）**，报告整体及各能力维度的结果\n- 评分遵循官方协议：使用 LLM 评判器，根据教师评分提示（teacher-grading prompt）将自由格式答案与参考答案对比，并对每个样本返回严格的 0/1 判定（`[reason]` / `[judge] True|False`）；论文中使用的 GPT-oss-120B 模型在 300 个样本的人工审核中与人类判断的一致性达 99.7%\n- 空输出或生成失败的样本直接记为 0 分，不调用评判器\n- 需要配置 `judge_model_args` 以指定 LLM 评判器\n- 数据集中图像以 base64 data URI 形式嵌入（首次使用时下载约 1.6 GB）\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `perception_bench` |\n| **数据集ID** | [moonshotai/PerceptionBench](https://modelscope.cn/datasets/moonshotai/PerceptionBench/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2607.24957) |\n| **标签** | `MultiModal`, `QA` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估切分** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 3,000 |\n| 提示词长度（平均） | 233.87 字符 |\n| 提示词长度（最小/最大） | 29 / 1076 字符 |\n\n**各子集统计信息：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `visual_relation_error` | 330 | 275.62 | 43 | 876 |\n| `visual_counting_error` | 330 | 161.11 | 37 | 831 |\n| `visual_attribute_error` | 330 | 225.58 | 34 | 1006 |\n| `depth_3d_perception_error` | 330 | 278.5 | 60 | 976 |\n| `visual_localization_error` | 330 | 284.79 | 62 | 1076 |\n| `visual_comparison_error` | 279 | 270.14 | 39 | 801 |\n| `fine_grained_recognition_error` | 290 | 225.91 | 44 | 917 |\n| `context_integration_error` | 255 | 277.04 | 58 | 845 |\n| `ocr_error` | 255 | 175.39 | 29 | 934 |\n| `hallucination` | 271 | 150.94 | 42 | 515 |\n\n**图像统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 3,567 |\n| 每样本图像数 | 最小: 1, 最大: 8, 平均: 1.19 |\n| 分辨率范围 | 101x64 - 5712x4953 |\n| 格式 | jpeg, png, webp |\n\n\n## 样例示例\n\n**子集**: `visual_relation_error`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"28bf28ec\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: png, ~97.9KB]\"\n },\n {\n \"text\": \"How many arrows does the dashed box intersect with? Just answer with the number.\"\n }\n ]\n }\n ],\n \"target\": \"4\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"visual_relation_error\",\n \"metadata\": {\n \"index\": 5,\n \"problem\": \"<|image_1|>How many arrows does the dashed box intersect with? Just answer with the number.\",\n \"error_category\": \"visual_relation_error\",\n \"source_bmk\": \"NA\",\n \"source_idx\": null\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets perception_bench \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['perception_bench'],\n dataset_args={\n 'perception_bench': {\n # subset_list: ['visual_relation_error', 'visual_counting_error', 'visual_attribute_error'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "855a0e910a4c38e634b44588a8b5269d", + "en": "# PerceptionBench\n\n\n## Overview\n\nPerceptionBench is a benchmark from Moonshot AI that evaluates the atomic visual perception\ncapabilities of multimodal large language models. It is built bottom-up: the earliest failure\npoints of frontier MLLMs on 42 existing benchmarks were diagnosed to derive an error taxonomy\nwhose perception branch defines ten atomic perceptual capabilities. Each question isolates a\nsingle capability, so difficulty stems from perception rather than reasoning or knowledge.\n\n## Task Description\n\n- **Task Type**: Visual Perception (open-ended question answering)\n- **Input**: One or more images interleaved with a question\n- **Output**: Free-form short answer with a uniquely determined reference\n- **Domain**: Atomic visual perception across ten capabilities\n\n## Key Features\n\n- 3,000 verified questions covering ten atomic perceptual capabilities\n- 1,800 questions (60%) are atomic sub-questions decomposed from attributed failures on source\n benchmarks; 1,200 (40%) are newly authored on supplemented images\n- Subsets follow the ten `error_category` labels: visual relation, counting, attribute,\n depth & 3D perception, localization, comparison, fine-grained recognition, contextual\n integration, OCR, and perception-related hallucination\n- Multi-image questions are supported: images are interleaved into the question via\n `<|image_N|>` placeholders\n- Samples carrying a `hint` (coordinate convention or image dimensions) pass it as a system\n message, matching the official message builder\n\n## Evaluation Notes\n\n- Default evaluation uses the **train** split (3,000 samples, single split dataset)\n- Primary metric: **Accuracy**, reported overall and per capability\n- Scoring follows the official protocol: an LLM judge grades the free-form answer against the\n reference with the teacher-grading prompt and returns a strict 0/1 verdict per item\n (`[reason]` / `[judge] True|False`); the paper uses GPT-oss-120B, whose agreement with human\n judgment is 99.7% on a 300-sample audit\n- Empty or failed generations are scored 0 without invoking the judge\n- Requires an LLM judge configured through `judge.models`\n- The dataset embeds images as base64 data URIs (~1.6 GB download on first use)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `perception_bench` |\n| **Dataset ID** | [moonshotai/PerceptionBench](https://modelscope.cn/datasets/moonshotai/PerceptionBench/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2607.24957) |\n| **Tags** | `MultiModal`, `QA` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 3,000 |\n| Prompt Length (Mean) | 233.87 chars |\n| Prompt Length (Min/Max) | 29 / 1076 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `visual_relation_error` | 330 | 275.62 | 43 | 876 |\n| `visual_counting_error` | 330 | 161.11 | 37 | 831 |\n| `visual_attribute_error` | 330 | 225.58 | 34 | 1006 |\n| `depth_3d_perception_error` | 330 | 278.5 | 60 | 976 |\n| `visual_localization_error` | 330 | 284.79 | 62 | 1076 |\n| `visual_comparison_error` | 279 | 270.14 | 39 | 801 |\n| `fine_grained_recognition_error` | 290 | 225.91 | 44 | 917 |\n| `context_integration_error` | 255 | 277.04 | 58 | 845 |\n| `ocr_error` | 255 | 175.39 | 29 | 934 |\n| `hallucination` | 271 | 150.94 | 42 | 515 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 3,567 |\n| Images per Sample | min: 1, max: 8, mean: 1.19 |\n| Resolution Range | 101x64 - 5712x4953 |\n| Formats | jpeg, png, webp |\n\n\n## Sample Example\n\n**Subset**: `visual_relation_error`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"28bf28ec\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: png, ~97.9KB]\"\n },\n {\n \"text\": \"How many arrows does the dashed box intersect with? Just answer with the number.\"\n }\n ]\n }\n ],\n \"target\": \"4\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"visual_relation_error\",\n \"metadata\": {\n \"index\": 5,\n \"problem\": \"<|image_1|>How many arrows does the dashed box intersect with? Just answer with the number.\",\n \"error_category\": \"visual_relation_error\",\n \"source_bmk\": \"NA\",\n \"source_idx\": null\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets perception_bench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['perception_bench'],\n dataset_args={\n 'perception_bench': {\n # subset_list: ['visual_relation_error', 'visual_counting_error', 'visual_attribute_error'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# PerceptionBench\n\n\n## 概述\n\nPerceptionBench 是由 Moonshot AI 推出的一项基准测试，用于评估多模态大语言模型（MLLM）的原子级视觉感知能力。该基准采用自下而上的构建方式：通过对前沿 MLLM 在 42 个现有基准上的最早失败点进行诊断，归纳出一个错误分类体系，其中“感知”分支定义了十种原子级感知能力。每个问题仅聚焦于单一能力，因此其难度来源于感知本身，而非推理或知识。\n\n## 任务描述\n\n- **任务类型**：视觉感知（开放式问答）\n- **输入**：一张或多张图像与一个问题交错排列\n- **输出**：自由格式的简短答案，且具有唯一确定的参考答案\n- **领域**：涵盖十种原子级视觉感知能力\n\n## 核心特性\n\n- 包含 3,000 个经过验证的问题，覆盖十种原子级感知能力\n- 其中 1,800 个问题（60%）是从源基准中归因失败案例分解出的原子子问题；1,200 个问题（40%）是在补充图像上全新编写的问题\n- 子集遵循十种 `error_category` 标签：视觉关系、计数、属性、深度与 3D 感知、定位、比较、细粒度识别、上下文整合、OCR 和感知相关幻觉\n- 支持多图像问题：图像通过 `<|image_N|>` 占位符交错嵌入问题中\n- 对于包含 `hint`（坐标约定或图像尺寸）的样本，会通过系统消息传递该提示，与官方消息构建器保持一致\n\n## 评估说明\n\n- 默认评估使用 **train** 切分（3,000 个样本，单切分数据集）\n- 主要指标：**准确率（Accuracy）**，报告整体及各能力维度的结果\n- 评分遵循官方协议：使用 LLM 评判器，通过教师评分提示（teacher-grading prompt）将自由格式答案与参考答案对比，并对每个样本返回严格的 0/1 判定（`[reason]` / `[judge] True|False`）；论文中使用 GPT-oss-120B 作为评判器，在 300 个样本的人工审核中与人类判断的一致性达 99.7%\n- 空输出或生成失败的答案直接记为 0 分，不调用评判器\n- 需通过 `judge.models` 配置 LLM 评判器\n- 数据集中图像以 base64 data URI 形式嵌入（首次使用时下载约 1.6 GB）\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `perception_bench` |\n| **数据集ID** | [moonshotai/PerceptionBench](https://modelscope.cn/datasets/moonshotai/PerceptionBench/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2607.24957) |\n| **标签** | `MultiModal`, `QA` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估切分** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 3,000 |\n| 提示词长度（平均） | 233.87 字符 |\n| 提示词长度（最小/最大） | 29 / 1076 字符 |\n\n**各子集统计信息：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `visual_relation_error` | 330 | 275.62 | 43 | 876 |\n| `visual_counting_error` | 330 | 161.11 | 37 | 831 |\n| `visual_attribute_error` | 330 | 225.58 | 34 | 1006 |\n| `depth_3d_perception_error` | 330 | 278.5 | 60 | 976 |\n| `visual_localization_error` | 330 | 284.79 | 62 | 1076 |\n| `visual_comparison_error` | 279 | 270.14 | 39 | 801 |\n| `fine_grained_recognition_error` | 290 | 225.91 | 44 | 917 |\n| `context_integration_error` | 255 | 277.04 | 58 | 845 |\n| `ocr_error` | 255 | 175.39 | 29 | 934 |\n| `hallucination` | 271 | 150.94 | 42 | 515 |\n\n**图像统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 3,567 |\n| 每样本图像数 | 最小: 1, 最大: 8, 平均: 1.19 |\n| 分辨率范围 | 101x64 - 5712x4953 |\n| 格式 | jpeg, png, webp |\n\n\n## 样例示例\n\n**子集**: `visual_relation_error`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"28bf28ec\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: png, ~97.9KB]\"\n },\n {\n \"text\": \"How many arrows does the dashed box intersect with? Just answer with the number.\"\n }\n ]\n }\n ],\n \"target\": \"4\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"visual_relation_error\",\n \"metadata\": {\n \"index\": 5,\n \"problem\": \"<|image_1|>How many arrows does the dashed box intersect with? Just answer with the number.\",\n \"error_category\": \"visual_relation_error\",\n \"source_bmk\": \"NA\",\n \"source_idx\": null\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 通过 CLI 使用\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets perception_bench \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 通过 Python 使用\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['perception_bench'],\n dataset_args={\n 'perception_bench': {\n # subset_list: ['visual_relation_error', 'visual_counting_error', 'visual_attribute_error'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "594673efeb002dc5abefd3ab621863e0", "needs_translation": false }, - "updated_at": "2026-08-11T15:30:53.625987", - "translation_updated_at": "2026-08-06T15:39:40" + "updated_at": "2026-08-20T10:40:24.366106", + "translation_updated_at": "2026-08-20T10:41:18" } diff --git a/evalscope/benchmarks/_meta/phyx_mc.json b/evalscope/benchmarks/_meta/phyx_mc.json index d49547476..1d8eb2b98 100644 --- a/evalscope/benchmarks/_meta/phyx_mc.json +++ b/evalscope/benchmarks/_meta/phyx_mc.json @@ -22,7 +22,7 @@ "optics", "modern_physics" ], - "description": "\n## Overview\n\nPhyX is the first large-scale benchmark for physical reasoning in realistic, visually grounded\nscenarios. This is its multiple-choice variant: each university-level physics problem is presented\nwith a figure and four answer options, and the model has to name the correct option letter.\n\n## Task Description\n\n- **Task Type**: Visual multiple-choice physics problem solving\n- **Input**: A figure plus the problem description, question and four labelled options\n- **Output**: A single option letter (A, B, C or D)\n- **Domain**: University-level physics (mechanics, electromagnetism, thermodynamics, wave/acoustics,\n optics, modern physics)\n\n## Key Features\n\n- 3,000 university-level problems (`test`) over 6 core domains and 25 sub-domains, each domain exposed\n as its own subset; `eval_split='test_mini'` selects the official 1,000-problem testmini set.\n- Every problem is grounded in a figure that carries information the text does not restate, so the\n model must combine visual cues with implicit physical laws.\n- 6 reasoning types are represented (physical model grounding, multi-formula, spatial relation,\n numerical, predictive and implicit condition reasoning).\n- Uses the default *Text-DeRedundancy* input style of the paper: the simplified problem description\n plus the question, with the figure attached.\n\n- The official prompt is reproduced verbatim, including its instruction to answer with the option\n letter only, so scores stay comparable with the published numbers.\n\n## Evaluation Notes\n\n- Primary metric: `acc`, mean over problems, reported overall and per domain.\n- Default scoring is the official string-level match: the chosen letter is extracted from the reply\n and compared with the ground truth, accepting replies that mark the correct option the way the\n prompt prints it (`D:`) or emphasises it (`**D**`).\n- Setting `judge_strategy='llm'` (with `judge_model_args`) reproduces the official LLM-judged mode.\n The judge is only consulted for replies whose option letter could not be extracted, matching\n upstream.\n- Figures are sent inline as base64 and the largest is ~5 MB; set `max_image_bytes` in `dataset_args`\n if the served model enforces a smaller per-image limit.\n- Resources: [Paper](https://arxiv.org/abs/2505.15929) | [GitHub](https://github.com/NastyMarcus/PhyX)\n | [Project page](https://killthefullmoon.github.io/projects/PhyX/index.html)\n", + "description": "\n## Overview\n\nPhyX is the first large-scale benchmark for physical reasoning in realistic, visually grounded\nscenarios. This is its multiple-choice variant: each university-level physics problem is presented\nwith a figure and four answer options, and the model has to name the correct option letter.\n\n## Task Description\n\n- **Task Type**: Visual multiple-choice physics problem solving\n- **Input**: A figure plus the problem description, question and four labelled options\n- **Output**: A single option letter (A, B, C or D)\n- **Domain**: University-level physics (mechanics, electromagnetism, thermodynamics, wave/acoustics,\n optics, modern physics)\n\n## Key Features\n\n- 3,000 university-level problems (`test`) over 6 core domains and 25 sub-domains, each domain exposed\n as its own subset; `eval_split='test_mini'` selects the official 1,000-problem testmini set.\n- Every problem is grounded in a figure that carries information the text does not restate, so the\n model must combine visual cues with implicit physical laws.\n- 6 reasoning types are represented (physical model grounding, multi-formula, spatial relation,\n numerical, predictive and implicit condition reasoning).\n- Uses the default *Text-DeRedundancy* input style of the paper: the simplified problem description\n plus the question, with the figure attached.\n\n- The official prompt is reproduced verbatim, including its instruction to answer with the option\n letter only, so scores stay comparable with the published numbers.\n\n## Evaluation Notes\n\n- Primary metric: `acc`, mean over problems, reported overall and per domain.\n- Default scoring is the official string-level match: the chosen letter is extracted from the reply\n and compared with the ground truth, accepting replies that mark the correct option the way the\n prompt prints it (`D:`) or emphasises it (`**D**`).\n- Setting `judge.strategy='llm'` with `judge.models` reproduces the official LLM-judged mode.\n The judge is only consulted for replies whose option letter could not be extracted, matching\n upstream.\n- Figures are sent inline as base64 and the largest is ~5 MB; set `max_image_bytes` in `dataset_args`\n if the served model enforces a smaller per-image limit.\n- Resources: [Paper](https://arxiv.org/abs/2505.15929) | [GitHub](https://github.com/NastyMarcus/PhyX)\n | [Project page](https://killthefullmoon.github.io/projects/PhyX/index.html)\n", "prompt_template": "", "system_prompt": "", "few_shot_prompt_template": "", @@ -358,11 +358,11 @@ "truncated": false }, "readme": { - "en": "# PhyX-MC\n\n\n## Overview\n\nPhyX is the first large-scale benchmark for physical reasoning in realistic, visually grounded\nscenarios. This is its multiple-choice variant: each university-level physics problem is presented\nwith a figure and four answer options, and the model has to name the correct option letter.\n\n## Task Description\n\n- **Task Type**: Visual multiple-choice physics problem solving\n- **Input**: A figure plus the problem description, question and four labelled options\n- **Output**: A single option letter (A, B, C or D)\n- **Domain**: University-level physics (mechanics, electromagnetism, thermodynamics, wave/acoustics,\n optics, modern physics)\n\n## Key Features\n\n- 3,000 university-level problems (`test`) over 6 core domains and 25 sub-domains, each domain exposed\n as its own subset; `eval_split='test_mini'` selects the official 1,000-problem testmini set.\n- Every problem is grounded in a figure that carries information the text does not restate, so the\n model must combine visual cues with implicit physical laws.\n- 6 reasoning types are represented (physical model grounding, multi-formula, spatial relation,\n numerical, predictive and implicit condition reasoning).\n- Uses the default *Text-DeRedundancy* input style of the paper: the simplified problem description\n plus the question, with the figure attached.\n\n- The official prompt is reproduced verbatim, including its instruction to answer with the option\n letter only, so scores stay comparable with the published numbers.\n\n## Evaluation Notes\n\n- Primary metric: `acc`, mean over problems, reported overall and per domain.\n- Default scoring is the official string-level match: the chosen letter is extracted from the reply\n and compared with the ground truth, accepting replies that mark the correct option the way the\n prompt prints it (`D:`) or emphasises it (`**D**`).\n- Setting `judge_strategy='llm'` (with `judge_model_args`) reproduces the official LLM-judged mode.\n The judge is only consulted for replies whose option letter could not be extracted, matching\n upstream.\n- Figures are sent inline as base64 and the largest is ~5 MB; set `max_image_bytes` in `dataset_args`\n if the served model enforces a smaller per-image limit.\n- Resources: [Paper](https://arxiv.org/abs/2505.15929) | [GitHub](https://github.com/NastyMarcus/PhyX)\n | [Project page](https://killthefullmoon.github.io/projects/PhyX/index.html)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `phyx_mc` |\n| **Dataset ID** | [evalscope/PhyX](https://modelscope.cn/datasets/evalscope/PhyX/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2505.15929) |\n| **Tags** | `MCQ`, `MultiModal`, `Reasoning` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 3,000 |\n| Prompt Length (Mean) | 487.19 chars |\n| Prompt Length (Min/Max) | 178 / 2039 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `mechanics` | 550 | 471.63 | 203 | 1364 |\n| `electromagnetism` | 550 | 466.88 | 189 | 1125 |\n| `thermodynamics` | 500 | 498.81 | 178 | 1283 |\n| `waves_acoustics` | 500 | 492.87 | 196 | 1880 |\n| `optics` | 500 | 478.61 | 194 | 1376 |\n| `modern_physics` | 400 | 525.59 | 199 | 2039 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 3,000 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 215x46 - 5712x4953 |\n| Formats | jpeg, png |\n\n\n## Sample Example\n\n**Subset**: `mechanics`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"4334f3a0\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: png, ~35.6KB]\"\n },\n {\n \"text\": \"A patient with a dislocated shoulder is put into a traction apparatus as shown in figure. The pulls $\\\\vec{A}$ and $\\\\vec{B} must combine to produce an outward traction force of 12.8 N on the patient’s arm. How large should these pulls be?Please directly answer the question and provide the correct OPTION LETTER ONLY, e.g., A, B, C, D. OPTION: A: 7.55N B: 5.55N C: 7.65N D: 6.65N\"\n }\n ]\n }\n ],\n \"target\": \"A\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"index\": \"0\",\n \"category\": \"Mechanics\",\n \"subfield\": \"Statics\",\n \"reasoning_type\": [\n \"Spatial Relation Reasoning\"\n ]\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets phyx_mc \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['phyx_mc'],\n dataset_args={\n 'phyx_mc': {\n # subset_list: ['mechanics', 'electromagnetism', 'thermodynamics'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# PhyX-MC\n\n\n## 概述\n\nPhyX 是首个面向现实、视觉接地场景中物理推理的大规模基准测试。这是其多项选择题（multiple-choice）变体：每个大学水平的物理问题均附带一张图和四个选项，模型需输出正确选项对应的字母。\n\n## 任务描述\n\n- **任务类型**：视觉多项选择物理问题求解\n- **输入**：一张图像，以及问题描述、提问和四个带标签的选项\n- **输出**：单个选项字母（A、B、C 或 D）\n- **领域**：大学水平物理（力学、电磁学、热力学、波动/声学、光学、现代物理）\n\n## 核心特性\n\n- 包含 3,000 道大学水平题目（`test`），覆盖 6 个核心领域和 25 个子领域，每个领域作为独立子集提供；设置 `eval_split='test_mini'` 可选用官方 1,000 题的 testmini 测试集。\n- 每道题均基于一张图像，其中包含文本未重复说明的关键信息，因此模型必须结合视觉线索与隐含的物理定律进行推理。\n- 涵盖 6 种推理类型（物理模型接地、多公式、空间关系、数值、预测性及隐含条件推理）。\n- 采用论文默认的 *Text-DeRedundancy* 输入格式：简化后的问题描述加提问，并附上图像。\n\n- 官方提示词被逐字复现，包括要求仅输出选项字母的指令，以确保评估结果与已发表数据具有可比性。\n\n## 评估说明\n\n- 主要指标：`acc`（准确率），对所有题目取平均值，并分别报告整体及各领域的得分。\n- 默认评分方式为官方字符串级匹配：从模型回复中提取选项字母并与标准答案比较，接受如 `D:` 或 `**D**` 等符合提示格式的正确选项标记形式。\n- 设置 `judge_strategy='llm'`（配合 `judge_model_args`）可复现官方 LLM 判定模式。仅当无法从回复中提取选项字母时，才调用判别模型，与上游实现一致。\n- 图像以内联 base64 形式发送，最大约 5 MB；若所用模型对单张图像有更小的大小限制，请在 `dataset_args` 中设置 `max_image_bytes`。\n- 资源链接：[论文](https://arxiv.org/abs/2505.15929) | [GitHub](https://github.com/NastyMarcus/PhyX) | [项目主页](https://killthefullmoon.github.io/projects/PhyX/index.html)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `phyx_mc` |\n| **数据集ID** | [evalscope/PhyX](https://modelscope.cn/datasets/evalscope/PhyX/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2505.15929) |\n| **标签** | `MCQ`, `MultiModal`, `Reasoning` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 3,000 |\n| 提示词长度（平均） | 487.19 字符 |\n| 提示词长度（最小/最大） | 178 / 2039 字符 |\n\n**各子集统计：**\n\n| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 |\n|--------|---------|-------------|------------|------------|\n| `mechanics` | 550 | 471.63 | 203 | 1364 |\n| `electromagnetism` | 550 | 466.88 | 189 | 1125 |\n| `thermodynamics` | 500 | 498.81 | 178 | 1283 |\n| `waves_acoustics` | 500 | 492.87 | 196 | 1880 |\n| `optics` | 500 | 478.61 | 194 | 1376 |\n| `modern_physics` | 400 | 525.59 | 199 | 2039 |\n\n**图像统计：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 3,000 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 215x46 - 5712x4953 |\n| 格式 | jpeg, png |\n\n\n## 样例示例\n\n**子集**: `mechanics`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"4334f3a0\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: png, ~35.6KB]\"\n },\n {\n \"text\": \"A patient with a dislocated shoulder is put into a traction apparatus as shown in figure. The pulls $\\\\vec{A}$ and $\\\\vec{B} must combine to produce an outward traction force of 12.8 N on the patient’s arm. How large should these pulls be?Please directly answer the question and provide the correct OPTION LETTER ONLY, e.g., A, B, C, D. OPTION: A: 7.55N B: 5.55N C: 7.65N D: 6.65N\"\n }\n ]\n }\n ],\n \"target\": \"A\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"index\": \"0\",\n \"category\": \"Mechanics\",\n \"subfield\": \"Statics\",\n \"reasoning_type\": [\n \"Spatial Relation Reasoning\"\n ]\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets phyx_mc \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['phyx_mc'],\n dataset_args={\n 'phyx_mc': {\n # subset_list: ['mechanics', 'electromagnetism', 'thermodynamics'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "80ee6a1fafe0c7ef19320236c4f93994", + "en": "# PhyX-MC\n\n\n## Overview\n\nPhyX is the first large-scale benchmark for physical reasoning in realistic, visually grounded\nscenarios. This is its multiple-choice variant: each university-level physics problem is presented\nwith a figure and four answer options, and the model has to name the correct option letter.\n\n## Task Description\n\n- **Task Type**: Visual multiple-choice physics problem solving\n- **Input**: A figure plus the problem description, question and four labelled options\n- **Output**: A single option letter (A, B, C or D)\n- **Domain**: University-level physics (mechanics, electromagnetism, thermodynamics, wave/acoustics,\n optics, modern physics)\n\n## Key Features\n\n- 3,000 university-level problems (`test`) over 6 core domains and 25 sub-domains, each domain exposed\n as its own subset; `eval_split='test_mini'` selects the official 1,000-problem testmini set.\n- Every problem is grounded in a figure that carries information the text does not restate, so the\n model must combine visual cues with implicit physical laws.\n- 6 reasoning types are represented (physical model grounding, multi-formula, spatial relation,\n numerical, predictive and implicit condition reasoning).\n- Uses the default *Text-DeRedundancy* input style of the paper: the simplified problem description\n plus the question, with the figure attached.\n\n- The official prompt is reproduced verbatim, including its instruction to answer with the option\n letter only, so scores stay comparable with the published numbers.\n\n## Evaluation Notes\n\n- Primary metric: `acc`, mean over problems, reported overall and per domain.\n- Default scoring is the official string-level match: the chosen letter is extracted from the reply\n and compared with the ground truth, accepting replies that mark the correct option the way the\n prompt prints it (`D:`) or emphasises it (`**D**`).\n- Setting `judge.strategy='llm'` with `judge.models` reproduces the official LLM-judged mode.\n The judge is only consulted for replies whose option letter could not be extracted, matching\n upstream.\n- Figures are sent inline as base64 and the largest is ~5 MB; set `max_image_bytes` in `dataset_args`\n if the served model enforces a smaller per-image limit.\n- Resources: [Paper](https://arxiv.org/abs/2505.15929) | [GitHub](https://github.com/NastyMarcus/PhyX)\n | [Project page](https://killthefullmoon.github.io/projects/PhyX/index.html)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `phyx_mc` |\n| **Dataset ID** | [evalscope/PhyX](https://modelscope.cn/datasets/evalscope/PhyX/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2505.15929) |\n| **Tags** | `MCQ`, `MultiModal`, `Reasoning` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 3,000 |\n| Prompt Length (Mean) | 487.19 chars |\n| Prompt Length (Min/Max) | 178 / 2039 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `mechanics` | 550 | 471.63 | 203 | 1364 |\n| `electromagnetism` | 550 | 466.88 | 189 | 1125 |\n| `thermodynamics` | 500 | 498.81 | 178 | 1283 |\n| `waves_acoustics` | 500 | 492.87 | 196 | 1880 |\n| `optics` | 500 | 478.61 | 194 | 1376 |\n| `modern_physics` | 400 | 525.59 | 199 | 2039 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 3,000 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 215x46 - 5712x4953 |\n| Formats | jpeg, png |\n\n\n## Sample Example\n\n**Subset**: `mechanics`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"4334f3a0\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: png, ~35.6KB]\"\n },\n {\n \"text\": \"A patient with a dislocated shoulder is put into a traction apparatus as shown in figure. The pulls $\\\\vec{A}$ and $\\\\vec{B} must combine to produce an outward traction force of 12.8 N on the patient’s arm. How large should these pulls be?Please directly answer the question and provide the correct OPTION LETTER ONLY, e.g., A, B, C, D. OPTION: A: 7.55N B: 5.55N C: 7.65N D: 6.65N\"\n }\n ]\n }\n ],\n \"target\": \"A\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"index\": \"0\",\n \"category\": \"Mechanics\",\n \"subfield\": \"Statics\",\n \"reasoning_type\": [\n \"Spatial Relation Reasoning\"\n ]\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets phyx_mc \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['phyx_mc'],\n dataset_args={\n 'phyx_mc': {\n # subset_list: ['mechanics', 'electromagnetism', 'thermodynamics'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# PhyX-MC\n\n\n## 概述\n\nPhyX 是首个面向现实、视觉接地场景中物理推理的大规模基准测试。这是其多项选择题（multiple-choice）变体：每个大学水平的物理问题均附带一张图和四个选项，模型需输出正确选项对应的字母。\n\n## 任务描述\n\n- **任务类型**：视觉多选物理问题求解 \n- **输入**：一张图像，以及问题描述、提问和四个带标签的选项 \n- **输出**：单个选项字母（A、B、C 或 D） \n- **领域**：大学物理（力学、电磁学、热力学、波动/声学、光学、近代物理）\n\n## 核心特性\n\n- 包含 3,000 道大学水平题目（`test`），覆盖 6 个核心领域和 25 个子领域，每个领域作为独立子集提供；设置 `eval_split='test_mini'` 可选择官方指定的 1,000 题 testmini 测试集。\n- 每道题均基于一张图像，其中包含文本未重复说明的关键信息，因此模型必须结合视觉线索与隐含的物理定律进行推理。\n- 涵盖 6 种推理类型（物理模型接地、多公式、空间关系、数值、预测性及隐含条件推理）。\n- 采用论文中默认的 *Text-DeRedundancy* 输入风格：简化后的问题描述加提问，并附上图像。\n\n- 官方提示词被逐字复现，包括要求仅输出选项字母的指令，以确保评分结果可与已发表数据直接比较。\n\n## 评估说明\n\n- 主要指标：`acc`（准确率），对所有题目取平均值，并报告整体及各领域的得分。\n- 默认评分方式为官方字符串级匹配：从模型回复中提取选项字母，并与标准答案比对，接受如 `D:` 或 `**D**` 等符合提示格式的正确选项标记形式。\n- 若设置 `judge.strategy='llm'` 并指定 `judge.models`，则复现官方 LLM 判定模式。仅当无法从回复中提取选项字母时，才调用判别模型，与上游实现一致。\n- 图像以内联 base64 形式发送，最大约 5 MB；若所用模型对单图大小有限制，可在 `dataset_args` 中设置 `max_image_bytes`。\n- 资源链接：[论文](https://arxiv.org/abs/2505.15929) | [GitHub](https://github.com/NastyMarcus/PhyX) | [项目主页](https://killthefullmoon.github.io/projects/PhyX/index.html)\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `phyx_mc` |\n| **数据集 ID** | [evalscope/PhyX](https://modelscope.cn/datasets/evalscope/PhyX/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2505.15929) |\n| **标签** | `MCQ`, `MultiModal`, `Reasoning` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 3,000 |\n| 提示词长度（平均） | 487.19 字符 |\n| 提示词长度（最小/最大） | 178 / 2039 字符 |\n\n**各子集统计：**\n\n| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 |\n|--------|---------|-------------|------------|------------|\n| `mechanics` | 550 | 471.63 | 203 | 1364 |\n| `electromagnetism` | 550 | 466.88 | 189 | 1125 |\n| `thermodynamics` | 500 | 498.81 | 178 | 1283 |\n| `waves_acoustics` | 500 | 492.87 | 196 | 1880 |\n| `optics` | 500 | 478.61 | 194 | 1376 |\n| `modern_physics` | 400 | 525.59 | 199 | 2039 |\n\n**图像统计：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 3,000 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 215x46 - 5712x4953 |\n| 格式 | jpeg, png |\n\n\n## 样例示例\n\n**子集**: `mechanics`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"4334f3a0\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: png, ~35.6KB]\"\n },\n {\n \"text\": \"A patient with a dislocated shoulder is put into a traction apparatus as shown in figure. The pulls $\\\\vec{A}$ and $\\\\vec{B} must combine to produce an outward traction force of 12.8 N on the patient’s arm. How large should these pulls be?Please directly answer the question and provide the correct OPTION LETTER ONLY, e.g., A, B, C, D. OPTION: A: 7.55N B: 5.55N C: 7.65N D: 6.65N\"\n }\n ]\n }\n ],\n \"target\": \"A\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"index\": \"0\",\n \"category\": \"Mechanics\",\n \"subfield\": \"Statics\",\n \"reasoning_type\": [\n \"Spatial Relation Reasoning\"\n ]\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets phyx_mc \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['phyx_mc'],\n dataset_args={\n 'phyx_mc': {\n # subset_list: ['mechanics', 'electromagnetism', 'thermodynamics'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "f3018c9e52f5e2f7d3b16fba413a26d1", "needs_translation": false }, - "updated_at": "2026-08-17T18:10:17.672873", - "translation_updated_at": "2026-08-17T18:10:23" + "updated_at": "2026-08-20T10:40:24.366160", + "translation_updated_at": "2026-08-20T10:41:18" } diff --git a/evalscope/benchmarks/_meta/phyx_oe.json b/evalscope/benchmarks/_meta/phyx_oe.json index 3f5110965..63bb1d613 100644 --- a/evalscope/benchmarks/_meta/phyx_oe.json +++ b/evalscope/benchmarks/_meta/phyx_oe.json @@ -22,7 +22,7 @@ "optics", "modern_physics" ], - "description": "\n## Overview\n\nPhyX is the first large-scale benchmark for physical reasoning in realistic, visually grounded\nscenarios. This is its open-ended variant: no options are shown, so the model has to derive the\nanswer of a university-level physics problem from the figure and state it.\n\n## Task Description\n\n- **Task Type**: Visual open-ended physics problem solving\n- **Input**: A figure plus the problem description and question\n- **Output**: A step-by-step derivation ending in the final answer (value with unit or a formula)\n- **Domain**: University-level physics (mechanics, electromagnetism, thermodynamics, wave/acoustics,\n optics, modern physics)\n\n## Key Features\n\n- 3,000 university-level problems (`test`) over 6 core domains and 25 sub-domains, each domain exposed\n as its own subset; `eval_split='test_mini'` selects the official 1,000-problem testmini set.\n- Every problem is grounded in a figure that carries information the text does not restate, so the\n model must combine visual cues with implicit physical laws.\n- 6 reasoning types are represented (physical model grounding, multi-formula, spatial relation,\n numerical, predictive and implicit condition reasoning).\n- Uses the default *Text-DeRedundancy* input style of the paper: the simplified problem description\n plus the question, with the figure attached.\n\n- The official prompt is reproduced verbatim, including its request for step-by-step reasoning, so\n scores stay comparable with the published numbers.\n\n## Evaluation Notes\n\n- Primary metric: `acc`, mean over problems, reported overall and per domain.\n- The final answer is read from `\\boxed{...}`, else from a 'final answer:' / 'correct answer:'\n statement, else the whole reply is compared. A reply truncated before its answer therefore scores\n 0 for reasons unrelated to physics ability; give the model a generous `generation_config.max_tokens`.\n- Answers are free-form values with units, so an LLM judge is used by default (the official\n recommendation): run with `judge_strategy='auto'` or `'llm'` and provide `judge_model_args`. The\n judge is only consulted when the answer does not already match as a string.\n- `judge_strategy='rule'` falls back to the official string-level mode, which understates accuracy\n because equivalent spellings (`0.5 m` vs `50 cm`) do not match literally.\n- Figures are sent inline as base64 and the largest is ~5 MB; set `max_image_bytes` in `dataset_args`\n if the served model enforces a smaller per-image limit.\n- Resources: [Paper](https://arxiv.org/abs/2505.15929) | [GitHub](https://github.com/NastyMarcus/PhyX)\n | [Project page](https://killthefullmoon.github.io/projects/PhyX/index.html)\n", + "description": "\n## Overview\n\nPhyX is the first large-scale benchmark for physical reasoning in realistic, visually grounded\nscenarios. This is its open-ended variant: no options are shown, so the model has to derive the\nanswer of a university-level physics problem from the figure and state it.\n\n## Task Description\n\n- **Task Type**: Visual open-ended physics problem solving\n- **Input**: A figure plus the problem description and question\n- **Output**: A step-by-step derivation ending in the final answer (value with unit or a formula)\n- **Domain**: University-level physics (mechanics, electromagnetism, thermodynamics, wave/acoustics,\n optics, modern physics)\n\n## Key Features\n\n- 3,000 university-level problems (`test`) over 6 core domains and 25 sub-domains, each domain exposed\n as its own subset; `eval_split='test_mini'` selects the official 1,000-problem testmini set.\n- Every problem is grounded in a figure that carries information the text does not restate, so the\n model must combine visual cues with implicit physical laws.\n- 6 reasoning types are represented (physical model grounding, multi-formula, spatial relation,\n numerical, predictive and implicit condition reasoning).\n- Uses the default *Text-DeRedundancy* input style of the paper: the simplified problem description\n plus the question, with the figure attached.\n\n- The official prompt is reproduced verbatim, including its request for step-by-step reasoning, so\n scores stay comparable with the published numbers.\n\n## Evaluation Notes\n\n- Primary metric: `acc`, mean over problems, reported overall and per domain.\n- The final answer is read from `\\boxed{...}`, else from a 'final answer:' / 'correct answer:'\n statement, else the whole reply is compared. A reply truncated before its answer therefore scores\n 0 for reasons unrelated to physics ability; give the model a generous `generation_config.max_tokens`.\n- Answers are free-form values with units, so an LLM judge is used by default (the official\n recommendation): set `judge.strategy='auto'` or `'llm'` and provide `judge.models`. The\n judge is only consulted when the answer does not already match as a string.\n- `judge.strategy='rule'` falls back to the official string-level mode, which understates accuracy\n because equivalent spellings (`0.5 m` vs `50 cm`) do not match literally.\n- Figures are sent inline as base64 and the largest is ~5 MB; set `max_image_bytes` in `dataset_args`\n if the served model enforces a smaller per-image limit.\n- Resources: [Paper](https://arxiv.org/abs/2505.15929) | [GitHub](https://github.com/NastyMarcus/PhyX)\n | [Project page](https://killthefullmoon.github.io/projects/PhyX/index.html)\n", "prompt_template": "", "system_prompt": "", "few_shot_prompt_template": "", @@ -358,11 +358,11 @@ "truncated": false }, "readme": { - "en": "# PhyX-OE\n\n\n## Overview\n\nPhyX is the first large-scale benchmark for physical reasoning in realistic, visually grounded\nscenarios. This is its open-ended variant: no options are shown, so the model has to derive the\nanswer of a university-level physics problem from the figure and state it.\n\n## Task Description\n\n- **Task Type**: Visual open-ended physics problem solving\n- **Input**: A figure plus the problem description and question\n- **Output**: A step-by-step derivation ending in the final answer (value with unit or a formula)\n- **Domain**: University-level physics (mechanics, electromagnetism, thermodynamics, wave/acoustics,\n optics, modern physics)\n\n## Key Features\n\n- 3,000 university-level problems (`test`) over 6 core domains and 25 sub-domains, each domain exposed\n as its own subset; `eval_split='test_mini'` selects the official 1,000-problem testmini set.\n- Every problem is grounded in a figure that carries information the text does not restate, so the\n model must combine visual cues with implicit physical laws.\n- 6 reasoning types are represented (physical model grounding, multi-formula, spatial relation,\n numerical, predictive and implicit condition reasoning).\n- Uses the default *Text-DeRedundancy* input style of the paper: the simplified problem description\n plus the question, with the figure attached.\n\n- The official prompt is reproduced verbatim, including its request for step-by-step reasoning, so\n scores stay comparable with the published numbers.\n\n## Evaluation Notes\n\n- Primary metric: `acc`, mean over problems, reported overall and per domain.\n- The final answer is read from `\\boxed{...}`, else from a 'final answer:' / 'correct answer:'\n statement, else the whole reply is compared. A reply truncated before its answer therefore scores\n 0 for reasons unrelated to physics ability; give the model a generous `generation_config.max_tokens`.\n- Answers are free-form values with units, so an LLM judge is used by default (the official\n recommendation): run with `judge_strategy='auto'` or `'llm'` and provide `judge_model_args`. The\n judge is only consulted when the answer does not already match as a string.\n- `judge_strategy='rule'` falls back to the official string-level mode, which understates accuracy\n because equivalent spellings (`0.5 m` vs `50 cm`) do not match literally.\n- Figures are sent inline as base64 and the largest is ~5 MB; set `max_image_bytes` in `dataset_args`\n if the served model enforces a smaller per-image limit.\n- Resources: [Paper](https://arxiv.org/abs/2505.15929) | [GitHub](https://github.com/NastyMarcus/PhyX)\n | [Project page](https://killthefullmoon.github.io/projects/PhyX/index.html)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `phyx_oe` |\n| **Dataset ID** | [evalscope/PhyX](https://modelscope.cn/datasets/evalscope/PhyX/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2505.15929) |\n| **Tags** | `MultiModal`, `QA`, `Reasoning` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 3,000 |\n| Prompt Length (Mean) | 364.68 chars |\n| Prompt Length (Min/Max) | 93 / 1874 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `mechanics` | 550 | 356.92 | 124 | 1273 |\n| `electromagnetism` | 550 | 326.73 | 107 | 1032 |\n| `thermodynamics` | 500 | 390.86 | 93 | 1174 |\n| `waves_acoustics` | 500 | 379.95 | 101 | 1731 |\n| `optics` | 500 | 361.15 | 109 | 1215 |\n| `modern_physics` | 400 | 380.12 | 106 | 1874 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 3,000 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 215x46 - 5712x4953 |\n| Formats | jpeg, png |\n\n\n## Sample Example\n\n**Subset**: `mechanics`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"508a6723\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: png, ~35.6KB]\"\n },\n {\n \"text\": \"A patient with a dislocated shoulder is put into a traction apparatus as shown in figure. The pulls $\\\\vec{A}$ and $\\\\vec{B} must combine to produce an outward traction force of 12.8 N on the patient’s arm. How large should these pulls be? Please answer the question with step by step reasoning.\"\n }\n ]\n }\n ],\n \"target\": \"7.55N\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"index\": \"0\",\n \"category\": \"Mechanics\",\n \"subfield\": \"Statics\",\n \"reasoning_type\": [\n \"Spatial Relation Reasoning\"\n ]\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets phyx_oe \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['phyx_oe'],\n dataset_args={\n 'phyx_oe': {\n # subset_list: ['mechanics', 'electromagnetism', 'thermodynamics'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# PhyX-OE\n\n\n## 概述\n\nPhyX 是首个面向现实、视觉接地场景中物理推理的大规模基准测试。这是其开放式变体：不提供选项，模型必须从图示中推导出大学水平物理问题的答案并直接陈述。\n\n## 任务描述\n\n- **任务类型**：视觉开放式物理问题求解\n- **输入**：一张图示，加上问题描述和提问\n- **输出**：包含逐步推导过程并以最终答案（带单位的数值或公式）结尾的解答\n- **领域**：大学水平物理（力学、电磁学、热力学、波动/声学、光学、现代物理）\n\n## 核心特性\n\n- 包含 3,000 道大学水平题目（`test`），覆盖 6 个核心领域和 25 个子领域，每个领域作为独立子集提供；使用 `eval_split='test_mini'` 可选择官方指定的 1,000 题 testmini 测试集。\n- 每道题均基于一张图示，其中包含文本未重复说明的信息，因此模型必须结合视觉线索与隐含的物理定律进行推理。\n- 涵盖 6 种推理类型（物理模型接地、多公式、空间关系、数值、预测性及隐含条件推理）。\n- 采用论文默认的 *Text-DeRedundancy* 输入风格：简化后的问题描述加提问，并附上图示。\n\n- 官方提示词被逐字复现，包括对逐步推理的要求，以确保评分结果与已发表数据具有可比性。\n\n## 评估说明\n\n- 主要指标：`acc`，即各题目的平均准确率，整体及按领域分别报告。\n- 最终答案从 `\\boxed{...}` 中提取；若无，则从 'final answer:' / 'correct answer:' 语句中提取；否则将整个回复内容用于比对。若回复在给出答案前被截断，则得分为 0（此情况与模型物理能力无关）；请为模型设置足够大的 `generation_config.max_tokens`。\n- 答案为自由格式的带单位数值，因此默认使用 LLM 评判器（官方推荐）：运行时设置 `judge_strategy='auto'` 或 `'llm'` 并提供 `judge_model_args`。仅当答案字符串不完全匹配时才调用评判器。\n- 若使用 `judge_strategy='rule'`，则回退到官方的字符串级匹配模式，但由于等效写法（如 `0.5 m` 与 `50 cm`）无法字面匹配，会导致准确率被低估。\n- 图像以内联 base64 形式发送，最大约 5 MB；若所用模型对单张图像有更小的大小限制，请在 `dataset_args` 中设置 `max_image_bytes`。\n- 资源链接：[论文](https://arxiv.org/abs/2505.15929) | [GitHub](https://github.com/NastyMarcus/PhyX) | [项目主页](https://killthefullmoon.github.io/projects/PhyX/index.html)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `phyx_oe` |\n| **数据集ID** | [evalscope/PhyX](https://modelscope.cn/datasets/evalscope/PhyX/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2505.15929) |\n| **标签** | `MultiModal`, `QA`, `Reasoning` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 3,000 |\n| 提示词长度（平均） | 364.68 字符 |\n| 提示词长度（最小/最大） | 93 / 1874 字符 |\n\n**各子集统计：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `mechanics` | 550 | 356.92 | 124 | 1273 |\n| `electromagnetism` | 550 | 326.73 | 107 | 1032 |\n| `thermodynamics` | 500 | 390.86 | 93 | 1174 |\n| `waves_acoustics` | 500 | 379.95 | 101 | 1731 |\n| `optics` | 500 | 361.15 | 109 | 1215 |\n| `modern_physics` | 400 | 380.12 | 106 | 1874 |\n\n**图像统计：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 3,000 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 215x46 - 5712x4953 |\n| 格式 | jpeg, png |\n\n\n## 样例示例\n\n**子集**: `mechanics`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"508a6723\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: png, ~35.6KB]\"\n },\n {\n \"text\": \"A patient with a dislocated shoulder is put into a traction apparatus as shown in figure. The pulls $\\\\vec{A}$ and $\\\\vec{B} must combine to produce an outward traction force of 12.8 N on the patient’s arm. How large should these pulls be? Please answer the question with step by step reasoning.\"\n }\n ]\n }\n ],\n \"target\": \"7.55N\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"index\": \"0\",\n \"category\": \"Mechanics\",\n \"subfield\": \"Statics\",\n \"reasoning_type\": [\n \"Spatial Relation Reasoning\"\n ]\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets phyx_oe \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['phyx_oe'],\n dataset_args={\n 'phyx_oe': {\n # subset_list: ['mechanics', 'electromagnetism', 'thermodynamics'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "50a31f2db9c1c06125cf7eda836f2ec4", + "en": "# PhyX-OE\n\n\n## Overview\n\nPhyX is the first large-scale benchmark for physical reasoning in realistic, visually grounded\nscenarios. This is its open-ended variant: no options are shown, so the model has to derive the\nanswer of a university-level physics problem from the figure and state it.\n\n## Task Description\n\n- **Task Type**: Visual open-ended physics problem solving\n- **Input**: A figure plus the problem description and question\n- **Output**: A step-by-step derivation ending in the final answer (value with unit or a formula)\n- **Domain**: University-level physics (mechanics, electromagnetism, thermodynamics, wave/acoustics,\n optics, modern physics)\n\n## Key Features\n\n- 3,000 university-level problems (`test`) over 6 core domains and 25 sub-domains, each domain exposed\n as its own subset; `eval_split='test_mini'` selects the official 1,000-problem testmini set.\n- Every problem is grounded in a figure that carries information the text does not restate, so the\n model must combine visual cues with implicit physical laws.\n- 6 reasoning types are represented (physical model grounding, multi-formula, spatial relation,\n numerical, predictive and implicit condition reasoning).\n- Uses the default *Text-DeRedundancy* input style of the paper: the simplified problem description\n plus the question, with the figure attached.\n\n- The official prompt is reproduced verbatim, including its request for step-by-step reasoning, so\n scores stay comparable with the published numbers.\n\n## Evaluation Notes\n\n- Primary metric: `acc`, mean over problems, reported overall and per domain.\n- The final answer is read from `\\boxed{...}`, else from a 'final answer:' / 'correct answer:'\n statement, else the whole reply is compared. A reply truncated before its answer therefore scores\n 0 for reasons unrelated to physics ability; give the model a generous `generation_config.max_tokens`.\n- Answers are free-form values with units, so an LLM judge is used by default (the official\n recommendation): set `judge.strategy='auto'` or `'llm'` and provide `judge.models`. The\n judge is only consulted when the answer does not already match as a string.\n- `judge.strategy='rule'` falls back to the official string-level mode, which understates accuracy\n because equivalent spellings (`0.5 m` vs `50 cm`) do not match literally.\n- Figures are sent inline as base64 and the largest is ~5 MB; set `max_image_bytes` in `dataset_args`\n if the served model enforces a smaller per-image limit.\n- Resources: [Paper](https://arxiv.org/abs/2505.15929) | [GitHub](https://github.com/NastyMarcus/PhyX)\n | [Project page](https://killthefullmoon.github.io/projects/PhyX/index.html)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `phyx_oe` |\n| **Dataset ID** | [evalscope/PhyX](https://modelscope.cn/datasets/evalscope/PhyX/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2505.15929) |\n| **Tags** | `MultiModal`, `QA`, `Reasoning` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 3,000 |\n| Prompt Length (Mean) | 364.68 chars |\n| Prompt Length (Min/Max) | 93 / 1874 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `mechanics` | 550 | 356.92 | 124 | 1273 |\n| `electromagnetism` | 550 | 326.73 | 107 | 1032 |\n| `thermodynamics` | 500 | 390.86 | 93 | 1174 |\n| `waves_acoustics` | 500 | 379.95 | 101 | 1731 |\n| `optics` | 500 | 361.15 | 109 | 1215 |\n| `modern_physics` | 400 | 380.12 | 106 | 1874 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 3,000 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 215x46 - 5712x4953 |\n| Formats | jpeg, png |\n\n\n## Sample Example\n\n**Subset**: `mechanics`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"508a6723\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: png, ~35.6KB]\"\n },\n {\n \"text\": \"A patient with a dislocated shoulder is put into a traction apparatus as shown in figure. The pulls $\\\\vec{A}$ and $\\\\vec{B} must combine to produce an outward traction force of 12.8 N on the patient’s arm. How large should these pulls be? Please answer the question with step by step reasoning.\"\n }\n ]\n }\n ],\n \"target\": \"7.55N\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"index\": \"0\",\n \"category\": \"Mechanics\",\n \"subfield\": \"Statics\",\n \"reasoning_type\": [\n \"Spatial Relation Reasoning\"\n ]\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets phyx_oe \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['phyx_oe'],\n dataset_args={\n 'phyx_oe': {\n # subset_list: ['mechanics', 'electromagnetism', 'thermodynamics'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# PhyX-OE\n\n\n## 概述\n\nPhyX 是首个面向现实、视觉接地场景中物理推理的大规模基准测试。这是其开放式变体：不提供选项，模型必须从图示中推导出大学水平物理问题的答案并明确陈述。\n\n## 任务描述\n\n- **任务类型**：视觉开放式物理问题求解\n- **输入**：一张图示，加上问题描述和提问\n- **输出**：包含逐步推导过程并以最终答案（带单位的数值或公式）结尾的解答\n- **领域**：大学水平物理（力学、电磁学、热力学、波动/声学、光学、现代物理）\n\n## 关键特性\n\n- 包含 3,000 道大学水平题目（`test`），覆盖 6 个核心领域和 25 个子领域，每个领域作为独立子集提供；设置 `eval_split='test_mini'` 可选择官方的 1,000 题 testmini 测试集。\n- 每道题均基于一张图示，其中包含文本未重复说明的信息，因此模型必须结合视觉线索与隐含的物理定律进行推理。\n- 涵盖 6 种推理类型（物理模型接地、多公式、空间关系、数值、预测性及隐含条件推理）。\n- 采用论文默认的 *Text-DeRedundancy* 输入风格：简化后的问题描述加提问，并附上图示。\n\n- 官方提示词被逐字复现，包括对逐步推理的要求，以确保评分结果与已发表数据具有可比性。\n\n## 评估说明\n\n- 主要指标：`acc`，即各题目的平均准确率，整体及按领域分别报告。\n- 最终答案从 `\\boxed{...}` 中提取；若无，则从 'final answer:' / 'correct answer:' 语句中提取；若仍无，则将整个回复与标准答案比较。若回复在给出答案前被截断，则得分为 0（此情况与模型物理能力无关）；请为模型设置足够大的 `generation_config.max_tokens`。\n- 答案为自由格式的带单位数值，因此默认使用 LLM 评判器（官方推荐）：设置 `judge.strategy='auto'` 或 `'llm'` 并提供 `judge.models`。仅当答案字符串不完全匹配时才调用评判器。\n- 若设置 `judge.strategy='rule'`，则回退到官方的字符串级匹配模式，但会低估准确率，因为等效表达（如 `0.5 m` 与 `50 cm`）在字面上并不相同。\n- 图像以内联 base64 形式发送，最大约 5 MB；若所用模型对单张图像有更小的大小限制，请在 `dataset_args` 中设置 `max_image_bytes`。\n- 资源链接：[论文](https://arxiv.org/abs/2505.15929) | [GitHub](https://github.com/NastyMarcus/PhyX) | [项目主页](https://killthefullmoon.github.io/projects/PhyX/index.html)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `phyx_oe` |\n| **数据集ID** | [evalscope/PhyX](https://modelscope.cn/datasets/evalscope/PhyX/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2505.15929) |\n| **标签** | `MultiModal`, `QA`, `Reasoning` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 3,000 |\n| 提示词长度（平均） | 364.68 字符 |\n| 提示词长度（最小/最大） | 93 / 1874 字符 |\n\n**各子集统计：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `mechanics` | 550 | 356.92 | 124 | 1273 |\n| `electromagnetism` | 550 | 326.73 | 107 | 1032 |\n| `thermodynamics` | 500 | 390.86 | 93 | 1174 |\n| `waves_acoustics` | 500 | 379.95 | 101 | 1731 |\n| `optics` | 500 | 361.15 | 109 | 1215 |\n| `modern_physics` | 400 | 380.12 | 106 | 1874 |\n\n**图像统计：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 3,000 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 215x46 - 5712x4953 |\n| 格式 | jpeg, png |\n\n\n## 样例示例\n\n**子集**: `mechanics`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"508a6723\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: png, ~35.6KB]\"\n },\n {\n \"text\": \"A patient with a dislocated shoulder is put into a traction apparatus as shown in figure. The pulls $\\\\vec{A}$ and $\\\\vec{B} must combine to produce an outward traction force of 12.8 N on the patient’s arm. How large should these pulls be? Please answer the question with step by step reasoning.\"\n }\n ]\n }\n ],\n \"target\": \"7.55N\",\n \"id\": 0,\n \"group_id\": 0,\n \"metadata\": {\n \"index\": \"0\",\n \"category\": \"Mechanics\",\n \"subfield\": \"Statics\",\n \"reasoning_type\": [\n \"Spatial Relation Reasoning\"\n ]\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets phyx_oe \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['phyx_oe'],\n dataset_args={\n 'phyx_oe': {\n # subset_list: ['mechanics', 'electromagnetism', 'thermodynamics'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "51b52a4303f836696ca863ac382e9788", "needs_translation": false }, - "updated_at": "2026-08-17T18:10:10.919722", - "translation_updated_at": "2026-08-17T18:10:23" + "updated_at": "2026-08-20T10:40:24.367691", + "translation_updated_at": "2026-08-20T10:41:18" } diff --git a/evalscope/benchmarks/_meta/plawbench.json b/evalscope/benchmarks/_meta/plawbench.json index 7465a87a7..f318cd968 100644 --- a/evalscope/benchmarks/_meta/plawbench.json +++ b/evalscope/benchmarks/_meta/plawbench.json @@ -16,11 +16,6 @@ "reasoning_acc", "law_acc" ], - "primary_metric": { - "name": "accuracy", - "aggregation": null, - "dimensions": {} - }, "few_shot_num": 0, "eval_split": "test", "train_split": "", @@ -30,20 +25,19 @@ "plaintiff_statement", "defendant_statement" ], - "description": "\n## Overview\n\nPLawBench is a rubric-based benchmark that evaluates large language models on real-world Chinese legal practice.\nIt mirrors the workflow of a practising lawyer across three hierarchical levels: eliciting facts during a public\nlegal consultation, analysing a case with structured legal reasoning, and drafting professional legal documents.\nEvery item ships with a rubric annotated by legal experts, and grading is performed by an LLM judge against that\nrubric rather than against a single reference answer.\n\n## Task Description\n\n- **Task Type**: Open-ended Chinese legal generation graded with expert rubrics\n- **Input**: A client statement, or a case description plus a legal question\n- **Output**: A question list, a structured case analysis, or a full legal document\n- **Domain**: Chinese legal practice (personal affairs, marriage and family, corporate governance, intellectual\n property, criminal and civil litigation, cross-border matters, labour, environmental safety, and more)\n\n## Key Features\n\n- 280 samples split into four subsets, one per PLawBench task:\n - `case_analysis` (250): case analysis scored on four dimensions — conclusion, case facts, reasoning, and cited\n statutes. Answers must follow the 【结论】/【案件事实】/【推理过程】/【法条依据】 structure.\n - `legal_consultation` (18): the model plays a lawyer and must produce 10-25 verifiable follow-up questions that\n surface the facts the client omitted or distorted.\n - `plaintiff_statement` (6): drafting a statement of complaint from the client's account.\n - `defendant_statement` (6): drafting a statement of defense from the client's account and the opposing complaint.\n- Client statements are deliberately vague, emotional, or misleading, so models must detect traps instead of\n restating the client's claims.\n- Task prompts and judge prompts are ported verbatim from the official release, and the `case_analysis` rubric\n retains its per-dimension point allocation.\n\n## Evaluation Notes\n\n- Requires an LLM judge: run with `judge_strategy='llm'` (or `'auto'`, which enables the judge for this benchmark)\n and provide `judge_model_args`. `judge_strategy='rule'` is not supported.\n- Metrics are point ratios in `[0, 1]`. `acc` is reported for every subset; `case_analysis` additionally reports\n `conclusion_acc`, `fact_acc`, `reasoning_acc`, and `law_acc`. These map one-to-one onto the official leaderboard\n columns: `legal_consultation` is Task1, `case_analysis` is Task2-Avg with its four dimensions, and the two\n drafting subsets are Task3-Plaintiff and Task3-Defendant.\n- Compare per-subset scores, not the `OVERALL` row. `OVERALL` is a per-sample mean, so `case_analysis` dominates it\n (250 of 280 samples). The paper's `Overall` column is an equal-weighted mean of the three task scores, which\n matches its published table far more closely (mean absolute error 0.72 versus 2.87 for a sample-weighted mean,\n fitted across the 24 models in the official ranking).\n- Rubric point totals come from the dataset, not from the judge output, and awarded points are clamped into\n `[0, max_points]`, so a judge that mis-reports the denominator cannot distort the score.\n- The judge output template for `case_analysis` is repaired relative to the official script, which ships malformed\n JSON and pins the conclusion section to zero points; every section is graded on its rubric allocation here.\n- Judge requests are retried up to `judge_retries` times when the response cannot be parsed; a sample that still\n fails is scored 0 and flagged via `judge_failed` in the review metadata.\n- Case-analysis judging returns a long per-item breakdown. Give the judge a generous `max_tokens`\n (for example 8192) in `judge_model_args.generation_config`.\n- The drafting subsets ask for a 2,500-3,000 character legal document, so the evaluated model also needs a generous\n `generation_config.max_tokens`. A truncated filing is graded as an incomplete document and scores near zero, which\n depresses Task3 for reasons unrelated to legal ability.\n\nResources: [GitHub](https://github.com/skylenage/PLawbench) |\n[Dataset](https://modelscope.cn/datasets/evalscope/PLawBench)\n", + "description": "\n## Overview\n\nPLawBench is a rubric-based benchmark that evaluates large language models on real-world Chinese legal practice.\nIt mirrors the workflow of a practising lawyer across three hierarchical levels: eliciting facts during a public\nlegal consultation, analysing a case with structured legal reasoning, and drafting professional legal documents.\nEvery item ships with a rubric annotated by legal experts, and grading is performed by an LLM judge against that\nrubric rather than against a single reference answer.\n\n## Task Description\n\n- **Task Type**: Open-ended Chinese legal generation graded with expert rubrics\n- **Input**: A client statement, or a case description plus a legal question\n- **Output**: A question list, a structured case analysis, or a full legal document\n- **Domain**: Chinese legal practice (personal affairs, marriage and family, corporate governance, intellectual\n property, criminal and civil litigation, cross-border matters, labour, environmental safety, and more)\n\n## Key Features\n\n- 280 samples split into four subsets, one per PLawBench task:\n - `case_analysis` (250): case analysis scored on four dimensions — conclusion, case facts, reasoning, and cited\n statutes. Answers must follow the 【结论】/【案件事实】/【推理过程】/【法条依据】 structure.\n - `legal_consultation` (18): the model plays a lawyer and must produce 10-25 verifiable follow-up questions that\n surface the facts the client omitted or distorted.\n - `plaintiff_statement` (6): drafting a statement of complaint from the client's account.\n - `defendant_statement` (6): drafting a statement of defense from the client's account and the opposing complaint.\n- Client statements are deliberately vague, emotional, or misleading, so models must detect traps instead of\n restating the client's claims.\n- Task prompts and judge prompts are ported verbatim from the official release, and the `case_analysis` rubric\n retains its per-dimension point allocation.\n\n## Evaluation Notes\n\n- Requires an LLM judge: set `judge.strategy='llm'` (or `'auto'`, which enables the judge for this benchmark)\n and provide `judge.models`. `judge.strategy='rule'` is not supported.\n- Metrics are point ratios in `[0, 1]`. `acc` is reported for every subset; `case_analysis` additionally reports\n `conclusion_acc`, `fact_acc`, `reasoning_acc`, and `law_acc`. These map one-to-one onto the official leaderboard\n columns: `legal_consultation` is Task1, `case_analysis` is Task2-Avg with its four dimensions, and the two\n drafting subsets are Task3-Plaintiff and Task3-Defendant.\n- Compare per-subset scores, not the `OVERALL` row. `OVERALL` is a per-sample mean, so `case_analysis` dominates it\n (250 of 280 samples). The paper's `Overall` column is an equal-weighted mean of the three task scores, which\n matches its published table far more closely (mean absolute error 0.72 versus 2.87 for a sample-weighted mean,\n fitted across the 24 models in the official ranking).\n- Rubric point totals come from the dataset, not from the judge output, and awarded points are clamped into\n `[0, max_points]`, so a judge that mis-reports the denominator cannot distort the score.\n- The judge output template for `case_analysis` is repaired relative to the official script, which ships malformed\n JSON and pins the conclusion section to zero points; every section is graded on its rubric allocation here.\n- The judge model's transport retry policy is configured through its `generation_config`. A reply that still\n fails the output contract is unavailable and excluded rather than silently scored as zero.\n- Case-analysis judging returns a long per-item breakdown. Give the judge a generous `max_tokens`\n (for example 8192) in `judge.models[].generation_config`.\n- The drafting subsets ask for a 2,500-3,000 character legal document, so the evaluated model also needs a generous\n `generation_config.max_tokens`. A truncated filing is graded as an incomplete document and scores near zero, which\n depresses Task3 for reasons unrelated to legal ability.\n\nResources: [GitHub](https://github.com/skylenage/PLawbench) |\n[Dataset](https://modelscope.cn/datasets/evalscope/PLawBench)\n", "prompt_template": "{question}", "system_prompt": "", "few_shot_prompt_template": "", "aggregation": "mean", - "extra_params": { - "judge_retries": { - "type": "int", - "description": "Maximum attempts per rubric judge request before the sample is scored as 0.", - "value": 3 - } - }, + "extra_params": {}, "sandbox_config": {}, - "category": "llm" + "category": "llm", + "primary_metric": { + "name": "accuracy", + "aggregation": null, + "dimensions": {} + } }, "statistics": { "total_samples": 280, @@ -120,11 +114,11 @@ "truncated": true }, "readme": { - "en": "# PLawBench\n\n\n## Overview\n\nPLawBench is a rubric-based benchmark that evaluates large language models on real-world Chinese legal practice.\nIt mirrors the workflow of a practising lawyer across three hierarchical levels: eliciting facts during a public\nlegal consultation, analysing a case with structured legal reasoning, and drafting professional legal documents.\nEvery item ships with a rubric annotated by legal experts, and grading is performed by an LLM judge against that\nrubric rather than against a single reference answer.\n\n## Task Description\n\n- **Task Type**: Open-ended Chinese legal generation graded with expert rubrics\n- **Input**: A client statement, or a case description plus a legal question\n- **Output**: A question list, a structured case analysis, or a full legal document\n- **Domain**: Chinese legal practice (personal affairs, marriage and family, corporate governance, intellectual\n property, criminal and civil litigation, cross-border matters, labour, environmental safety, and more)\n\n## Key Features\n\n- 280 samples split into four subsets, one per PLawBench task:\n - `case_analysis` (250): case analysis scored on four dimensions — conclusion, case facts, reasoning, and cited\n statutes. Answers must follow the 【结论】/【案件事实】/【推理过程】/【法条依据】 structure.\n - `legal_consultation` (18): the model plays a lawyer and must produce 10-25 verifiable follow-up questions that\n surface the facts the client omitted or distorted.\n - `plaintiff_statement` (6): drafting a statement of complaint from the client's account.\n - `defendant_statement` (6): drafting a statement of defense from the client's account and the opposing complaint.\n- Client statements are deliberately vague, emotional, or misleading, so models must detect traps instead of\n restating the client's claims.\n- Task prompts and judge prompts are ported verbatim from the official release, and the `case_analysis` rubric\n retains its per-dimension point allocation.\n\n## Evaluation Notes\n\n- Requires an LLM judge: run with `judge_strategy='llm'` (or `'auto'`, which enables the judge for this benchmark)\n and provide `judge_model_args`. `judge_strategy='rule'` is not supported.\n- Metrics are point ratios in `[0, 1]`. `acc` is reported for every subset; `case_analysis` additionally reports\n `conclusion_acc`, `fact_acc`, `reasoning_acc`, and `law_acc`. These map one-to-one onto the official leaderboard\n columns: `legal_consultation` is Task1, `case_analysis` is Task2-Avg with its four dimensions, and the two\n drafting subsets are Task3-Plaintiff and Task3-Defendant.\n- Compare per-subset scores, not the `OVERALL` row. `OVERALL` is a per-sample mean, so `case_analysis` dominates it\n (250 of 280 samples). The paper's `Overall` column is an equal-weighted mean of the three task scores, which\n matches its published table far more closely (mean absolute error 0.72 versus 2.87 for a sample-weighted mean,\n fitted across the 24 models in the official ranking).\n- Rubric point totals come from the dataset, not from the judge output, and awarded points are clamped into\n `[0, max_points]`, so a judge that mis-reports the denominator cannot distort the score.\n- The judge output template for `case_analysis` is repaired relative to the official script, which ships malformed\n JSON and pins the conclusion section to zero points; every section is graded on its rubric allocation here.\n- Judge requests are retried up to `judge_retries` times when the response cannot be parsed; a sample that still\n fails is scored 0 and flagged via `judge_failed` in the review metadata.\n- Case-analysis judging returns a long per-item breakdown. Give the judge a generous `max_tokens`\n (for example 8192) in `judge_model_args.generation_config`.\n- The drafting subsets ask for a 2,500-3,000 character legal document, so the evaluated model also needs a generous\n `generation_config.max_tokens`. A truncated filing is graded as an incomplete document and scores near zero, which\n depresses Task3 for reasons unrelated to legal ability.\n\nResources: [GitHub](https://github.com/skylenage/PLawbench) |\n[Dataset](https://modelscope.cn/datasets/evalscope/PLawBench)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `plawbench` |\n| **Dataset ID** | [evalscope/PLawBench](https://modelscope.cn/datasets/evalscope/PLawBench/summary) |\n| **Paper** | [Paper](https://github.com/skylenage/PLawbench) |\n| **Tags** | `Chinese`, `Knowledge`, `QA`, `Reasoning` |\n| **Metrics** | `accuracy`, `conclusion_acc`, `fact_acc`, `reasoning_acc`, `law_acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 280 |\n| Prompt Length (Mean) | 2669.88 chars |\n| Prompt Length (Min/Max) | 1267 / 5890 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `case_analysis` | 250 | 2720.18 | 2137 | 4873 |\n| `legal_consultation` | 18 | 1851.94 | 1512 | 2611 |\n| `plaintiff_statement` | 6 | 1494.67 | 1267 | 2050 |\n| `defendant_statement` | 6 | 4203 | 2794 | 5890 |\n\n## Sample Example\n\n**Subset**: `case_analysis`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"06c8a1d7\",\n \"content\": \"\\n## 角色\\n\\n你是一名具有十年以上执业经验的法律实务专家，精通中国现行法律法规与司法实践。你擅长将复杂的法律问题分解为清晰的逻辑模块，并严格依据“结论先行、事实为重、推理严密、依据支撑”的专业风格进行解答。\\n\\n## 核心要求\\n\\n1. 严格顺序：回答必须按照以下四部分顺序展开，并使用对应标题：\\n【结论】\\n【案件事实】\\n【推理过程】\\n【法条依据】\\n2. 内容规范：\\n结论：直接、明确，针对提问的核心争议点给出肯定或否定的判断。\\n案件事实：基于用户提供的案情，简明、客观地摘录与法律判断 ... [TRUNCATED 1912 chars] ... 并赔偿精神损害抚慰金。庭审中查明，某摄影服务公司已完成除摄像外的其他服务项目；某文化传媒公司系独立法人，其工作人员在操作设备时存在重大过失。另查，某甲在签订合同时未特别声明婚礼录像的重要性，但合同附件中列有\\\"全程跟拍记录\\\"服务项目。某摄影服务公司辩称其仅需承担合同违约责任，精神损害赔偿缺乏依据。某文化传媒公司以非合同相对方为由拒绝承担责任。\\n\\n## 问题\\n以【结论 + 案情简述 + 分析过程+依据法条】的逻辑回答以下问题：在上述案例中，某甲能否向某摄影服务公司主张精神损害赔偿？\\n\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"case_analysis\",\n \"metadata\": {\n \"id\": \"case_analysis-1\",\n \"task\": \"case_analysis\",\n \"judge_type\": \"case_analysis\",\n \"category\": \"个人生活\",\n \"rubrics\": \"[{\\\"criterion\\\": \\\"【结论得分】\\\\n(+5分) 某甲有权向某摄影服务公司主张精神损害赔偿。\\\", \\\"points\\\": \\\"5\\\", \\\"tags\\\": \\\"结论得分\\\"}, {\\\"criterion\\\": \\\"【案情简述得分】\\\\n(+5分) 某甲与某摄影服务公司签订《婚庆服务合同》，并支付全款，合同附件明确包含\\\\\\\"全程跟拍记录\\\\\\\"服务项目。\\\\n(+5分) 婚礼当日，某摄影服务公司未经告知将摄像服务转包给文化传媒公司。\\\\n(+5分) 文化传媒公司工作室将录像全部丢失，未能交付原告。\\\\n(+ ... [TRUNCATED 762 chars] ... 人具有人身意义的特定物造成严重精神损害的，被侵权人有权请求精神损害赔偿。\\\\n（+5分）《最高人民法院关于确定民事侵权精神损害赔偿责任若干问题的解释》第五条\\\\n精神损害的赔偿数额根据以下因素确定：(一)侵权人的过错程度，但是法律另有规定的除外；(二)侵权行为的目的、方式、场合等具体情节；(三)侵权行为所造成的后果；(四)侵权人的获利情况；(五)侵权人承担责任的经济能力；(六)受理诉讼法院所在地的平均生活水平。\\\", \\\"points\\\": \\\"15\\\", \\\"tags\\\": \\\"法条依据得分\\\"}]\",\n \"max_points\": 60,\n \"prompt\": \"\\n## 角色\\n\\n你是一名具有十年以上执业经验的法律实务专家，精通中国现行法律法规与司法实践。你擅长将复杂的法律问题分解为清晰的逻辑模块，并严格依据“结论先行、事实为重、推理严密、依据支撑”的专业风格进行解答。\\n\\n## 核心要求\\n\\n1. 严格顺序：回答必须按照以下四部分顺序展开，并使用对应标题：\\n【结论】\\n【案件事实】\\n【推理过程】\\n【法条依据】\\n2. 内容规范：\\n结论：直接、明确，针对提问的核心争议点给出肯定或否定的判断。\\n案件事实：基于用户提供的案情，简明、客观地摘录与法律判断 ... [TRUNCATED 1912 chars] ... 并赔偿精神损害抚慰金。庭审中查明，某摄影服务公司已完成除摄像外的其他服务项目；某文化传媒公司系独立法人，其工作人员在操作设备时存在重大过失。另查，某甲在签订合同时未特别声明婚礼录像的重要性，但合同附件中列有\\\"全程跟拍记录\\\"服务项目。某摄影服务公司辩称其仅需承担合同违约责任，精神损害赔偿缺乏依据。某文化传媒公司以非合同相对方为由拒绝承担责任。\\n\\n## 问题\\n以【结论 + 案情简述 + 分析过程+依据法条】的逻辑回答以下问题：在上述案例中，某甲能否向某摄影服务公司主张精神损害赔偿？\\n\"\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `judge_retries` | `int` | `3` | Maximum attempts per rubric judge request before the sample is scored as 0. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets plawbench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['plawbench'],\n dataset_args={\n 'plawbench': {\n # subset_list: ['case_analysis', 'legal_consultation', 'plaintiff_statement'] # optional, evaluate specific subsets\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# PLawBench\n\n\n## 概述\n\nPLawBench 是一个基于评分标准（rubric）的基准测试，用于评估大语言模型在真实中国法律实务场景中的表现。 \n该基准模拟执业律师的工作流程，涵盖三个层级的任务：在公共法律咨询中梳理事实、通过结构化法律推理分析案件，以及起草专业法律文书。 \n每个样本均附有法律专家标注的评分标准，评分由 LLM 裁判模型依据该评分标准进行，而非与单一参考答案比对。\n\n## 任务描述\n\n- **任务类型**：开放式中文法律生成任务，使用专家制定的评分标准进行评分\n- **输入**：客户陈述，或案件描述加法律问题\n- **输出**：问题清单、结构化案件分析，或完整的法律文书\n- **领域**：中国法律实务（个人事务、婚姻家庭、公司治理、知识产权、刑事与民事诉讼、跨境事务、劳动、环境安全等）\n\n## 核心特性\n\n- 共 280 个样本，划分为四个子集，对应 PLawBench 的四项任务：\n - `case_analysis`（250 个）：案件分析，从四个维度评分——结论、案件事实、推理过程、引用法条。回答必须遵循【结论】/【案件事实】/【推理过程】/【法条依据】的结构。\n - `legal_consultation`（18 个）：模型扮演律师，需提出 10–25 个可验证的后续问题，以揭示客户遗漏或歪曲的事实。\n - `plaintiff_statement`（6 个）：根据客户陈述起草起诉状。\n - `defendant_statement`（6 个）：根据客户陈述及对方起诉状起草答辩状。\n- 客户陈述故意设计得模糊、情绪化或具有误导性，因此模型必须识别陷阱，而非简单复述客户主张。\n- 任务提示词和裁判提示词直接沿用官方发布版本，`case_analysis` 的评分标准保留了各维度的分值分配。\n\n## 评估说明\n\n- 需要 LLM 裁判模型：运行时需设置 `judge_strategy='llm'`（或 `'auto'`，该选项会为此基准自动启用裁判模型），并提供 `judge_model_args`。不支持 `judge_strategy='rule'`。\n- 指标为 `[0, 1]` 区间内的得分比例。每个子集均报告 `acc`；`case_analysis` 额外报告 `conclusion_acc`、`fact_acc`、`reasoning_acc` 和 `law_acc`。这些指标与官方排行榜列一一对应：`legal_consultation` 对应 Task1，`case_analysis` 对应 Task2-Avg 及其四个维度，两个文书起草子集分别对应 Task3-Plaintiff 和 Task3-Defendant。\n- 应比较各子集得分，而非 `OVERALL` 行。`OVERALL` 是按样本数量计算的平均值，因此被 `case_analysis` 主导（250/280 个样本）。论文中的 `Overall` 列是三项任务得分的等权重平均值，与官方表格更吻合（在官方排名的 24 个模型上拟合，平均绝对误差为 0.72，而样本加权平均的误差为 2.87）。\n- 评分标准的总分来自数据集本身，而非裁判模型输出，且实际得分会被限制在 `[0, max_points]` 范围内，因此即使裁判模型错误报告分母，也不会扭曲最终得分。\n- 相较于官方脚本，本实现修复了 `case_analysis` 的裁判输出模板。官方脚本输出格式错误的 JSON，并将结论部分固定为零分；此处每个部分均按其评分标准分配进行评分。\n- 当裁判响应无法解析时，会重试最多 `judge_retries` 次；若仍失败，则该样本得分为 0，并在评审元数据中标记 `judge_failed`。\n- 案件分析的裁判会返回详细的逐项分解。建议在 `judge_model_args.generation_config` 中为裁判模型设置较大的 `max_tokens`（例如 8192）。\n- 文书起草子集要求生成 2,500–3,000 字符的法律文书，因此被评估模型也需要较大的 `generation_config.max_tokens`。若生成内容被截断，将被视为不完整文书，得分接近零，从而因非法律能力原因拉低 Task3 得分。\n\n资源：[GitHub](https://github.com/skylenage/PLawbench) |\n[数据集](https://modelscope.cn/datasets/evalscope/PLawBench)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `plawbench` |\n| **数据集ID** | [evalscope/PLawBench](https://modelscope.cn/datasets/evalscope/PLawBench/summary) |\n| **论文** | [Paper](https://github.com/skylenage/PLawbench) |\n| **标签** | `Chinese`, `Knowledge`, `QA`, `Reasoning` |\n| **指标** | `accuracy`, `conclusion_acc`, `fact_acc`, `reasoning_acc`, `law_acc` |\n| **默认示例数** | 0-shot |\n| **评估分割** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 280 |\n| 提示词长度（平均） | 2669.88 字符 |\n| 提示词长度（最小/最大） | 1267 / 5890 字符 |\n\n**各子集统计：**\n\n| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 |\n|--------|---------|-------------|------------|------------|\n| `case_analysis` | 250 | 2720.18 | 2137 | 4873 |\n| `legal_consultation` | 18 | 1851.94 | 1512 | 2611 |\n| `plaintiff_statement` | 6 | 1494.67 | 1267 | 2050 |\n| `defendant_statement` | 6 | 4203 | 2794 | 5890 |\n\n## 样例示例\n\n**子集**: `case_analysis`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"06c8a1d7\",\n \"content\": \"\\n## 角色\\n\\n你是一名具有十年以上执业经验的法律实务专家，精通中国现行法律法规与司法实践。你擅长将复杂的法律问题分解为清晰的逻辑模块，并严格依据“结论先行、事实为重、推理严密、依据支撑”的专业风格进行解答。\\n\\n## 核心要求\\n\\n1. 严格顺序：回答必须按照以下四部分顺序展开，并使用对应标题：\\n【结论】\\n【案件事实】\\n【推理过程】\\n【法条依据】\\n2. 内容规范：\\n结论：直接、明确，针对提问的核心争议点给出肯定或否定的判断。\\n案件事实：基于用户提供的案情，简明、客观地摘录与法律判断 ... [TRUNCATED 1912 chars] ... 并赔偿精神损害抚慰金。庭审中查明，某摄影服务公司已完成除摄像外的其他服务项目；某文化传媒公司系独立法人，其工作人员在操作设备时存在重大过失。另查，某甲在签订合同时未特别声明婚礼录像的重要性，但合同附件中列有\\\"全程跟拍记录\\\"服务项目。某摄影服务公司辩称其仅需承担合同违约责任，精神损害赔偿缺乏依据。某文化传媒公司以非合同相对方为由拒绝承担责任。\\n\\n## 问题\\n以【结论 + 案情简述 + 分析过程+依据法条】的逻辑回答以下问题：在上述案例中，某甲能否向某摄影服务公司主张精神损害赔偿？\\n\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"case_analysis\",\n \"metadata\": {\n \"id\": \"case_analysis-1\",\n \"task\": \"case_analysis\",\n \"judge_type\": \"case_analysis\",\n \"category\": \"个人生活\",\n \"rubrics\": \"[{\\\"criterion\\\": \\\"【结论得分】\\\\n(+5分) 某甲有权向某摄影服务公司主张精神损害赔偿。\\\", \\\"points\\\": \\\"5\\\", \\\"tags\\\": \\\"结论得分\\\"}, {\\\"criterion\\\": \\\"【案情简述得分】\\\\n(+5分) 某甲与某摄影服务公司签订《婚庆服务合同》，并支付全款，合同附件明确包含\\\\\\\"全程跟拍记录\\\\\\\"服务项目。\\\\n(+5分) 婚礼当日，某摄影服务公司未经告知将摄像服务转包给文化传媒公司。\\\\n(+5分) 文化传媒公司工作室将录像全部丢失，未能交付原告。\\\\n(+ ... [TRUNCATED 762 chars] ... 人具有人身意义的特定物造成严重精神损害的，被侵权人有权请求精神损害赔偿。\\\\n（+5分）《最高人民法院关于确定民事侵权精神损害赔偿责任若干问题的解释》第五条\\\\n精神损害的赔偿数额根据以下因素确定：(一)侵权人的过错程度，但是法律另有规定的除外；(二)侵权行为的目的、方式、场合等具体情节；(三)侵权行为所造成的后果；(四)侵权人的获利情况；(五)侵权人承担责任的经济能力；(六)受理诉讼法院所在地的平均生活水平。\\\", \\\"points\\\": \\\"15\\\", \\\"tags\\\": \\\"法条依据得分\\\"}]\",\n \"max_points\": 60,\n \"prompt\": \"\\n## 角色\\n\\n你是一名具有十年以上执业经验的法律实务专家，精通中国现行法律法规与司法实践。你擅长将复杂的法律问题分解为清晰的逻辑模块，并严格依据“结论先行、事实为重、推理严密、依据支撑”的专业风格进行解答。\\n\\n## 核心要求\\n\\n1. 严格顺序：回答必须按照以下四部分顺序展开，并使用对应标题：\\n【结论】\\n【案件事实】\\n【推理过程】\\n【法条依据】\\n2. 内容规范：\\n结论：直接、明确，针对提问的核心争议点给出肯定或否定的判断。\\n案件事实：基于用户提供的案情，简明、客观地摘录与法律判断 ... [TRUNCATED 1912 chars] ... 并赔偿精神损害抚慰金。庭审中查明，某摄影服务公司已完成除摄像外的其他服务项目；某文化传媒公司系独立法人，其工作人员在操作设备时存在重大过失。另查，某甲在签订合同时未特别声明婚礼录像的重要性，但合同附件中列有\\\"全程跟拍记录\\\"服务项目。某摄影服务公司辩称其仅需承担合同违约责任，精神损害赔偿缺乏依据。某文化传媒公司以非合同相对方为由拒绝承担责任。\\n\\n## 问题\\n以【结论 + 案情简述 + 分析过程+依据法条】的逻辑回答以下问题：在上述案例中，某甲能否向某摄影服务公司主张精神损害赔偿？\\n\"\n }\n}\n```\n\n*注：部分内容因显示需要已被截断。*\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `judge_retries` | `int` | `3` | 每次评分标准裁判请求的最大重试次数，超过后该样本得分为 0。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets plawbench \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['plawbench'],\n dataset_args={\n 'plawbench': {\n # subset_list: ['case_analysis', 'legal_consultation', 'plaintiff_statement'] # 可选，用于评估指定子集\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "6747f9433ef93a1863e3befbe222b883", + "en": "# PLawBench\n\n\n## Overview\n\nPLawBench is a rubric-based benchmark that evaluates large language models on real-world Chinese legal practice.\nIt mirrors the workflow of a practising lawyer across three hierarchical levels: eliciting facts during a public\nlegal consultation, analysing a case with structured legal reasoning, and drafting professional legal documents.\nEvery item ships with a rubric annotated by legal experts, and grading is performed by an LLM judge against that\nrubric rather than against a single reference answer.\n\n## Task Description\n\n- **Task Type**: Open-ended Chinese legal generation graded with expert rubrics\n- **Input**: A client statement, or a case description plus a legal question\n- **Output**: A question list, a structured case analysis, or a full legal document\n- **Domain**: Chinese legal practice (personal affairs, marriage and family, corporate governance, intellectual\n property, criminal and civil litigation, cross-border matters, labour, environmental safety, and more)\n\n## Key Features\n\n- 280 samples split into four subsets, one per PLawBench task:\n - `case_analysis` (250): case analysis scored on four dimensions — conclusion, case facts, reasoning, and cited\n statutes. Answers must follow the 【结论】/【案件事实】/【推理过程】/【法条依据】 structure.\n - `legal_consultation` (18): the model plays a lawyer and must produce 10-25 verifiable follow-up questions that\n surface the facts the client omitted or distorted.\n - `plaintiff_statement` (6): drafting a statement of complaint from the client's account.\n - `defendant_statement` (6): drafting a statement of defense from the client's account and the opposing complaint.\n- Client statements are deliberately vague, emotional, or misleading, so models must detect traps instead of\n restating the client's claims.\n- Task prompts and judge prompts are ported verbatim from the official release, and the `case_analysis` rubric\n retains its per-dimension point allocation.\n\n## Evaluation Notes\n\n- Requires an LLM judge: set `judge.strategy='llm'` (or `'auto'`, which enables the judge for this benchmark)\n and provide `judge.models`. `judge.strategy='rule'` is not supported.\n- Metrics are point ratios in `[0, 1]`. `acc` is reported for every subset; `case_analysis` additionally reports\n `conclusion_acc`, `fact_acc`, `reasoning_acc`, and `law_acc`. These map one-to-one onto the official leaderboard\n columns: `legal_consultation` is Task1, `case_analysis` is Task2-Avg with its four dimensions, and the two\n drafting subsets are Task3-Plaintiff and Task3-Defendant.\n- Compare per-subset scores, not the `OVERALL` row. `OVERALL` is a per-sample mean, so `case_analysis` dominates it\n (250 of 280 samples). The paper's `Overall` column is an equal-weighted mean of the three task scores, which\n matches its published table far more closely (mean absolute error 0.72 versus 2.87 for a sample-weighted mean,\n fitted across the 24 models in the official ranking).\n- Rubric point totals come from the dataset, not from the judge output, and awarded points are clamped into\n `[0, max_points]`, so a judge that mis-reports the denominator cannot distort the score.\n- The judge output template for `case_analysis` is repaired relative to the official script, which ships malformed\n JSON and pins the conclusion section to zero points; every section is graded on its rubric allocation here.\n- The judge model's transport retry policy is configured through its `generation_config`. A reply that still\n fails the output contract is unavailable and excluded rather than silently scored as zero.\n- Case-analysis judging returns a long per-item breakdown. Give the judge a generous `max_tokens`\n (for example 8192) in `judge.models[].generation_config`.\n- The drafting subsets ask for a 2,500-3,000 character legal document, so the evaluated model also needs a generous\n `generation_config.max_tokens`. A truncated filing is graded as an incomplete document and scores near zero, which\n depresses Task3 for reasons unrelated to legal ability.\n\nResources: [GitHub](https://github.com/skylenage/PLawbench) |\n[Dataset](https://modelscope.cn/datasets/evalscope/PLawBench)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `plawbench` |\n| **Dataset ID** | [evalscope/PLawBench](https://modelscope.cn/datasets/evalscope/PLawBench/summary) |\n| **Paper** | [Paper](https://github.com/skylenage/PLawbench) |\n| **Tags** | `Chinese`, `Knowledge`, `QA`, `Reasoning` |\n| **Metrics** | `accuracy`, `conclusion_acc`, `fact_acc`, `reasoning_acc`, `law_acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 280 |\n| Prompt Length (Mean) | 2669.88 chars |\n| Prompt Length (Min/Max) | 1267 / 5890 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `case_analysis` | 250 | 2720.18 | 2137 | 4873 |\n| `legal_consultation` | 18 | 1851.94 | 1512 | 2611 |\n| `plaintiff_statement` | 6 | 1494.67 | 1267 | 2050 |\n| `defendant_statement` | 6 | 4203 | 2794 | 5890 |\n\n## Sample Example\n\n**Subset**: `case_analysis`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"06c8a1d7\",\n \"content\": \"\\n## 角色\\n\\n你是一名具有十年以上执业经验的法律实务专家，精通中国现行法律法规与司法实践。你擅长将复杂的法律问题分解为清晰的逻辑模块，并严格依据“结论先行、事实为重、推理严密、依据支撑”的专业风格进行解答。\\n\\n## 核心要求\\n\\n1. 严格顺序：回答必须按照以下四部分顺序展开，并使用对应标题：\\n【结论】\\n【案件事实】\\n【推理过程】\\n【法条依据】\\n2. 内容规范：\\n结论：直接、明确，针对提问的核心争议点给出肯定或否定的判断。\\n案件事实：基于用户提供的案情，简明、客观地摘录与法律判断 ... [TRUNCATED 1912 chars] ... 并赔偿精神损害抚慰金。庭审中查明，某摄影服务公司已完成除摄像外的其他服务项目；某文化传媒公司系独立法人，其工作人员在操作设备时存在重大过失。另查，某甲在签订合同时未特别声明婚礼录像的重要性，但合同附件中列有\\\"全程跟拍记录\\\"服务项目。某摄影服务公司辩称其仅需承担合同违约责任，精神损害赔偿缺乏依据。某文化传媒公司以非合同相对方为由拒绝承担责任。\\n\\n## 问题\\n以【结论 + 案情简述 + 分析过程+依据法条】的逻辑回答以下问题：在上述案例中，某甲能否向某摄影服务公司主张精神损害赔偿？\\n\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"case_analysis\",\n \"metadata\": {\n \"id\": \"case_analysis-1\",\n \"task\": \"case_analysis\",\n \"judge_type\": \"case_analysis\",\n \"category\": \"个人生活\",\n \"rubrics\": \"[{\\\"criterion\\\": \\\"【结论得分】\\\\n(+5分) 某甲有权向某摄影服务公司主张精神损害赔偿。\\\", \\\"points\\\": \\\"5\\\", \\\"tags\\\": \\\"结论得分\\\"}, {\\\"criterion\\\": \\\"【案情简述得分】\\\\n(+5分) 某甲与某摄影服务公司签订《婚庆服务合同》，并支付全款，合同附件明确包含\\\\\\\"全程跟拍记录\\\\\\\"服务项目。\\\\n(+5分) 婚礼当日，某摄影服务公司未经告知将摄像服务转包给文化传媒公司。\\\\n(+5分) 文化传媒公司工作室将录像全部丢失，未能交付原告。\\\\n(+ ... [TRUNCATED 762 chars] ... 人具有人身意义的特定物造成严重精神损害的，被侵权人有权请求精神损害赔偿。\\\\n（+5分）《最高人民法院关于确定民事侵权精神损害赔偿责任若干问题的解释》第五条\\\\n精神损害的赔偿数额根据以下因素确定：(一)侵权人的过错程度，但是法律另有规定的除外；(二)侵权行为的目的、方式、场合等具体情节；(三)侵权行为所造成的后果；(四)侵权人的获利情况；(五)侵权人承担责任的经济能力；(六)受理诉讼法院所在地的平均生活水平。\\\", \\\"points\\\": \\\"15\\\", \\\"tags\\\": \\\"法条依据得分\\\"}]\",\n \"max_points\": 60,\n \"prompt\": \"\\n## 角色\\n\\n你是一名具有十年以上执业经验的法律实务专家，精通中国现行法律法规与司法实践。你擅长将复杂的法律问题分解为清晰的逻辑模块，并严格依据“结论先行、事实为重、推理严密、依据支撑”的专业风格进行解答。\\n\\n## 核心要求\\n\\n1. 严格顺序：回答必须按照以下四部分顺序展开，并使用对应标题：\\n【结论】\\n【案件事实】\\n【推理过程】\\n【法条依据】\\n2. 内容规范：\\n结论：直接、明确，针对提问的核心争议点给出肯定或否定的判断。\\n案件事实：基于用户提供的案情，简明、客观地摘录与法律判断 ... [TRUNCATED 1912 chars] ... 并赔偿精神损害抚慰金。庭审中查明，某摄影服务公司已完成除摄像外的其他服务项目；某文化传媒公司系独立法人，其工作人员在操作设备时存在重大过失。另查，某甲在签订合同时未特别声明婚礼录像的重要性，但合同附件中列有\\\"全程跟拍记录\\\"服务项目。某摄影服务公司辩称其仅需承担合同违约责任，精神损害赔偿缺乏依据。某文化传媒公司以非合同相对方为由拒绝承担责任。\\n\\n## 问题\\n以【结论 + 案情简述 + 分析过程+依据法条】的逻辑回答以下问题：在上述案例中，某甲能否向某摄影服务公司主张精神损害赔偿？\\n\"\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets plawbench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['plawbench'],\n dataset_args={\n 'plawbench': {\n # subset_list: ['case_analysis', 'legal_consultation', 'plaintiff_statement'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# PLawBench\n\n\n## 概述\n\nPLawBench 是一个基于评分标准（rubric）的基准测试，用于评估大语言模型在真实中文法律实务场景中的表现。 \n该基准模拟执业律师的工作流程，涵盖三个层级的任务：在公共法律咨询中梳理事实、通过结构化法律推理分析案件，以及起草专业法律文书。 \n每个样本均附有法律专家标注的评分标准，评分由 LLM 评判模型依据该标准进行，而非与单一参考答案比对。\n\n## 任务描述\n\n- **任务类型**：基于专家评分标准的开放式中文法律生成任务 \n- **输入**：客户陈述，或案件描述加法律问题 \n- **输出**：问题清单、结构化案件分析，或完整的法律文书 \n- **领域**：中文法律实务（个人事务、婚姻家庭、公司治理、知识产权、刑事与民事诉讼、跨境事务、劳动法、环境安全等）\n\n## 核心特性\n\n- 共 280 个样本，划分为四个子集，对应 PLawBench 的四项任务：\n - `case_analysis`（250 个）：案件分析，从四个维度评分——结论、案件事实、推理过程、引用法条。回答必须严格遵循【结论】/【案件事实】/【推理过程】/【法条依据】的结构。\n - `legal_consultation`（18 个）：模型扮演律师，需提出 10–25 个可验证的后续问题，以揭示客户遗漏或歪曲的事实。\n - `plaintiff_statement`（6 个）：根据客户陈述起草起诉状。\n - `defendant_statement`（6 个）：根据客户陈述及对方起诉状起草答辩状。\n- 客户陈述故意设计得模糊、情绪化或具有误导性，因此模型必须识别陷阱，而非简单复述客户主张。\n- 任务提示词和评判提示词均直接采用官方发布版本，`case_analysis` 子集的评分标准保留了各维度的原始分值分配。\n\n## 评估说明\n\n- **必须使用 LLM 评判模型**：需设置 `judge.strategy='llm'`（或 `'auto'`，该选项会为此基准自动启用评判模型），并提供 `judge.models`。不支持 `judge.strategy='rule'`。\n- **指标为 [0, 1] 区间内的得分比例**。所有子集均报告 `acc`；`case_analysis` 额外报告 `conclusion_acc`、`fact_acc`、`reasoning_acc` 和 `law_acc`。这些指标与官方排行榜列一一对应：`legal_consultation` 对应 Task1，`case_analysis` 对应 Task2-Avg 及其四个维度，两个文书起草子集分别对应 Task3-Plaintiff 和 Task3-Defendant。\n- **应比较各子集得分，而非 `OVERALL` 行**。`OVERALL` 是按样本数量计算的平均值，因此被 `case_analysis` 主导（250/280）。论文中的 `Overall` 列是三项任务得分的等权重平均值，与官方公布表格更吻合（在官方排名的 24 个模型上拟合，平均绝对误差为 0.72，而样本加权平均的误差为 2.87）。\n- **评分标准的总分来自数据集本身，而非评判模型输出**，且实际得分会被限制在 `[0, max_points]` 范围内，因此即使评判模型错误报告分母，也不会扭曲最终分数。\n- `case_analysis` 的评判输出模板已修复。官方脚本存在格式错误的 JSON，并将结论部分固定为零分；本实现中每个部分均按其评分标准独立打分。\n- 评判模型的重试策略通过其 `generation_config` 配置。若回复仍不符合输出格式要求，则视为无效并排除，而非静默记为零分。\n- 案件分析评判会返回详细的逐项分解结果。建议为评判模型设置较大的 `max_tokens`（例如 8192），配置于 `judge.models[].generation_config`。\n- 文书起草子集要求生成 2,500–3,000 字符的法律文书，因此被评估模型也需要较大的 `generation_config.max_tokens`。若生成内容被截断，将被视为不完整文书，得分接近零，从而因非法律能力原因拉低 Task3 得分。\n\n资源：[GitHub](https://github.com/skylenage/PLawbench) |\n[数据集](https://modelscope.cn/datasets/evalscope/PLawBench)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `plawbench` |\n| **数据集ID** | [evalscope/PLawBench](https://modelscope.cn/datasets/evalscope/PLawBench/summary) |\n| **论文** | [Paper](https://github.com/skylenage/PLawbench) |\n| **标签** | `Chinese`, `Knowledge`, `QA`, `Reasoning` |\n| **指标** | `accuracy`, `conclusion_acc`, `fact_acc`, `reasoning_acc`, `law_acc` |\n| **默认示例数** | 0-shot |\n| **评估分割** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 280 |\n| 提示词长度（平均） | 2669.88 字符 |\n| 提示词长度（最小/最大） | 1267 / 5890 字符 |\n\n**各子集统计信息：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `case_analysis` | 250 | 2720.18 | 2137 | 4873 |\n| `legal_consultation` | 18 | 1851.94 | 1512 | 2611 |\n| `plaintiff_statement` | 6 | 1494.67 | 1267 | 2050 |\n| `defendant_statement` | 6 | 4203 | 2794 | 5890 |\n\n## 样例示例\n\n**子集**: `case_analysis`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"06c8a1d7\",\n \"content\": \"\\n## 角色\\n\\n你是一名具有十年以上执业经验的法律实务专家，精通中国现行法律法规与司法实践。你擅长将复杂的法律问题分解为清晰的逻辑模块，并严格依据“结论先行、事实为重、推理严密、依据支撑”的专业风格进行解答。\\n\\n## 核心要求\\n\\n1. 严格顺序：回答必须按照以下四部分顺序展开，并使用对应标题：\\n【结论】\\n【案件事实】\\n【推理过程】\\n【法条依据】\\n2. 内容规范：\\n结论：直接、明确，针对提问的核心争议点给出肯定或否定的判断。\\n案件事实：基于用户提供的案情，简明、客观地摘录与法律判断 ... [TRUNCATED 1912 chars] ... 并赔偿精神损害抚慰金。庭审中查明，某摄影服务公司已完成除摄像外的其他服务项目；某文化传媒公司系独立法人，其工作人员在操作设备时存在重大过失。另查，某甲在签订合同时未特别声明婚礼录像的重要性，但合同附件中列有\\\"全程跟拍记录\\\"服务项目。某摄影服务公司辩称其仅需承担合同违约责任，精神损害赔偿缺乏依据。某文化传媒公司以非合同相对方为由拒绝承担责任。\\n\\n## 问题\\n以【结论 + 案情简述 + 分析过程+依据法条】的逻辑回答以下问题：在上述案例中，某甲能否向某摄影服务公司主张精神损害赔偿？\\n\"\n }\n ],\n \"target\": \"\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"case_analysis\",\n \"metadata\": {\n \"id\": \"case_analysis-1\",\n \"task\": \"case_analysis\",\n \"judge_type\": \"case_analysis\",\n \"category\": \"个人生活\",\n \"rubrics\": \"[{\\\"criterion\\\": \\\"【结论得分】\\\\n(+5分) 某甲有权向某摄影服务公司主张精神损害赔偿。\\\", \\\"points\\\": \\\"5\\\", \\\"tags\\\": \\\"结论得分\\\"}, {\\\"criterion\\\": \\\"【案情简述得分】\\\\n(+5分) 某甲与某摄影服务公司签订《婚庆服务合同》，并支付全款，合同附件明确包含\\\\\\\"全程跟拍记录\\\\\\\"服务项目。\\\\n(+5分) 婚礼当日，某摄影服务公司未经告知将摄像服务转包给文化传媒公司。\\\\n(+5分) 文化传媒公司工作室将录像全部丢失，未能交付原告。\\\\n(+ ... [TRUNCATED 762 chars] ... 人具有人身意义的特定物造成严重精神损害的，被侵权人有权请求精神损害赔偿。\\\\n（+5分）《最高人民法院关于确定民事侵权精神损害赔偿责任若干问题的解释》第五条\\\\n精神损害的赔偿数额根据以下因素确定：(一)侵权人的过错程度，但是法律另有规定的除外；(二)侵权行为的目的、方式、场合等具体情节；(三)侵权行为所造成的后果；(四)侵权人的获利情况；(五)侵权人承担责任的经济能力；(六)受理诉讼法院所在地的平均生活水平。\\\", \\\"points\\\": \\\"15\\\", \\\"tags\\\": \\\"法条依据得分\\\"}]\",\n \"max_points\": 60,\n \"prompt\": \"\\n## 角色\\n\\n你是一名具有十年以上执业经验的法律实务专家，精通中国现行法律法规与司法实践。你擅长将复杂的法律问题分解为清晰的逻辑模块，并严格依据“结论先行、事实为重、推理严密、依据支撑”的专业风格进行解答。\\n\\n## 核心要求\\n\\n1. 严格顺序：回答必须按照以下四部分顺序展开，并使用对应标题：\\n【结论】\\n【案件事实】\\n【推理过程】\\n【法条依据】\\n2. 内容规范：\\n结论：直接、明确，针对提问的核心争议点给出肯定或否定的判断。\\n案件事实：基于用户提供的案情，简明、客观地摘录与法律判断 ... [TRUNCATED 1912 chars] ... 并赔偿精神损害抚慰金。庭审中查明，某摄影服务公司已完成除摄像外的其他服务项目；某文化传媒公司系独立法人，其工作人员在操作设备时存在重大过失。另查，某甲在签订合同时未特别声明婚礼录像的重要性，但合同附件中列有\\\"全程跟拍记录\\\"服务项目。某摄影服务公司辩称其仅需承担合同违约责任，精神损害赔偿缺乏依据。某文化传媒公司以非合同相对方为由拒绝承担责任。\\n\\n## 问题\\n以【结论 + 案情简述 + 分析过程+依据法条】的逻辑回答以下问题：在上述案例中，某甲能否向某摄影服务公司主张精神损害赔偿？\\n\"\n }\n}\n```\n\n*注：部分内容因展示需要已被截断。*\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets plawbench \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['plawbench'],\n dataset_args={\n 'plawbench': {\n # subset_list: ['case_analysis', 'legal_consultation', 'plaintiff_statement'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "e704fd450eeb00621f084f5be0b7eb36", "needs_translation": false }, - "updated_at": "2026-08-11T15:30:53.622307", - "translation_updated_at": "2026-08-07T15:04:14+08:00" + "updated_at": "2026-08-20T10:40:24.368587", + "translation_updated_at": "2026-08-20T10:41:18" } diff --git a/evalscope/benchmarks/_meta/researchrubrics.json b/evalscope/benchmarks/_meta/researchrubrics.json index a4b5f8dc4..1f11f64af 100644 --- a/evalscope/benchmarks/_meta/researchrubrics.json +++ b/evalscope/benchmarks/_meta/researchrubrics.json @@ -18,7 +18,7 @@ "subset_list": [ "default" ], - "description": "\n## Overview\n\nResearchRubrics evaluates Deep Research agents on realistic, open-ended research tasks. Each task pairs a user prompt\nwith expert-written, fine-grained rubrics covering explicit and implicit requirements, information synthesis,\nreferences, communication quality, and instruction following.\n\n## Task Description\n\n- **Task Type**: Multi-turn research agent / long-form report generation\n- **Input**: One open-ended research prompt\n- **Output**: A Markdown research report produced after iterative tool use\n- **Dataset**: 101 tasks and 2,593 weighted rubric criteria\n- **Metric**: Binary rubric compliance score\n\n## Agent Environment\n\n- Uses EvalScope's built-in agent environment by default and does not require ``agent_config``. The agent can use ``bash``\n to access the network, gather information, and produce a final report.\n- The default environment uses the host network and a temporary working directory, but does not provide complete filesystem\n isolation. Do not run untrusted models on shared or sensitive machines.\n- The default strategy is ``function_calling`` with a 50-step limit. Use ``NativeAgentConfig`` to override the strategy\n or step limit; ``react`` is also available. Both strategies require native function calling support.\n- Add dedicated search or web-fetching tools through ``NativeAgentConfig``, or use ``ExternalAgentConfig`` to run the\n task with another agent framework.\n- When the step limit is reached, the model is asked to produce a final report from the information already collected so\n the result can still be reviewed and scored.\n\n## Evaluation Notes\n\n- ResearchRubrics requires ``judge_model_args`` and ``judge_strategy='auto'`` or ``'llm'``. Gemini 2.5 Pro is the\n recommended judge for comparison with the paper, but no provider or model is hard-coded.\n- Every rubric is graded independently as Satisfied (1) or Not Satisfied (0), matching the public binary grader. The\n paper's ternary scores are not directly comparable.\n- Negative-weight criteria subtract from the numerator when the undesirable behavior is present. Scores are not\n clipped.\n- Long reports are evaluated with the official chunk-evidence-synthesis approach when they exceed the configured judge\n context threshold.\n- A full run performs 2,593 rubric evaluations and can be expensive. Current-events tasks are also sensitive to the\n date and web sources available at evaluation time.\n\n## Configuration\n\n- ``judge_context_limit``: 150,000 estimated tokens\n- ``judge_chunk_size``: 100,000 estimated tokens\n- ``judge_retries``: 3 attempts per judge request\n\nThe judge must be configured explicitly. For example:\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model='YOUR_AGENT_MODEL',\n datasets=['researchrubrics'],\n judge_strategy='llm',\n judge_model_args={\n 'model_id': 'YOUR_JUDGE_MODEL',\n 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL',\n 'api_key': 'YOUR_JUDGE_API_KEY',\n 'generation_config': {'temperature': 0.0},\n },\n limit=1,\n))\n```\n\nResources: [Paper](https://arxiv.org/abs/2511.07685) |\n[GitHub](https://github.com/scaleapi/researchrubrics) |\n[Dataset](https://modelscope.cn/datasets/evalscope/researchrubrics)\n", + "description": "\n## Overview\n\nResearchRubrics evaluates Deep Research agents on realistic, open-ended research tasks. Each task pairs a user prompt\nwith expert-written, fine-grained rubrics covering explicit and implicit requirements, information synthesis,\nreferences, communication quality, and instruction following.\n\n## Task Description\n\n- **Task Type**: Multi-turn research agent / long-form report generation\n- **Input**: One open-ended research prompt\n- **Output**: A Markdown research report produced after iterative tool use\n- **Dataset**: 101 tasks and 2,593 weighted rubric criteria\n- **Metric**: Binary rubric compliance score\n\n## Key Features\n\n- 101 open-ended Deep Research tasks paired with 2,593 expert-written, weighted rubric criteria.\n- Rubrics cover explicit and implicit requirements, information synthesis, references, communication\n quality, and instruction following, and each is graded independently.\n- Negative-weight criteria capture undesirable behaviours and subtract from the score when present.\n- Long reports are graded with the official chunk-evidence-synthesis procedure once they exceed the\n configured judge context threshold.\n\n## Agent Environment\n\n- Uses EvalScope's built-in agent environment by default and does not require ``agent_config``. The agent can use ``bash``\n to access the network, gather information, and produce a final report.\n- The default environment uses the host network and a temporary working directory, but does not provide complete filesystem\n isolation. Do not run untrusted models on shared or sensitive machines.\n- The default strategy is ``function_calling`` with a 50-step limit. Use ``NativeAgentConfig`` to override the strategy\n or step limit; ``react`` is also available. Both strategies require native function calling support.\n- Add dedicated search or web-fetching tools through ``NativeAgentConfig``, or use ``ExternalAgentConfig`` to run the\n task with another agent framework.\n- When the step limit is reached, the model is asked to produce a final report from the information already collected so\n the result can still be reviewed and scored.\n\n## Evaluation Notes\n\n- ResearchRubrics requires ``judge.models`` and ``judge.strategy='auto'`` or ``'llm'``. Gemini 2.5 Pro is the\n recommended judge for comparison with the paper, but no provider or model is hard-coded.\n- Every rubric is graded independently as Satisfied (1) or Not Satisfied (0), matching the public binary grader. The\n paper's ternary scores are not directly comparable.\n- Negative-weight criteria subtract from the numerator when the undesirable behavior is present. Scores are not\n clipped.\n- Long reports are evaluated with the official chunk-evidence-synthesis approach when they exceed the configured judge\n context threshold.\n- A full run performs 2,593 rubric evaluations and can be expensive. Current-events tasks are also sensitive to the\n date and web sources available at evaluation time.\n\n## Configuration\n\n- ``judge_context_limit``: 150,000 estimated tokens\n- ``judge_chunk_size``: 100,000 estimated tokens\n\nThe judge must be configured explicitly. For example:\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model='YOUR_AGENT_MODEL',\n datasets=['researchrubrics'],\n judge={\n 'strategy': 'llm',\n 'models': {\n 'model_id': 'YOUR_JUDGE_MODEL',\n 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL',\n 'api_key': 'YOUR_JUDGE_API_KEY',\n 'generation_config': {'temperature': 0.0},\n },\n },\n limit=1,\n))\n```\n\nResources: [Paper](https://arxiv.org/abs/2511.07685) |\n[GitHub](https://github.com/scaleapi/researchrubrics) |\n[Dataset](https://modelscope.cn/datasets/evalscope/researchrubrics)\n", "prompt_template": "{question}", "system_prompt": "", "few_shot_prompt_template": "", @@ -33,11 +33,6 @@ "type": "int", "description": "Maximum estimated tokens in each document chunk sent to the judge.", "value": 100000 - }, - "judge_retries": { - "type": "int", - "description": "Maximum attempts for each rubric judge request and JSON parse.", - "value": 3 } }, "sandbox_config": {}, @@ -114,11 +109,11 @@ "truncated": true }, "readme": { - "en": "# ResearchRubrics\n\n\n## Overview\n\nResearchRubrics evaluates Deep Research agents on realistic, open-ended research tasks. Each task pairs a user prompt\nwith expert-written, fine-grained rubrics covering explicit and implicit requirements, information synthesis,\nreferences, communication quality, and instruction following.\n\n## Task Description\n\n- **Task Type**: Multi-turn research agent / long-form report generation\n- **Input**: One open-ended research prompt\n- **Output**: A Markdown research report produced after iterative tool use\n- **Dataset**: 101 tasks and 2,593 weighted rubric criteria\n- **Metric**: Binary rubric compliance score\n\n## Agent Environment\n\n- Uses EvalScope's built-in agent environment by default and does not require ``agent_config``. The agent can use ``bash``\n to access the network, gather information, and produce a final report.\n- The default environment uses the host network and a temporary working directory, but does not provide complete filesystem\n isolation. Do not run untrusted models on shared or sensitive machines.\n- The default strategy is ``function_calling`` with a 50-step limit. Use ``NativeAgentConfig`` to override the strategy\n or step limit; ``react`` is also available. Both strategies require native function calling support.\n- Add dedicated search or web-fetching tools through ``NativeAgentConfig``, or use ``ExternalAgentConfig`` to run the\n task with another agent framework.\n- When the step limit is reached, the model is asked to produce a final report from the information already collected so\n the result can still be reviewed and scored.\n\n## Evaluation Notes\n\n- ResearchRubrics requires ``judge_model_args`` and ``judge_strategy='auto'`` or ``'llm'``. Gemini 2.5 Pro is the\n recommended judge for comparison with the paper, but no provider or model is hard-coded.\n- Every rubric is graded independently as Satisfied (1) or Not Satisfied (0), matching the public binary grader. The\n paper's ternary scores are not directly comparable.\n- Negative-weight criteria subtract from the numerator when the undesirable behavior is present. Scores are not\n clipped.\n- Long reports are evaluated with the official chunk-evidence-synthesis approach when they exceed the configured judge\n context threshold.\n- A full run performs 2,593 rubric evaluations and can be expensive. Current-events tasks are also sensitive to the\n date and web sources available at evaluation time.\n\n## Configuration\n\n- ``judge_context_limit``: 150,000 estimated tokens\n- ``judge_chunk_size``: 100,000 estimated tokens\n- ``judge_retries``: 3 attempts per judge request\n\nThe judge must be configured explicitly. For example:\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model='YOUR_AGENT_MODEL',\n datasets=['researchrubrics'],\n judge_strategy='llm',\n judge_model_args={\n 'model_id': 'YOUR_JUDGE_MODEL',\n 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL',\n 'api_key': 'YOUR_JUDGE_API_KEY',\n 'generation_config': {'temperature': 0.0},\n },\n limit=1,\n))\n```\n\nResources: [Paper](https://arxiv.org/abs/2511.07685) |\n[GitHub](https://github.com/scaleapi/researchrubrics) |\n[Dataset](https://modelscope.cn/datasets/evalscope/researchrubrics)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `researchrubrics` |\n| **Dataset ID** | [evalscope/researchrubrics](https://modelscope.cn/datasets/evalscope/researchrubrics/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2511.07685) |\n| **Tags** | `Agent`, `MultiTurn`, `Reasoning`, `Retrieval` |\n| **Metrics** | `compliance_score` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 101 |\n| Prompt Length (Mean) | 555.35 chars |\n| Prompt Length (Min/Max) | 102 / 1747 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"f1132c85\",\n \"content\": \"I want to create a plan for July 4, 2025, i.e., Independence Day in Washington DC. I would like an itinerary of all the things to do and all the activities that are planned for Independence Day. Create a plan for the whole day and also extend it to the weekend, if required. Provide some reviews or explain why one should visit the place or engage in the activity. Add any additional information that is required.\"\n }\n ],\n \"target\": \"[{\\\"criterion\\\": \\\"The response covers the period from 9:00 AM or earlier through at least 10:00 PM on 4 July 2025\\\", \\\"weight\\\": 5.0, \\\"axis\\\": \\\"Explicit Criteria\\\"}, {\\\"criterion\\\": \\\"The response contains clear section headers for parts of the schedul ... [TRUNCATED 4470 chars] ... events from years other than 2025 (e.g., seeing a miltiary parade, information about \\\\\\\"A Capital Fourth\\\\\\\" for 2024, information the parade route for 2023, referencing a cancellation from 2020).\\\", \\\"weight\\\": -4.0, \\\"axis\\\": \\\"Explicit Criteria\\\"}]\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"sample_id\": \"6847465956a0f6376a605427\",\n \"domain\": \"Current Events\",\n \"conceptual_breadth\": \"Simple\",\n \"logical_nesting\": \"Intermediate\",\n \"exploration\": \"Medium\"\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `judge_context_limit` | `int` | `150000` | Estimated token limit before rubric judging switches to chunking. |\n| `judge_chunk_size` | `int` | `100000` | Maximum estimated tokens in each document chunk sent to the judge. |\n| `judge_retries` | `int` | `3` | Maximum attempts for each rubric judge request and JSON parse. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets researchrubrics \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":50}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['researchrubrics'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=50,\n ),\n dataset_args={\n 'researchrubrics': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# ResearchRubrics\n\n\n## 概述\n\nResearchRubrics 用于评估深度研究（Deep Research）智能体在真实、开放式研究任务上的表现。每个任务包含一个用户提示，以及由专家编写的细粒度评分标准（rubrics），涵盖显式与隐式需求、信息综合、参考文献、沟通质量以及指令遵循等方面。\n\n## 任务描述\n\n- **任务类型**：多轮研究智能体 / 长篇报告生成\n- **输入**：一个开放式研究提示\n- **输出**：通过迭代使用工具后生成的 Markdown 格式研究报告\n- **数据集**：101 个任务，共包含 2,593 条带权重的评分标准\n- **评估指标**：二元评分标准符合度得分（Binary rubric compliance score）\n\n## 智能体环境\n\n- 默认使用 EvalScope 内置的智能体环境，无需提供 ``agent_config``。智能体可通过 ``bash`` 访问网络、收集信息并生成最终报告。\n- 默认环境使用主机网络和临时工作目录，但不提供完整的文件系统隔离。请勿在共享或敏感机器上运行不可信模型。\n- 默认策略为 ``function_calling``，最多执行 50 步。可通过 ``NativeAgentConfig`` 覆盖策略或步数限制；也可选择 ``react`` 策略。两种策略均需模型原生支持函数调用。\n- 可通过 ``NativeAgentConfig`` 添加专用搜索或网页抓取工具，或使用 ``ExternalAgentConfig`` 在其他智能体框架中运行任务。\n- 当达到步数限制时，模型会被要求基于已收集的信息生成最终报告，以便进行人工评审和评分。\n\n## 评估说明\n\n- ResearchRubrics 要求显式配置 ``judge_model_args``，且 ``judge_strategy`` 必须为 ``'auto'`` 或 ``'llm'``。论文推荐使用 Gemini 2.5 Pro 作为评判模型，但未硬编码任何特定提供商或模型。\n- 每条评分标准独立判分为“满足”（1）或“不满足”（0），与公开的二元评分器一致。论文中使用的三元评分无法直接比较。\n- 负权重标准在出现不良行为时会从分子中扣除相应分数，最终得分不会被截断。\n- 当报告长度超过配置的评判模型上下文阈值时，将采用官方的分块-证据-综合（chunk-evidence-synthesis）方法进行评估。\n- 完整运行需执行 2,593 次评分标准评估，成本较高。涉及近期事件的任务对评估时的日期和可用网络资源较为敏感。\n\n## 配置\n\n- ``judge_context_limit``: 150,000 个估算 token\n- ``judge_chunk_size``: 100,000 个估算 token\n- ``judge_retries``: 每次评判请求最多重试 3 次\n\n评判模型必须显式配置。例如：\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model='YOUR_AGENT_MODEL',\n datasets=['researchrubrics'],\n judge_strategy='llm',\n judge_model_args={\n 'model_id': 'YOUR_JUDGE_MODEL',\n 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL',\n 'api_key': 'YOUR_JUDGE_API_KEY',\n 'generation_config': {'temperature': 0.0},\n },\n limit=1,\n))\n```\n\n资源链接：[论文](https://arxiv.org/abs/2511.07685) |\n[GitHub](https://github.com/scaleapi/researchrubrics) |\n[数据集](https://modelscope.cn/datasets/evalscope/researchrubrics)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `researchrubrics` |\n| **数据集ID** | [evalscope/researchrubrics](https://modelscope.cn/datasets/evalscope/researchrubrics/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2511.07685) |\n| **标签** | `Agent`, `MultiTurn`, `Reasoning`, `Retrieval` |\n| **指标** | `compliance_score` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 101 |\n| 提示词长度（平均） | 555.35 字符 |\n| 提示词长度（最小/最大） | 102 / 1747 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"f1132c85\",\n \"content\": \"I want to create a plan for July 4, 2025, i.e., Independence Day in Washington DC. I would like an itinerary of all the things to do and all the activities that are planned for Independence Day. Create a plan for the whole day and also extend it to the weekend, if required. Provide some reviews or explain why one should visit the place or engage in the activity. Add any additional information that is required.\"\n }\n ],\n \"target\": \"[{\\\"criterion\\\": \\\"The response covers the period from 9:00 AM or earlier through at least 10:00 PM on 4 July 2025\\\", \\\"weight\\\": 5.0, \\\"axis\\\": \\\"Explicit Criteria\\\"}, {\\\"criterion\\\": \\\"The response contains clear section headers for parts of the schedul ... [TRUNCATED 4470 chars] ... events from years other than 2025 (e.g., seeing a miltiary parade, information about \\\\\\\"A Capital Fourth\\\\\\\" for 2024, information the parade route for 2023, referencing a cancellation from 2020).\\\", \\\"weight\\\": -4.0, \\\"axis\\\": \\\"Explicit Criteria\\\"}]\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"sample_id\": \"6847465956a0f6376a605427\",\n \"domain\": \"Current Events\",\n \"conceptual_breadth\": \"Simple\",\n \"logical_nesting\": \"Intermediate\",\n \"exploration\": \"Medium\"\n }\n}\n```\n\n*注：部分内容因显示需要已被截断。*\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `judge_context_limit` | `int` | `150000` | 评判前允许的最大估算 token 数，超过此值将启用分块处理。 |\n| `judge_chunk_size` | `int` | `100000` | 发送给评判模型的每个文档分块的最大估算 token 数。 |\n| `judge_retries` | `int` | `3` | 每次评分标准评判请求及 JSON 解析的最大重试次数。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets researchrubrics \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":50}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['researchrubrics'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=50,\n ),\n dataset_args={\n 'researchrubrics': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "5d00cf9f9ee9689e26d2cc30cfb1eebb", + "en": "# ResearchRubrics\n\n\n## Overview\n\nResearchRubrics evaluates Deep Research agents on realistic, open-ended research tasks. Each task pairs a user prompt\nwith expert-written, fine-grained rubrics covering explicit and implicit requirements, information synthesis,\nreferences, communication quality, and instruction following.\n\n## Task Description\n\n- **Task Type**: Multi-turn research agent / long-form report generation\n- **Input**: One open-ended research prompt\n- **Output**: A Markdown research report produced after iterative tool use\n- **Dataset**: 101 tasks and 2,593 weighted rubric criteria\n- **Metric**: Binary rubric compliance score\n\n## Key Features\n\n- 101 open-ended Deep Research tasks paired with 2,593 expert-written, weighted rubric criteria.\n- Rubrics cover explicit and implicit requirements, information synthesis, references, communication\n quality, and instruction following, and each is graded independently.\n- Negative-weight criteria capture undesirable behaviours and subtract from the score when present.\n- Long reports are graded with the official chunk-evidence-synthesis procedure once they exceed the\n configured judge context threshold.\n\n## Agent Environment\n\n- Uses EvalScope's built-in agent environment by default and does not require ``agent_config``. The agent can use ``bash``\n to access the network, gather information, and produce a final report.\n- The default environment uses the host network and a temporary working directory, but does not provide complete filesystem\n isolation. Do not run untrusted models on shared or sensitive machines.\n- The default strategy is ``function_calling`` with a 50-step limit. Use ``NativeAgentConfig`` to override the strategy\n or step limit; ``react`` is also available. Both strategies require native function calling support.\n- Add dedicated search or web-fetching tools through ``NativeAgentConfig``, or use ``ExternalAgentConfig`` to run the\n task with another agent framework.\n- When the step limit is reached, the model is asked to produce a final report from the information already collected so\n the result can still be reviewed and scored.\n\n## Evaluation Notes\n\n- ResearchRubrics requires ``judge.models`` and ``judge.strategy='auto'`` or ``'llm'``. Gemini 2.5 Pro is the\n recommended judge for comparison with the paper, but no provider or model is hard-coded.\n- Every rubric is graded independently as Satisfied (1) or Not Satisfied (0), matching the public binary grader. The\n paper's ternary scores are not directly comparable.\n- Negative-weight criteria subtract from the numerator when the undesirable behavior is present. Scores are not\n clipped.\n- Long reports are evaluated with the official chunk-evidence-synthesis approach when they exceed the configured judge\n context threshold.\n- A full run performs 2,593 rubric evaluations and can be expensive. Current-events tasks are also sensitive to the\n date and web sources available at evaluation time.\n\n## Configuration\n\n- ``judge_context_limit``: 150,000 estimated tokens\n- ``judge_chunk_size``: 100,000 estimated tokens\n\nThe judge must be configured explicitly. For example:\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model='YOUR_AGENT_MODEL',\n datasets=['researchrubrics'],\n judge={\n 'strategy': 'llm',\n 'models': {\n 'model_id': 'YOUR_JUDGE_MODEL',\n 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL',\n 'api_key': 'YOUR_JUDGE_API_KEY',\n 'generation_config': {'temperature': 0.0},\n },\n },\n limit=1,\n))\n```\n\nResources: [Paper](https://arxiv.org/abs/2511.07685) |\n[GitHub](https://github.com/scaleapi/researchrubrics) |\n[Dataset](https://modelscope.cn/datasets/evalscope/researchrubrics)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `researchrubrics` |\n| **Dataset ID** | [evalscope/researchrubrics](https://modelscope.cn/datasets/evalscope/researchrubrics/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2511.07685) |\n| **Tags** | `Agent`, `MultiTurn`, `Reasoning`, `Retrieval` |\n| **Metrics** | `compliance_score` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `train` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 101 |\n| Prompt Length (Mean) | 555.35 chars |\n| Prompt Length (Min/Max) | 102 / 1747 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"f1132c85\",\n \"content\": \"I want to create a plan for July 4, 2025, i.e., Independence Day in Washington DC. I would like an itinerary of all the things to do and all the activities that are planned for Independence Day. Create a plan for the whole day and also extend it to the weekend, if required. Provide some reviews or explain why one should visit the place or engage in the activity. Add any additional information that is required.\"\n }\n ],\n \"target\": \"[{\\\"criterion\\\": \\\"The response covers the period from 9:00 AM or earlier through at least 10:00 PM on 4 July 2025\\\", \\\"weight\\\": 5.0, \\\"axis\\\": \\\"Explicit Criteria\\\"}, {\\\"criterion\\\": \\\"The response contains clear section headers for parts of the schedul ... [TRUNCATED 4470 chars] ... events from years other than 2025 (e.g., seeing a miltiary parade, information about \\\\\\\"A Capital Fourth\\\\\\\" for 2024, information the parade route for 2023, referencing a cancellation from 2020).\\\", \\\"weight\\\": -4.0, \\\"axis\\\": \\\"Explicit Criteria\\\"}]\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"sample_id\": \"6847465956a0f6376a605427\",\n \"domain\": \"Current Events\",\n \"conceptual_breadth\": \"Simple\",\n \"logical_nesting\": \"Intermediate\",\n \"exploration\": \"Medium\"\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `judge_context_limit` | `int` | `150000` | Estimated token limit before rubric judging switches to chunking. |\n| `judge_chunk_size` | `int` | `100000` | Maximum estimated tokens in each document chunk sent to the judge. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets researchrubrics \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":50}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['researchrubrics'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=50,\n ),\n dataset_args={\n 'researchrubrics': {\n # extra_params: {} # uses default extra parameters\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# ResearchRubrics\n\n\n## 概述\n\nResearchRubrics 用于评估深度研究（Deep Research）智能体在真实、开放式研究任务上的表现。每个任务包含一个用户提示，以及由专家编写的细粒度评分标准（rubrics），涵盖显性和隐性需求、信息综合、参考文献、沟通质量以及指令遵循等方面。\n\n## 任务描述\n\n- **任务类型**：多轮研究智能体 / 长篇报告生成 \n- **输入**：一个开放式研究提示 \n- **输出**：通过迭代使用工具生成的 Markdown 研究报告 \n- **数据集**：101 个任务，共包含 2,593 条带权重的评分标准 \n- **评估指标**：二元评分标准符合度得分（Binary rubric compliance score）\n\n## 核心特性\n\n- 包含 101 个开放式深度研究任务，配以 2,593 条由专家编写并赋予权重的评分标准。\n- 评分标准覆盖显性与隐性要求、信息综合、参考文献、沟通质量及指令遵循等多个维度，每条标准独立评分。\n- 负权重标准用于捕捉不良行为，当此类行为出现时会从总分中扣除相应分数。\n- 当报告长度超过配置的评判器上下文阈值时，将采用官方的分块-证据-综合（chunk-evidence-synthesis）流程进行评分。\n\n## 智能体环境\n\n- 默认使用 EvalScope 内置的智能体环境，无需显式提供 ``agent_config``。智能体可通过 ``bash`` 访问网络、收集信息并生成最终报告。\n- 默认环境使用主机网络和临时工作目录，但不提供完整的文件系统隔离。请勿在共享或敏感机器上运行不可信模型。\n- 默认策略为 ``function_calling``，最多执行 50 步。可通过 ``NativeAgentConfig`` 覆盖策略或步数限制；也可选用 ``react`` 策略。两种策略均需模型原生支持函数调用。\n- 可通过 ``NativeAgentConfig`` 添加专用搜索或网页抓取工具，或使用 ``ExternalAgentConfig`` 在其他智能体框架中运行任务。\n- 若达到步数上限，模型将被要求基于已收集的信息生成最终报告，以便后续评审和打分。\n\n## 评估说明\n\n- ResearchRubrics 要求配置 ``judge.models``，且 ``judge.strategy`` 必须为 ``'auto'`` 或 ``'llm'``。论文推荐使用 Gemini 2.5 Pro 作为评判器，但未硬编码任何特定提供商或模型。\n- 每条评分标准独立判为“满足”（1）或“不满足”（0），与公开的二元评分器一致。论文中使用的三元评分不可直接比较。\n- 当出现不良行为时，负权重标准会从分子中扣除对应分数，且最终得分不会被截断（clipped）。\n- 当报告长度超过配置的评判器上下文阈值时，将采用官方的分块-证据-综合方法进行评估。\n- 完整运行需执行 2,593 次评分标准评估，成本较高。此外，涉及当前事件的任务对评估时的日期和可用网络资源较为敏感。\n\n## 配置\n\n- ``judge_context_limit``: 150,000 个估算 token \n- ``judge_chunk_size``: 100,000 个估算 token \n\n评判器必须显式配置。例如：\n\n```python\nfrom evalscope import TaskConfig, run_task\n\nrun_task(TaskConfig(\n model='YOUR_AGENT_MODEL',\n datasets=['researchrubrics'],\n judge={\n 'strategy': 'llm',\n 'models': {\n 'model_id': 'YOUR_JUDGE_MODEL',\n 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL',\n 'api_key': 'YOUR_JUDGE_API_KEY',\n 'generation_config': {'temperature': 0.0},\n },\n },\n limit=1,\n))\n```\n\n资源链接：[论文](https://arxiv.org/abs/2511.07685) |\n[GitHub](https://github.com/scaleapi/researchrubrics) |\n[数据集](https://modelscope.cn/datasets/evalscope/researchrubrics)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `researchrubrics` |\n| **数据集ID** | [evalscope/researchrubrics](https://modelscope.cn/datasets/evalscope/researchrubrics/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2511.07685) |\n| **标签** | `Agent`, `MultiTurn`, `Reasoning`, `Retrieval` |\n| **指标** | `compliance_score` |\n| **默认示例数量** | 0-shot |\n| **评估划分** | `train` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 101 |\n| 提示词长度（平均） | 555.35 字符 |\n| 提示词长度（最小/最大） | 102 / 1747 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"f1132c85\",\n \"content\": \"I want to create a plan for July 4, 2025, i.e., Independence Day in Washington DC. I would like an itinerary of all the things to do and all the activities that are planned for Independence Day. Create a plan for the whole day and also extend it to the weekend, if required. Provide some reviews or explain why one should visit the place or engage in the activity. Add any additional information that is required.\"\n }\n ],\n \"target\": \"[{\\\"criterion\\\": \\\"The response covers the period from 9:00 AM or earlier through at least 10:00 PM on 4 July 2025\\\", \\\"weight\\\": 5.0, \\\"axis\\\": \\\"Explicit Criteria\\\"}, {\\\"criterion\\\": \\\"The response contains clear section headers for parts of the schedul ... [TRUNCATED 4470 chars] ... events from years other than 2025 (e.g., seeing a miltiary parade, information about \\\\\\\"A Capital Fourth\\\\\\\" for 2024, information the parade route for 2023, referencing a cancellation from 2020).\\\", \\\"weight\\\": -4.0, \\\"axis\\\": \\\"Explicit Criteria\\\"}]\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"sample_id\": \"6847465956a0f6376a605427\",\n \"domain\": \"Current Events\",\n \"conceptual_breadth\": \"Simple\",\n \"logical_nesting\": \"Intermediate\",\n \"exploration\": \"Medium\"\n }\n}\n```\n\n*注：部分内容因展示需要已被截断。*\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `judge_context_limit` | `int` | `150000` | 切换至分块评分前的估算 token 上限。 |\n| `judge_chunk_size` | `int` | `100000` | 发送给评判器的每个文档分块的最大估算 token 数。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets researchrubrics \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":50}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['researchrubrics'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=50,\n ),\n dataset_args={\n 'researchrubrics': {\n # extra_params: {} # 使用默认额外参数\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "bd8d9e7ae0c589595bfd835bdd26dde3", "needs_translation": false }, - "updated_at": "2026-07-29T23:46:22.699960", - "translation_updated_at": "2026-07-29T23:46:24" + "updated_at": "2026-08-20T10:40:24.369638", + "translation_updated_at": "2026-08-20T10:41:18" } diff --git a/evalscope/benchmarks/_meta/slake.json b/evalscope/benchmarks/_meta/slake.json index b017f8246..417372c27 100644 --- a/evalscope/benchmarks/_meta/slake.json +++ b/evalscope/benchmarks/_meta/slake.json @@ -20,7 +20,7 @@ "zh_open", "zh_closed" ], - "description": "\n## Overview\n\nSLAKE is a bilingual (English / Chinese) radiology visual question answering benchmark built by\nphysicians on CT, MRI and X-Ray images. Questions cover both purely visual properties of the scan\nand medical knowledge that has to be recalled on top of what the image shows.\n\n## Task Description\n\n- **Task Type**: Medical visual question answering (free-form short answer)\n- **Input**: A radiology image plus a question in English or Chinese\n- **Output**: A single word or short phrase, in the language of the question\n- **Domain**: Radiology (chest, abdomen, brain, pelvis, neck)\n\n## Key Features\n\n- 2,094 test questions over 180 images, roughly balanced between English (1,061) and Chinese (1,033)\n- Every question is labelled `OPEN` (free answer) or `CLOSED` (answer drawn from a small closed set,\n mostly yes/no), which is the breakdown the original paper reports\n- Questions span ten semantic types: organ, position, abnormality, knowledge-graph, modality, size,\n plane, quantity, color and shape\n- Knowledge-graph questions (`base_type=kvqa`) ask about causes, symptoms, treatments and functions\n that cannot be read off the image\n\n## Evaluation Notes\n\n- Primary metric: **Accuracy** by normalized exact match against the single reference answer\n- Reported as four subsets, `_`, grouped into an English and a Chinese\n category; the overall score is the sample-weighted mean\n- Normalization follows the official answer preprocessing (lower-case, punctuation and article\n removal, word-form numbers mapped to digits, `x ray` unified to `xray`) and adds what generative\n answering requires: yes/no synonyms collapse onto one label, because Chinese references express\n the same polarity as 是的 / 有 / 包含 / 可以 or 不是 / 没有 / 不包含 / 不可以, and the Chinese X光 / X射线\n and 两个 / 二 spellings resolve to the English `X-Ray` and digit references the Chinese half keeps\n- Answers are read from the `ANSWER:` line requested by the prompt; when the model does not emit\n one, the whole reply is normalized instead, so a reply that only restates the question scores 0\n- Exact match is strict by design, matching the original classification-style evaluation: a\n reference such as `Lung, Spinal Cord`, a knowledge-graph list of treatments, or `T2` answered as\n `T2-weighted` only counts when the model reproduces the reference wording, so open-ended\n accuracy on the knowledge-graph questions is expected to be low\n- Strictness also costs answers that differ from the reference only in wording or granularity\n (`Right` answered as `Right Side`, 胸腔 answered as 胸部, one item of a multi-item reference). Set\n `judge_strategy='llm_recall'` to have an LLM judge re-check only the rule-based failures; that\n score is more lenient than the published numbers and is not directly comparable to them\n- Images ship as a single `imgs.zip` (about 200 MB) and are read directly from the archive\n- [Paper](https://arxiv.org/abs/2102.09542) | [Project page](https://www.med-vqa.com/slake/)\n", + "description": "\n## Overview\n\nSLAKE is a bilingual (English / Chinese) radiology visual question answering benchmark built by\nphysicians on CT, MRI and X-Ray images. Questions cover both purely visual properties of the scan\nand medical knowledge that has to be recalled on top of what the image shows.\n\n## Task Description\n\n- **Task Type**: Medical visual question answering (free-form short answer)\n- **Input**: A radiology image plus a question in English or Chinese\n- **Output**: A single word or short phrase, in the language of the question\n- **Domain**: Radiology (chest, abdomen, brain, pelvis, neck)\n\n## Key Features\n\n- 2,094 test questions over 180 images, roughly balanced between English (1,061) and Chinese (1,033)\n- Every question is labelled `OPEN` (free answer) or `CLOSED` (answer drawn from a small closed set,\n mostly yes/no), which is the breakdown the original paper reports\n- Questions span ten semantic types: organ, position, abnormality, knowledge-graph, modality, size,\n plane, quantity, color and shape\n- Knowledge-graph questions (`base_type=kvqa`) ask about causes, symptoms, treatments and functions\n that cannot be read off the image\n\n## Evaluation Notes\n\n- Primary metric: **Accuracy** by normalized exact match against the single reference answer\n- Reported as four subsets, `_`, grouped into an English and a Chinese\n category; the overall score is the sample-weighted mean\n- Normalization follows the official answer preprocessing (lower-case, punctuation and article\n removal, word-form numbers mapped to digits, `x ray` unified to `xray`) and adds what generative\n answering requires: yes/no synonyms collapse onto one label, because Chinese references express\n the same polarity as 是的 / 有 / 包含 / 可以 or 不是 / 没有 / 不包含 / 不可以, and the Chinese X光 / X射线\n and 两个 / 二 spellings resolve to the English `X-Ray` and digit references the Chinese half keeps\n- Answers are read from the `ANSWER:` line requested by the prompt; when the model does not emit\n one, the whole reply is normalized instead, so a reply that only restates the question scores 0\n- Exact match is strict by design, matching the original classification-style evaluation: a\n reference such as `Lung, Spinal Cord`, a knowledge-graph list of treatments, or `T2` answered as\n `T2-weighted` only counts when the model reproduces the reference wording, so open-ended\n accuracy on the knowledge-graph questions is expected to be low\n- Strictness also costs answers that differ from the reference only in wording or granularity\n (`Right` answered as `Right Side`, 胸腔 answered as 胸部, one item of a multi-item reference). Set\n `judge.strategy='llm_recall'` to have an LLM judge re-check only the rule-based failures; that\n score is more lenient than the published numbers and is not directly comparable to them\n- Images ship as a single `imgs.zip` (about 200 MB) and are read directly from the archive\n- [Paper](https://arxiv.org/abs/2102.09542) | [Project page](https://www.med-vqa.com/slake/)\n", "prompt_template": "", "system_prompt": "", "few_shot_prompt_template": "", @@ -244,11 +244,11 @@ "truncated": false }, "readme": { - "en": "# SLAKE\n\n\n## Overview\n\nSLAKE is a bilingual (English / Chinese) radiology visual question answering benchmark built by\nphysicians on CT, MRI and X-Ray images. Questions cover both purely visual properties of the scan\nand medical knowledge that has to be recalled on top of what the image shows.\n\n## Task Description\n\n- **Task Type**: Medical visual question answering (free-form short answer)\n- **Input**: A radiology image plus a question in English or Chinese\n- **Output**: A single word or short phrase, in the language of the question\n- **Domain**: Radiology (chest, abdomen, brain, pelvis, neck)\n\n## Key Features\n\n- 2,094 test questions over 180 images, roughly balanced between English (1,061) and Chinese (1,033)\n- Every question is labelled `OPEN` (free answer) or `CLOSED` (answer drawn from a small closed set,\n mostly yes/no), which is the breakdown the original paper reports\n- Questions span ten semantic types: organ, position, abnormality, knowledge-graph, modality, size,\n plane, quantity, color and shape\n- Knowledge-graph questions (`base_type=kvqa`) ask about causes, symptoms, treatments and functions\n that cannot be read off the image\n\n## Evaluation Notes\n\n- Primary metric: **Accuracy** by normalized exact match against the single reference answer\n- Reported as four subsets, `_`, grouped into an English and a Chinese\n category; the overall score is the sample-weighted mean\n- Normalization follows the official answer preprocessing (lower-case, punctuation and article\n removal, word-form numbers mapped to digits, `x ray` unified to `xray`) and adds what generative\n answering requires: yes/no synonyms collapse onto one label, because Chinese references express\n the same polarity as 是的 / 有 / 包含 / 可以 or 不是 / 没有 / 不包含 / 不可以, and the Chinese X光 / X射线\n and 两个 / 二 spellings resolve to the English `X-Ray` and digit references the Chinese half keeps\n- Answers are read from the `ANSWER:` line requested by the prompt; when the model does not emit\n one, the whole reply is normalized instead, so a reply that only restates the question scores 0\n- Exact match is strict by design, matching the original classification-style evaluation: a\n reference such as `Lung, Spinal Cord`, a knowledge-graph list of treatments, or `T2` answered as\n `T2-weighted` only counts when the model reproduces the reference wording, so open-ended\n accuracy on the knowledge-graph questions is expected to be low\n- Strictness also costs answers that differ from the reference only in wording or granularity\n (`Right` answered as `Right Side`, 胸腔 answered as 胸部, one item of a multi-item reference). Set\n `judge_strategy='llm_recall'` to have an LLM judge re-check only the rule-based failures; that\n score is more lenient than the published numbers and is not directly comparable to them\n- Images ship as a single `imgs.zip` (about 200 MB) and are read directly from the archive\n- [Paper](https://arxiv.org/abs/2102.09542) | [Project page](https://www.med-vqa.com/slake/)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `slake` |\n| **Dataset ID** | [evalscope/SLAKE](https://modelscope.cn/datasets/evalscope/SLAKE/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2102.09542) |\n| **Tags** | `Medical`, `MultiModal`, `QA` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 2,094 |\n| Prompt Length (Mean) | 130.2 chars |\n| Prompt Length (Min/Max) | 60 / 257 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `en_open` | 645 | 195.09 | 168 | 257 |\n| `en_closed` | 416 | 187.99 | 162 | 253 |\n| `zh_open` | 613 | 67.07 | 61 | 79 |\n| `zh_closed` | 420 | 65.44 | 60 | 82 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 2,094 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 240x240 - 1024x1024 |\n| Formats | jpeg |\n\n\n## Sample Example\n\n**Subset**: `en_open`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"4366e0b3\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~63.2KB]\"\n },\n {\n \"text\": \"What modality is used to take this image?\\nAnswer the question with a single word or phrase in English.\\nThe last line of your response must be of the form \\\"ANSWER: \\\" (without quotes).\"\n }\n ]\n }\n ],\n \"target\": \"CT\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"en_open\",\n \"metadata\": {\n \"qid\": 11934,\n \"img_name\": \"xmlab102/source.jpg\",\n \"answer_type\": \"OPEN\",\n \"content_type\": \"Modality\",\n \"modality\": \"CT\"\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets slake \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['slake'],\n dataset_args={\n 'slake': {\n # subset_list: ['en_open', 'en_closed', 'zh_open'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# SLAKE\n\n\n## 概述\n\nSLAKE 是一个双语（英语 / 中文）的放射学视觉问答基准数据集，由医生基于 CT、MRI 和 X 光图像构建而成。问题既涵盖影像本身的纯视觉属性，也包含需结合医学知识才能回答的内容。\n\n## 任务描述\n\n- **任务类型**：医学视觉问答（自由形式的简短回答）\n- **输入**：一张放射学图像以及一道英语或中文问题\n- **输出**：一个单词或短语，语言与问题一致\n- **领域**：放射学（胸部、腹部、脑部、骨盆、颈部）\n\n## 主要特点\n\n- 包含 180 张图像上的 2,094 道测试问题，英语（1,061）和中文（1,033）大致均衡\n- 每个问题均标注为 `OPEN`（自由回答）或 `CLOSED`（答案来自一个小的封闭集合，多为是/否类问题），这是原始论文所采用的划分方式\n- 问题涵盖十种语义类型：器官、位置、异常、知识图谱、模态、大小、平面、数量、颜色和形状\n- 知识图谱类问题（`base_type=kvqa`）涉及病因、症状、治疗和功能等无法直接从图像中读取的信息\n\n## 评估说明\n\n- **主要指标**：通过归一化后的精确匹配（exact match）计算**准确率**，与单一参考答案对比\n- 结果按四个子集报告：`_`，分为英语和中文两类；总体得分为样本加权平均值\n- 归一化遵循官方答案预处理规则（转小写、去除标点和冠词、将文字数字转换为阿拉伯数字、统一 `x ray` 为 `xray`），并额外加入生成式回答所需处理： \n - 英语中 yes/no 的同义词统一归为一个标签； \n - 中文中表达相同极性的 是的 / 有 / 包含 / 可以 或 不是 / 没有 / 不包含 / 不可以 也统一归为一个标签； \n - 中文中的 X光 / X射线 和 两个 / 二 分别映射为英文的 `X-Ray` 和数字形式（中文部分保留数字引用）\n- 答案从提示中要求的 `ANSWER:` 行读取；若模型未输出该行，则对整个回复进行归一化处理，因此仅复述问题的回复得分为 0\n- 精确匹配设计上较为严格，与原始分类式评估一致：例如参考答案为 `Lung, Spinal Cord`、知识图谱中的治疗列表，或 `T2` 被回答为 `T2-weighted`，只有当模型完全复现参考措辞时才计为正确，因此知识图谱类开放问题的准确率预期较低\n- 严格性也会导致仅在措辞或粒度上与参考答案不同的回答被判错（如 `Right` 回答为 `Right Side`，胸腔 回答为 胸部，或多答案参考中只答对其中一项）。设置 `judge_strategy='llm_recall'` 可让大语言模型重新检查仅因规则判断失败的情况；该得分比已发表结果更宽松，不可直接与其比较\n- 图像以单个 `imgs.zip` 文件形式提供（约 200 MB），直接从压缩包中读取\n- [论文](https://arxiv.org/abs/2102.09542) | [项目主页](https://www.med-vqa.com/slake/)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `slake` |\n| **数据集ID** | [evalscope/SLAKE](https://modelscope.cn/datasets/evalscope/SLAKE/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2102.09542) |\n| **标签** | `Medical`, `MultiModal`, `QA` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估分割** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 2,094 |\n| 提示词长度（平均） | 130.2 字符 |\n| 提示词长度（最小/最大） | 60 / 257 字符 |\n\n**各子集统计信息：**\n\n| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 |\n|--------|---------|-------------|------------|------------|\n| `en_open` | 645 | 195.09 | 168 | 257 |\n| `en_closed` | 416 | 187.99 | 162 | 253 |\n| `zh_open` | 613 | 67.07 | 61 | 79 |\n| `zh_closed` | 420 | 65.44 | 60 | 82 |\n\n**图像统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 总图像数 | 2,094 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 240x240 - 1024x1024 |\n| 格式 | jpeg |\n\n\n## 样例示例\n\n**子集**: `en_open`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"4366e0b3\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~63.2KB]\"\n },\n {\n \"text\": \"What modality is used to take this image?\\nAnswer the question with a single word or phrase in English.\\nThe last line of your response must be of the form \\\"ANSWER: \\\" (without quotes).\"\n }\n ]\n }\n ],\n \"target\": \"CT\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"en_open\",\n \"metadata\": {\n \"qid\": 11934,\n \"img_name\": \"xmlab102/source.jpg\",\n \"answer_type\": \"OPEN\",\n \"content_type\": \"Modality\",\n \"modality\": \"CT\"\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets slake \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['slake'],\n dataset_args={\n 'slake': {\n # subset_list: ['en_open', 'en_closed', 'zh_open'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "41500e6bc450c71fc3711c415d117129", + "en": "# SLAKE\n\n\n## Overview\n\nSLAKE is a bilingual (English / Chinese) radiology visual question answering benchmark built by\nphysicians on CT, MRI and X-Ray images. Questions cover both purely visual properties of the scan\nand medical knowledge that has to be recalled on top of what the image shows.\n\n## Task Description\n\n- **Task Type**: Medical visual question answering (free-form short answer)\n- **Input**: A radiology image plus a question in English or Chinese\n- **Output**: A single word or short phrase, in the language of the question\n- **Domain**: Radiology (chest, abdomen, brain, pelvis, neck)\n\n## Key Features\n\n- 2,094 test questions over 180 images, roughly balanced between English (1,061) and Chinese (1,033)\n- Every question is labelled `OPEN` (free answer) or `CLOSED` (answer drawn from a small closed set,\n mostly yes/no), which is the breakdown the original paper reports\n- Questions span ten semantic types: organ, position, abnormality, knowledge-graph, modality, size,\n plane, quantity, color and shape\n- Knowledge-graph questions (`base_type=kvqa`) ask about causes, symptoms, treatments and functions\n that cannot be read off the image\n\n## Evaluation Notes\n\n- Primary metric: **Accuracy** by normalized exact match against the single reference answer\n- Reported as four subsets, `_`, grouped into an English and a Chinese\n category; the overall score is the sample-weighted mean\n- Normalization follows the official answer preprocessing (lower-case, punctuation and article\n removal, word-form numbers mapped to digits, `x ray` unified to `xray`) and adds what generative\n answering requires: yes/no synonyms collapse onto one label, because Chinese references express\n the same polarity as 是的 / 有 / 包含 / 可以 or 不是 / 没有 / 不包含 / 不可以, and the Chinese X光 / X射线\n and 两个 / 二 spellings resolve to the English `X-Ray` and digit references the Chinese half keeps\n- Answers are read from the `ANSWER:` line requested by the prompt; when the model does not emit\n one, the whole reply is normalized instead, so a reply that only restates the question scores 0\n- Exact match is strict by design, matching the original classification-style evaluation: a\n reference such as `Lung, Spinal Cord`, a knowledge-graph list of treatments, or `T2` answered as\n `T2-weighted` only counts when the model reproduces the reference wording, so open-ended\n accuracy on the knowledge-graph questions is expected to be low\n- Strictness also costs answers that differ from the reference only in wording or granularity\n (`Right` answered as `Right Side`, 胸腔 answered as 胸部, one item of a multi-item reference). Set\n `judge.strategy='llm_recall'` to have an LLM judge re-check only the rule-based failures; that\n score is more lenient than the published numbers and is not directly comparable to them\n- Images ship as a single `imgs.zip` (about 200 MB) and are read directly from the archive\n- [Paper](https://arxiv.org/abs/2102.09542) | [Project page](https://www.med-vqa.com/slake/)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `slake` |\n| **Dataset ID** | [evalscope/SLAKE](https://modelscope.cn/datasets/evalscope/SLAKE/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2102.09542) |\n| **Tags** | `Medical`, `MultiModal`, `QA` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 2,094 |\n| Prompt Length (Mean) | 130.2 chars |\n| Prompt Length (Min/Max) | 60 / 257 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `en_open` | 645 | 195.09 | 168 | 257 |\n| `en_closed` | 416 | 187.99 | 162 | 253 |\n| `zh_open` | 613 | 67.07 | 61 | 79 |\n| `zh_closed` | 420 | 65.44 | 60 | 82 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 2,094 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 240x240 - 1024x1024 |\n| Formats | jpeg |\n\n\n## Sample Example\n\n**Subset**: `en_open`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"4366e0b3\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~63.2KB]\"\n },\n {\n \"text\": \"What modality is used to take this image?\\nAnswer the question with a single word or phrase in English.\\nThe last line of your response must be of the form \\\"ANSWER: \\\" (without quotes).\"\n }\n ]\n }\n ],\n \"target\": \"CT\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"en_open\",\n \"metadata\": {\n \"qid\": 11934,\n \"img_name\": \"xmlab102/source.jpg\",\n \"answer_type\": \"OPEN\",\n \"content_type\": \"Modality\",\n \"modality\": \"CT\"\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets slake \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['slake'],\n dataset_args={\n 'slake': {\n # subset_list: ['en_open', 'en_closed', 'zh_open'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# SLAKE\n\n\n## 概述\n\nSLAKE 是一个双语（英语 / 中文）放射学视觉问答基准数据集，由医生基于 CT、MRI 和 X 光图像构建而成。问题既涵盖影像的纯视觉属性，也包含需要结合医学知识才能回答的内容。\n\n## 任务描述\n\n- **任务类型**：医学视觉问答（自由形式的简短答案）\n- **输入**：一张放射学图像以及一条英文或中文问题\n- **输出**：单个单词或短语，语言需与问题一致\n- **领域**：放射学（胸部、腹部、脑部、盆腔、颈部）\n\n## 主要特点\n\n- 包含 180 张图像上的 2,094 个测试问题，英语（1,061）和中文（1,033）大致均衡\n- 每个问题均标注为 `OPEN`（自由作答）或 `CLOSED`（答案来自小型封闭集合，多为是/否类问题），此分类方式与原始论文一致\n- 问题涵盖十种语义类型：器官、位置、异常、知识图谱、模态、大小、层面、数量、颜色和形状\n- 知识图谱类问题（`base_type=kvqa`）涉及病因、症状、治疗和功能等无法直接从图像中读取的信息\n\n## 评估说明\n\n- **主要指标**：通过归一化精确匹配（normalized exact match）计算**准确率**，并与唯一参考答案对比\n- 结果按四个子集报告：`_`，分为英语和中文两类；总体得分为样本加权平均值\n- 归一化遵循官方答案预处理规则（转小写、去除标点和冠词、将文字数字转换为阿拉伯数字、统一 `x ray` 为 `xray`），并额外加入生成式问答所需处理： \n - 英语中 yes/no 的同义词统一归为一个标签 \n - 中文中表达相同极性的 是的 / 有 / 包含 / 可以 或 不是 / 没有 / 不包含 / 不可以 也统一归为一个标签 \n - 中文中的 X光 / X射线 和 两个 / 二 统一映射为英文的 `X-Ray` 和对应数字形式\n- 答案从提示中要求的 `ANSWER:` 行读取；若模型未输出该行，则对整个回复进行归一化处理，因此仅复述问题的回复得分为 0\n- 精确匹配设计上较为严格，与原始分类式评估一致：例如参考答案为 `Lung, Spinal Cord`、知识图谱中的治疗列表，或 `T2` 被回答为 `T2-weighted` 时，仅当模型完全复现参考措辞才算正确，因此知识图谱类开放问题的准确率预期较低\n- 严格性也会导致仅在措辞或粒度上与参考答案不同的回答被判错（如 `Right` 回答为 `Right Side`，胸腔 回答为 胸部，或多答案参考中只答对其中一项）。设置 `judge.strategy='llm_recall'` 可让大语言模型（LLM）重新审核仅因规则判断失败的情况；该分数比已发表结果更宽松，不可直接比较\n- 图像以单个 `imgs.zip` 文件形式提供（约 200 MB），直接从压缩包中读取\n- [论文](https://arxiv.org/abs/2102.09542) | [项目主页](https://www.med-vqa.com/slake/)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `slake` |\n| **数据集ID** | [evalscope/SLAKE](https://modelscope.cn/datasets/evalscope/SLAKE/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2102.09542) |\n| **标签** | `Medical`, `MultiModal`, `QA` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估分割** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 2,094 |\n| 提示词长度（平均） | 130.2 字符 |\n| 提示词长度（最小/最大） | 60 / 257 字符 |\n\n**各子集统计信息：**\n\n| 子集 | 样本数 | 提示平均长度 | 提示最小长度 | 提示最大长度 |\n|--------|---------|-------------|------------|------------|\n| `en_open` | 645 | 195.09 | 168 | 257 |\n| `en_closed` | 416 | 187.99 | 162 | 253 |\n| `zh_open` | 613 | 67.07 | 61 | 79 |\n| `zh_closed` | 420 | 65.44 | 60 | 82 |\n\n**图像统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 总图像数 | 2,094 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 240x240 - 1024x1024 |\n| 格式 | jpeg |\n\n\n## 样例示例\n\n**子集**: `en_open`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"4366e0b3\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpeg, ~63.2KB]\"\n },\n {\n \"text\": \"What modality is used to take this image?\\nAnswer the question with a single word or phrase in English.\\nThe last line of your response must be of the form \\\"ANSWER: \\\" (without quotes).\"\n }\n ]\n }\n ],\n \"target\": \"CT\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"en_open\",\n \"metadata\": {\n \"qid\": 11934,\n \"img_name\": \"xmlab102/source.jpg\",\n \"answer_type\": \"OPEN\",\n \"content_type\": \"Modality\",\n \"modality\": \"CT\"\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets slake \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['slake'],\n dataset_args={\n 'slake': {\n # subset_list: ['en_open', 'en_closed', 'zh_open'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "ad13b8df34b6e046a4e93f32c5af3381", "needs_translation": false }, - "updated_at": "2026-08-18T10:50:00.854118", - "translation_updated_at": "2026-08-18T10:50:05" + "updated_at": "2026-08-20T10:40:24.370520", + "translation_updated_at": "2026-08-20T10:41:18" } diff --git a/evalscope/benchmarks/_meta/tir_bench.json b/evalscope/benchmarks/_meta/tir_bench.json index 551eb7d64..70c7cca85 100644 --- a/evalscope/benchmarks/_meta/tir_bench.json +++ b/evalscope/benchmarks/_meta/tir_bench.json @@ -29,7 +29,7 @@ "jigsaw", "maze" ], - "description": "\n## Overview\n\nTIR-Bench (Thinking-with-Images Reasoning Benchmark) is a comprehensive multimodal benchmark\nthat evaluates agentic visual reasoning capabilities of vision-language models. It covers\ndiverse task categories requiring spatial, compositional, and multi-step visual reasoning.\n\n## Task Description\n\n- **Task Type**: Multi-task Visual Reasoning (MCQ, OCR, word search, spot difference, jigsaw, etc.)\n- **Input**: One or two images + question (most tasks use multiple-choice format)\n- **Output**: Answer letter (MCQ) or numeric/text response depending on task type\n- **Domains**: instrument, color, refcoco, rotation_game, math, word_search, visual_search, ocr, symbolic, spot_difference, contrast, jigsaw, maze\n\n## Key Features\n\n- 1,215 test samples across 13 diverse visual reasoning task categories\n- Covers single-image and dual-image reasoning scenarios\n- Answers span letter choices (A-J), integers, floats, and text\n- Task-specific scoring with LLM-as-judge fallback for robust evaluation\n\n## Evaluation Notes\n\n- Default evaluation uses the **test** split (1,215 samples)\n- Primary metric: **Accuracy** (`accuracy`)\n- Images are downloaded as `data.zip` from ModelScope and extracted automatically\n- Rule-based scoring: OCR (substring match), jigsaw (grid IoU), spot_difference (set IoU),\n word_search (numeric match), all other tasks (MCQ / numeric judge)\n- **Recommended**: set `judge_strategy=JudgeStrategy.LLM_RECALL` and provide `judge_model_args`\n to activate LLM-as-judge as a recall mechanism — the judge is called only when rule-based\n scoring gives 0, providing more accurate evaluation without unnecessary API overhead\n- [Paper](https://arxiv.org/abs/2511.01833) | [GitHub](https://github.com/agents-x-project/TIR-Bench)\n", + "description": "\n## Overview\n\nTIR-Bench (Thinking-with-Images Reasoning Benchmark) is a comprehensive multimodal benchmark\nthat evaluates agentic visual reasoning capabilities of vision-language models. It covers\ndiverse task categories requiring spatial, compositional, and multi-step visual reasoning.\n\n## Task Description\n\n- **Task Type**: Multi-task Visual Reasoning (MCQ, OCR, word search, spot difference, jigsaw, etc.)\n- **Input**: One or two images + question (most tasks use multiple-choice format)\n- **Output**: Answer letter (MCQ) or numeric/text response depending on task type\n- **Domains**: instrument, color, refcoco, rotation_game, math, word_search, visual_search, ocr, symbolic, spot_difference, contrast, jigsaw, maze\n\n## Key Features\n\n- 1,215 test samples across 13 diverse visual reasoning task categories\n- Covers single-image and dual-image reasoning scenarios\n- Answers span letter choices (A-J), integers, floats, and text\n- Task-specific scoring with LLM-as-judge fallback for robust evaluation\n\n## Evaluation Notes\n\n- Default evaluation uses the **test** split (1,215 samples)\n- Primary metric: **Accuracy** (`accuracy`)\n- Images are downloaded as `data.zip` from ModelScope and extracted automatically\n- Rule-based scoring: OCR (substring match), jigsaw (grid IoU), spot_difference (set IoU),\n word_search (numeric match), all other tasks (MCQ / numeric judge)\n- **Recommended**: set `judge.strategy='llm_recall'` and provide `judge.models`\n to activate LLM-as-judge as a recall mechanism — the judge is called only when rule-based\n scoring gives 0, providing more accurate evaluation without unnecessary API overhead\n- [Paper](https://arxiv.org/abs/2511.01833) | [GitHub](https://github.com/agents-x-project/TIR-Bench)\n", "prompt_template": "", "system_prompt": "", "few_shot_prompt_template": "", @@ -635,11 +635,11 @@ "truncated": false }, "readme": { - "en": "# TIR-Bench\n\n\n## Overview\n\nTIR-Bench (Thinking-with-Images Reasoning Benchmark) is a comprehensive multimodal benchmark\nthat evaluates agentic visual reasoning capabilities of vision-language models. It covers\ndiverse task categories requiring spatial, compositional, and multi-step visual reasoning.\n\n## Task Description\n\n- **Task Type**: Multi-task Visual Reasoning (MCQ, OCR, word search, spot difference, jigsaw, etc.)\n- **Input**: One or two images + question (most tasks use multiple-choice format)\n- **Output**: Answer letter (MCQ) or numeric/text response depending on task type\n- **Domains**: instrument, color, refcoco, rotation_game, math, word_search, visual_search, ocr, symbolic, spot_difference, contrast, jigsaw, maze\n\n## Key Features\n\n- 1,215 test samples across 13 diverse visual reasoning task categories\n- Covers single-image and dual-image reasoning scenarios\n- Answers span letter choices (A-J), integers, floats, and text\n- Task-specific scoring with LLM-as-judge fallback for robust evaluation\n\n## Evaluation Notes\n\n- Default evaluation uses the **test** split (1,215 samples)\n- Primary metric: **Accuracy** (`accuracy`)\n- Images are downloaded as `data.zip` from ModelScope and extracted automatically\n- Rule-based scoring: OCR (substring match), jigsaw (grid IoU), spot_difference (set IoU),\n word_search (numeric match), all other tasks (MCQ / numeric judge)\n- **Recommended**: set `judge_strategy=JudgeStrategy.LLM_RECALL` and provide `judge_model_args`\n to activate LLM-as-judge as a recall mechanism — the judge is called only when rule-based\n scoring gives 0, providing more accurate evaluation without unnecessary API overhead\n- [Paper](https://arxiv.org/abs/2511.01833) | [GitHub](https://github.com/agents-x-project/TIR-Bench)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `tir_bench` |\n| **Dataset ID** | [evalscope/TIR-Bench](https://modelscope.cn/datasets/evalscope/TIR-Bench/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2511.01833) |\n| **Tags** | `MultiModal`, `QA`, `Reasoning` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 1,215 |\n| Prompt Length (Mean) | 384.97 chars |\n| Prompt Length (Min/Max) | 19 / 4039 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `instrument` | 80 | 110.96 | 57 | 196 |\n| `color` | 100 | 130.26 | 98 | 241 |\n| `refcoco` | 120 | 144.51 | 132 | 182 |\n| `rotation_game` | 75 | 146.44 | 140 | 148 |\n| `math` | 120 | 126.69 | 50 | 397 |\n| `word_search` | 100 | 126.72 | 24 | 307 |\n| `visual_search` | 120 | 111.64 | 19 | 501 |\n| `ocr` | 60 | 35.08 | 29 | 116 |\n| `symbolic` | 50 | 88.18 | 66 | 243 |\n| `spot_difference` | 100 | 1114.79 | 93 | 1379 |\n| `contrast` | 50 | 48.1 | 31 | 123 |\n| `jigsaw` | 120 | 605 | 605 | 605 |\n| `maze` | 120 | 1527.06 | 626 | 4039 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 1,255 |\n| Images per Sample | min: 1, max: 2, mean: 1.03 |\n| Resolution Range | 60x23 - 6944x9280 |\n| Formats | jpeg, mpo, png, webp |\n\n\n## Sample Example\n\n**Subset**: `instrument`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"dd25f10d\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpg, ~2.9MB]\"\n },\n {\n \"text\": \"According to the image, what is the thermometer reading in Fahrenheit? Answer as an integer like 1,2,3.\"\n }\n ]\n }\n ],\n \"target\": \"72\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"instrument\",\n \"metadata\": {\n \"task\": \"instrument\",\n \"meta_data\": {},\n \"id\": 6\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets tir_bench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['tir_bench'],\n dataset_args={\n 'tir_bench': {\n # subset_list: ['instrument', 'color', 'refcoco'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# TIR-Bench\n\n\n## 概述\n\nTIR-Bench（Thinking-with-Images Reasoning Benchmark）是一个全面的多模态基准测试，用于评估视觉语言模型的具身视觉推理能力。该基准涵盖多种任务类别，要求模型具备空间、组合式以及多步骤的视觉推理能力。\n\n## 任务描述\n\n- **任务类型**：多任务视觉推理（包括选择题、OCR、单词搜索、找不同、拼图等）\n- **输入**：一张或两张图像 + 问题（大多数任务采用选择题格式）\n- **输出**：答案字母（选择题）或数字/文本形式的答案（取决于任务类型）\n- **领域**：instrument（仪器读数）、color（颜色识别）、refcoco（指代表达理解）、rotation_game（旋转游戏）、math（数学题）、word_search（单词搜索）、visual_search（视觉搜索）、ocr（光学字符识别）、symbolic（符号推理）、spot_difference（找不同）、contrast（对比分析）、jigsaw（拼图）、maze（迷宫）\n\n## 核心特性\n\n- 包含13种多样化的视觉推理任务类别，共计1,215个测试样本\n- 覆盖单图和双图推理场景\n- 答案形式包括字母选项（A-J）、整数、浮点数和文本\n- 采用任务特定的评分机制，并在必要时回退至基于大语言模型（LLM-as-judge）的评分方式，以实现稳健评估\n\n## 评估说明\n\n- 默认使用 **test** 分割进行评估（共1,215个样本）\n- 主要指标：**准确率**（`accuracy`）\n- 图像数据通过 ModelScope 下载为 `data.zip` 并自动解压\n- 基于规则的评分方式：\n - OCR：子字符串匹配\n - 拼图（jigsaw）：网格 IoU\n - 找不同（spot_difference）：集合 IoU\n - 单词搜索（word_search）：数值匹配\n - 其他所有任务：选择题或数值判断\n- **推荐设置**：将 `judge_strategy=JudgeStrategy.LLM_RECALL` 并提供 `judge_model_args`，以启用 LLM-as-judge 作为召回机制——仅当基于规则的评分结果为0时才调用评判模型，从而在避免不必要 API 开销的同时提升评估准确性\n- [论文](https://arxiv.org/abs/2511.01833) | [GitHub](https://github.com/agents-x-project/TIR-Bench)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `tir_bench` |\n| **数据集ID** | [evalscope/TIR-Bench](https://modelscope.cn/datasets/evalscope/TIR-Bench/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2511.01833) |\n| **标签** | `MultiModal`, `QA`, `Reasoning` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 1,215 |\n| 提示词长度（平均） | 384.97 字符 |\n| 提示词长度（最小/最大） | 19 / 4039 字符 |\n\n**各子集统计数据：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `instrument` | 80 | 110.96 | 57 | 196 |\n| `color` | 100 | 130.26 | 98 | 241 |\n| `refcoco` | 120 | 144.51 | 132 | 182 |\n| `rotation_game` | 75 | 146.44 | 140 | 148 |\n| `math` | 120 | 126.69 | 50 | 397 |\n| `word_search` | 100 | 126.72 | 24 | 307 |\n| `visual_search` | 120 | 111.64 | 19 | 501 |\n| `ocr` | 60 | 35.08 | 29 | 116 |\n| `symbolic` | 50 | 88.18 | 66 | 243 |\n| `spot_difference` | 100 | 1114.79 | 93 | 1379 |\n| `contrast` | 50 | 48.1 | 31 | 123 |\n| `jigsaw` | 120 | 605 | 605 | 605 |\n| `maze` | 120 | 1527.06 | 626 | 4039 |\n\n**图像统计数据：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 1,255 |\n| 每样本图像数 | 最小: 1, 最大: 2, 平均: 1.03 |\n| 分辨率范围 | 60x23 - 6944x9280 |\n| 格式 | jpeg, mpo, png, webp |\n\n\n## 样例示例\n\n**子集**: `instrument`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"dd25f10d\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpg, ~2.9MB]\"\n },\n {\n \"text\": \"According to the image, what is the thermometer reading in Fahrenheit? Answer as an integer like 1,2,3.\"\n }\n ]\n }\n ],\n \"target\": \"72\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"instrument\",\n \"metadata\": {\n \"task\": \"instrument\",\n \"meta_data\": {},\n \"id\": 6\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets tir_bench \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['tir_bench'],\n dataset_args={\n 'tir_bench': {\n # subset_list: ['instrument', 'color', 'refcoco'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "765987b71ac4d750efdd827e6c2b2758", + "en": "# TIR-Bench\n\n\n## Overview\n\nTIR-Bench (Thinking-with-Images Reasoning Benchmark) is a comprehensive multimodal benchmark\nthat evaluates agentic visual reasoning capabilities of vision-language models. It covers\ndiverse task categories requiring spatial, compositional, and multi-step visual reasoning.\n\n## Task Description\n\n- **Task Type**: Multi-task Visual Reasoning (MCQ, OCR, word search, spot difference, jigsaw, etc.)\n- **Input**: One or two images + question (most tasks use multiple-choice format)\n- **Output**: Answer letter (MCQ) or numeric/text response depending on task type\n- **Domains**: instrument, color, refcoco, rotation_game, math, word_search, visual_search, ocr, symbolic, spot_difference, contrast, jigsaw, maze\n\n## Key Features\n\n- 1,215 test samples across 13 diverse visual reasoning task categories\n- Covers single-image and dual-image reasoning scenarios\n- Answers span letter choices (A-J), integers, floats, and text\n- Task-specific scoring with LLM-as-judge fallback for robust evaluation\n\n## Evaluation Notes\n\n- Default evaluation uses the **test** split (1,215 samples)\n- Primary metric: **Accuracy** (`accuracy`)\n- Images are downloaded as `data.zip` from ModelScope and extracted automatically\n- Rule-based scoring: OCR (substring match), jigsaw (grid IoU), spot_difference (set IoU),\n word_search (numeric match), all other tasks (MCQ / numeric judge)\n- **Recommended**: set `judge.strategy='llm_recall'` and provide `judge.models`\n to activate LLM-as-judge as a recall mechanism — the judge is called only when rule-based\n scoring gives 0, providing more accurate evaluation without unnecessary API overhead\n- [Paper](https://arxiv.org/abs/2511.01833) | [GitHub](https://github.com/agents-x-project/TIR-Bench)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `tir_bench` |\n| **Dataset ID** | [evalscope/TIR-Bench](https://modelscope.cn/datasets/evalscope/TIR-Bench/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2511.01833) |\n| **Tags** | `MultiModal`, `QA`, `Reasoning` |\n| **Metrics** | `accuracy` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 1,215 |\n| Prompt Length (Mean) | 384.97 chars |\n| Prompt Length (Min/Max) | 19 / 4039 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `instrument` | 80 | 110.96 | 57 | 196 |\n| `color` | 100 | 130.26 | 98 | 241 |\n| `refcoco` | 120 | 144.51 | 132 | 182 |\n| `rotation_game` | 75 | 146.44 | 140 | 148 |\n| `math` | 120 | 126.69 | 50 | 397 |\n| `word_search` | 100 | 126.72 | 24 | 307 |\n| `visual_search` | 120 | 111.64 | 19 | 501 |\n| `ocr` | 60 | 35.08 | 29 | 116 |\n| `symbolic` | 50 | 88.18 | 66 | 243 |\n| `spot_difference` | 100 | 1114.79 | 93 | 1379 |\n| `contrast` | 50 | 48.1 | 31 | 123 |\n| `jigsaw` | 120 | 605 | 605 | 605 |\n| `maze` | 120 | 1527.06 | 626 | 4039 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 1,255 |\n| Images per Sample | min: 1, max: 2, mean: 1.03 |\n| Resolution Range | 60x23 - 6944x9280 |\n| Formats | jpeg, mpo, png, webp |\n\n\n## Sample Example\n\n**Subset**: `instrument`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"dd25f10d\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpg, ~2.9MB]\"\n },\n {\n \"text\": \"According to the image, what is the thermometer reading in Fahrenheit? Answer as an integer like 1,2,3.\"\n }\n ]\n }\n ],\n \"target\": \"72\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"instrument\",\n \"metadata\": {\n \"task\": \"instrument\",\n \"meta_data\": {},\n \"id\": 6\n }\n}\n```\n\n## Prompt Template\n\n*No prompt template defined.*\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets tir_bench \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['tir_bench'],\n dataset_args={\n 'tir_bench': {\n # subset_list: ['instrument', 'color', 'refcoco'] # optional, evaluate specific subsets\n }\n },\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# TIR-Bench\n\n\n## 概述\n\nTIR-Bench（Thinking-with-Images Reasoning Benchmark）是一个全面的多模态基准测试，用于评估视觉语言模型的具身视觉推理能力。该基准涵盖多种任务类别，要求模型具备空间、组合式以及多步骤的视觉推理能力。\n\n## 任务描述\n\n- **任务类型**：多任务视觉推理（包括选择题、OCR、单词搜索、找不同、拼图等）\n- **输入**：一张或两张图像 + 问题（大多数任务采用选择题格式）\n- **输出**：答案字母（选择题）或数字/文本响应（取决于任务类型）\n- **领域**：instrument（仪器）、color（颜色）、refcoco、rotation_game（旋转游戏）、math（数学）、word_search（单词搜索）、visual_search（视觉搜索）、ocr、symbolic（符号）、spot_difference（找不同）、contrast（对比）、jigsaw（拼图）、maze（迷宫）\n\n## 核心特性\n\n- 包含13种多样化的视觉推理任务类别，共计1,215个测试样本\n- 覆盖单图和双图推理场景\n- 答案形式包括字母选项（A-J）、整数、浮点数和文本\n- 采用任务特定评分机制，并在必要时回退至LLM-as-judge以实现稳健评估\n\n## 评估说明\n\n- 默认使用 **test** 划分进行评估（共1,215个样本）\n- 主要指标：**准确率**（`accuracy`）\n- 图像数据通过 ModelScope 下载为 `data.zip` 并自动解压\n- 基于规则的评分方式：\n - OCR（子串匹配）\n - 拼图（grid IoU）\n - 找不同（set IoU）\n - 单词搜索（数值匹配）\n - 其他所有任务（选择题 / 数值判断）\n- **推荐设置**：将 `judge.strategy='llm_recall'` 并提供 `judge.models`，以启用 LLM-as-judge 作为召回机制——仅当基于规则的评分结果为0时才调用判别器，在避免不必要API开销的同时提升评估准确性\n- [论文](https://arxiv.org/abs/2511.01833) | [GitHub](https://github.com/agents-x-project/TIR-Bench)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `tir_bench` |\n| **数据集ID** | [evalscope/TIR-Bench](https://modelscope.cn/datasets/evalscope/TIR-Bench/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2511.01833) |\n| **标签** | `MultiModal`, `QA`, `Reasoning` |\n| **指标** | `accuracy` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 1,215 |\n| 提示词长度（平均） | 384.97 字符 |\n| 提示词长度（最小/最大） | 19 / 4039 字符 |\n\n**各子集统计数据：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `instrument` | 80 | 110.96 | 57 | 196 |\n| `color` | 100 | 130.26 | 98 | 241 |\n| `refcoco` | 120 | 144.51 | 132 | 182 |\n| `rotation_game` | 75 | 146.44 | 140 | 148 |\n| `math` | 120 | 126.69 | 50 | 397 |\n| `word_search` | 100 | 126.72 | 24 | 307 |\n| `visual_search` | 120 | 111.64 | 19 | 501 |\n| `ocr` | 60 | 35.08 | 29 | 116 |\n| `symbolic` | 50 | 88.18 | 66 | 243 |\n| `spot_difference` | 100 | 1114.79 | 93 | 1379 |\n| `contrast` | 50 | 48.1 | 31 | 123 |\n| `jigsaw` | 120 | 605 | 605 | 605 |\n| `maze` | 120 | 1527.06 | 626 | 4039 |\n\n**图像统计数据：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 1,255 |\n| 每样本图像数 | 最小: 1, 最大: 2, 平均: 1.03 |\n| 分辨率范围 | 60x23 - 6944x9280 |\n| 格式 | jpeg, mpo, png, webp |\n\n\n## 样例示例\n\n**子集**: `instrument`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"dd25f10d\",\n \"content\": [\n {\n \"image\": \"[BASE64_IMAGE: jpg, ~2.9MB]\"\n },\n {\n \"text\": \"According to the image, what is the thermometer reading in Fahrenheit? Answer as an integer like 1,2,3.\"\n }\n ]\n }\n ],\n \"target\": \"72\",\n \"id\": 0,\n \"group_id\": 0,\n \"subset_key\": \"instrument\",\n \"metadata\": {\n \"task\": \"instrument\",\n \"meta_data\": {},\n \"id\": 6\n }\n}\n```\n\n## 提示模板\n\n*未定义提示模板。*\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets tir_bench \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['tir_bench'],\n dataset_args={\n 'tir_bench': {\n # subset_list: ['instrument', 'color', 'refcoco'] # 可选，用于评估特定子集\n }\n },\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "ea386c22b8410fb672d24b936fb281eb", "needs_translation": false }, - "updated_at": "2026-08-11T16:26:49.351098", - "translation_updated_at": "2026-04-15T19:08:56Z" + "updated_at": "2026-08-20T10:40:24.371338", + "translation_updated_at": "2026-08-20T10:41:18" } diff --git a/evalscope/benchmarks/_meta/wide_search.json b/evalscope/benchmarks/_meta/wide_search.json index 39d6194d0..4bb42b62d 100644 --- a/evalscope/benchmarks/_meta/wide_search.json +++ b/evalscope/benchmarks/_meta/wide_search.json @@ -17,20 +17,13 @@ "item_recall", "item_f1" ], - "primary_metric": { - "name": "success_rate", - "aggregation": "pass_at_k", - "dimensions": { - "scope": "all" - } - }, "few_shot_num": 0, "eval_split": "full", "train_split": "", "subset_list": [ "default" ], - "description": "\n## Overview\n\nWideSearch evaluates search agents on broad web information-seeking tasks. Each task asks the agent to collect many\natomic facts and return one structured Markdown table. EvalScope uses the ModelScope\n`bytedance-community/WideSearch` dataset.\n\n## Task Description\n\n- **Task Type**: Multi-turn search agent\n- **Input**: Natural-language collection request with an explicit table schema\n- **Output**: Complete Markdown table\n- **Dataset**: 200 tasks in the ``full`` split; 100 English and 100 Chinese\n\n## Key Features\n\n- Official single-agent protocol: language-specific system prompt, ``function_calling``, and 50 default steps.\n- Bash is available by default in a per-sample temporary local directory; Docker sandbox and MCP servers are optional.\n- A single full run derives ``all``, ``en``, and ``zh`` reports without repeated inference.\n\n## Evaluation Notes\n\n- Uses the official Markdown table alignment and hybrid rule/LLM scoring semantics.\n- Requires ``judge_strategy='auto'`` or ``'llm'`` with explicit ``judge_model_args``; rule-only scoring is unsupported.\n- See the [WideSearch usage guide](https://evalscope.readthedocs.io/en/latest/third_party/wide_search.html) for runtime\n examples and paper-style repeat settings.\n", + "description": "\n## Overview\n\nWideSearch evaluates search agents on broad web information-seeking tasks. Each task asks the agent to collect many\natomic facts and return one structured Markdown table. EvalScope uses the ModelScope\n`bytedance-community/WideSearch` dataset.\n\n## Task Description\n\n- **Task Type**: Multi-turn search agent\n- **Input**: Natural-language collection request with an explicit table schema\n- **Output**: Complete Markdown table\n- **Dataset**: 200 tasks in the ``full`` split; 100 English and 100 Chinese\n\n## Key Features\n\n- Official single-agent protocol: language-specific system prompt, ``function_calling``, and 50 default steps.\n- Bash is available by default in a per-sample temporary local directory; Docker sandbox and MCP servers are optional.\n- A single full run derives ``all``, ``en``, and ``zh`` reports without repeated inference.\n\n## Evaluation Notes\n\n- Uses the official Markdown table alignment and hybrid rule/LLM scoring semantics.\n- Requires ``judge.strategy='auto'`` or ``'llm'`` with at least one ``judge.models`` entry; rule-only scoring is unsupported.\n- See the [WideSearch usage guide](https://evalscope.readthedocs.io/en/latest/third_party/wide_search.html) for runtime\n examples and paper-style repeat settings.\n", "prompt_template": "{question}", "system_prompt": "", "few_shot_prompt_template": "", @@ -38,6 +31,13 @@ "extra_params": {}, "sandbox_config": {}, "category": "agent", + "primary_metric": { + "name": "success_rate", + "aggregation": "pass_at_k", + "dimensions": { + "scope": "all" + } + }, "agent_config": { "strategy": "function_calling", "max_steps": 50 @@ -191,11 +191,11 @@ "truncated": true }, "readme": { - "en": "# WideSearch\n\n\n## Overview\n\nWideSearch evaluates search agents on broad web information-seeking tasks. Each task asks the agent to collect many\natomic facts and return one structured Markdown table. EvalScope uses the ModelScope\n`bytedance-community/WideSearch` dataset.\n\n## Task Description\n\n- **Task Type**: Multi-turn search agent\n- **Input**: Natural-language collection request with an explicit table schema\n- **Output**: Complete Markdown table\n- **Dataset**: 200 tasks in the ``full`` split; 100 English and 100 Chinese\n\n## Key Features\n\n- Official single-agent protocol: language-specific system prompt, ``function_calling``, and 50 default steps.\n- Bash is available by default in a per-sample temporary local directory; Docker sandbox and MCP servers are optional.\n- A single full run derives ``all``, ``en``, and ``zh`` reports without repeated inference.\n\n## Evaluation Notes\n\n- Uses the official Markdown table alignment and hybrid rule/LLM scoring semantics.\n- Requires ``judge_strategy='auto'`` or ``'llm'`` with explicit ``judge_model_args``; rule-only scoring is unsupported.\n- See the [WideSearch usage guide](https://evalscope.readthedocs.io/en/latest/third_party/wide_search.html) for runtime\n examples and paper-style repeat settings.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `wide_search` |\n| **Dataset ID** | [bytedance-community/WideSearch](https://modelscope.cn/datasets/bytedance-community/WideSearch/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2508.07999) |\n| **Tags** | `Agent`, `MultiTurn`, `Retrieval` |\n| **Metrics** | `success_rate`, `row_precision`, `row_recall`, `row_f1`, `item_precision`, `item_recall`, `item_f1` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `full` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 200 |\n| Prompt Length (Mean) | 631.01 chars |\n| Prompt Length (Min/Max) | 182 / 1807 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"59f736f6\",\n \"content\": \"My son is about to start his university applications in 2025 for postgraduates but he’s still uncertain about both his major and which universities to apply to. Could you help me find the top five universities in each of the five broad subjec ... [TRUNCATED 691 chars] ... names in English. \\nUse only Arabic numerals in the ranking, for example: 1.\\nDon't ask me any questions, just output the results according to the columns without omitting cells arbitrarily. The output format is \\n```markdown\\n{data_content}\\n```.\"\n }\n ],\n \"target\": \"Subject,University,Country,QS World University Rankings by Subject 2025,QS World University Rankings 2025,Times Higher Education World University Rankings 2025,Home Page,Application Deadline,Application Fee\\nArts & Humanities,Harvard Universi ... [TRUNCATED 2837 chars] ... nuary 5,$90\\nSocial Sciences & Management,Massachusetts Institute of Technology,United States,4,1,2,https://www.mit.edu/,January 6 ,$75\\nSocial Sciences & Management,University of Cambridge,United Kingdom,5,5,5,https://www.cam.ac.uk/,Oct 15,£60\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"instance_id\": \"ws_en_001\",\n \"language\": \"en\",\n \"evaluation\": {\n \"unique_columns\": [\n \"subject\",\n \"university\"\n ],\n \"required\": [\n \"subject\",\n \"university\",\n \"qsworlduniversityrankingsbysubject2025\",\n \"qsworlduniversityrankings2025\",\n \"timeshighereducationworlduniversityrankings2025\",\n \"homepage\",\n \"applicationdeadline\",\n \"applicationfee\"\n ],\n \"eval_pipeline\": {\n \"applicationdeadline\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"llm_judge\"\n ],\n \"criterion\": \"It is sufficient if the semantics are approximately the same as the reference answer or if they point to the same entity. There is no need for a word-for-word correspondence.\\nThe month and day must be correct\"\n },\n \"applicationfee\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"llm_judge\"\n ],\n \"criterion\": \"It is sufficient if the semantics are approximately the same as the reference answer or if they point to the same entity. There is no need for a word-for-word correspondence.\\nIf there are multiple fees in the reference answer, all must be included.\"\n },\n \"homepage\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"url_match\"\n ]\n },\n \"subject\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"university\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"qsworlduniversityrankingsbysubject2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"qsworlduniversityrankings2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"timeshighereducationworlduniversityrankings2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n }\n }\n }\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets wide_search \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":50}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['wide_search'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=50,\n ),\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", - "zh": "# WideSearch\n\n\n## 概述\n\nWideSearch 用于评估搜索智能体在广泛网络信息检索任务上的表现。每个任务要求智能体收集大量原子事实，并返回一个结构化的 Markdown 表格。EvalScope 使用 ModelScope 上的 `bytedance-community/WideSearch` 数据集。\n\n## 任务描述\n\n- **任务类型**：多轮搜索智能体\n- **输入**：包含明确表格结构的自然语言信息收集请求\n- **输出**：完整的 Markdown 表格\n- **数据集**：`full` 划分中共有 200 个任务；其中 100 个为英文，100 个为中文\n\n## 核心特性\n\n- 官方单智能体协议：支持语言特定的系统提示词、``function_calling``，以及默认 50 步的最大执行步数。\n- 默认为每个样本提供临时本地目录中的 Bash 环境；Docker 沙箱和 MCP 服务器为可选项。\n- 单次完整运行即可生成 ``all``、``en`` 和 ``zh`` 三份报告，无需重复推理。\n\n## 评估说明\n\n- 采用官方的 Markdown 表格对齐方式及混合规则/LLM 评分语义。\n- 必须设置 ``judge_strategy='auto'`` 或 ``'llm'`` 并显式指定 ``judge_model_args``；不支持仅使用规则评分。\n- 运行示例及论文风格的重复实验设置，请参阅 [WideSearch 使用指南](https://evalscope.readthedocs.io/zh-cn/latest/third_party/wide_search.html)。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `wide_search` |\n| **数据集ID** | [bytedance-community/WideSearch](https://modelscope.cn/datasets/bytedance-community/WideSearch/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2508.07999) |\n| **标签** | `Agent`, `MultiTurn`, `Retrieval` |\n| **指标** | `success_rate`, `row_precision`, `row_recall`, `row_f1`, `item_precision`, `item_recall`, `item_f1` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `full` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 200 |\n| 提示词长度（平均） | 631.01 字符 |\n| 提示词长度（最小/最大） | 182 / 1807 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"59f736f6\",\n \"content\": \"My son is about to start his university applications in 2025 for postgraduates but he’s still uncertain about both his major and which universities to apply to. Could you help me find the top five universities in each of the five broad subjec ... [TRUNCATED 691 chars] ... names in English. \\nUse only Arabic numerals in the ranking, for example: 1.\\nDon't ask me any questions, just output the results according to the columns without omitting cells arbitrarily. The output format is \\n```markdown\\n{data_content}\\n```.\"\n }\n ],\n \"target\": \"Subject,University,Country,QS World University Rankings by Subject 2025,QS World University Rankings 2025,Times Higher Education World University Rankings 2025,Home Page,Application Deadline,Application Fee\\nArts & Humanities,Harvard Universi ... [TRUNCATED 2837 chars] ... nuary 5,$90\\nSocial Sciences & Management,Massachusetts Institute of Technology,United States,4,1,2,https://www.mit.edu/,January 6 ,$75\\nSocial Sciences & Management,University of Cambridge,United Kingdom,5,5,5,https://www.cam.ac.uk/,Oct 15,£60\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"instance_id\": \"ws_en_001\",\n \"language\": \"en\",\n \"evaluation\": {\n \"unique_columns\": [\n \"subject\",\n \"university\"\n ],\n \"required\": [\n \"subject\",\n \"university\",\n \"qsworlduniversityrankingsbysubject2025\",\n \"qsworlduniversityrankings2025\",\n \"timeshighereducationworlduniversityrankings2025\",\n \"homepage\",\n \"applicationdeadline\",\n \"applicationfee\"\n ],\n \"eval_pipeline\": {\n \"applicationdeadline\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"llm_judge\"\n ],\n \"criterion\": \"It is sufficient if the semantics are approximately the same as the reference answer or if they point to the same entity. There is no need for a word-for-word correspondence.\\nThe month and day must be correct\"\n },\n \"applicationfee\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"llm_judge\"\n ],\n \"criterion\": \"It is sufficient if the semantics are approximately the same as the reference answer or if they point to the same entity. There is no need for a word-for-word correspondence.\\nIf there are multiple fees in the reference answer, all must be included.\"\n },\n \"homepage\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"url_match\"\n ]\n },\n \"subject\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"university\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"qsworlduniversityrankingsbysubject2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"qsworlduniversityrankings2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"timeshighereducationworlduniversityrankings2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n }\n }\n }\n }\n}\n```\n\n*注：部分内容因展示需要已被截断。*\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets wide_search \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":50}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['wide_search'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=50,\n ),\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", - "content_hash": "8c8350ecbd9216b962060b62c92b9895", + "en": "# WideSearch\n\n\n## Overview\n\nWideSearch evaluates search agents on broad web information-seeking tasks. Each task asks the agent to collect many\natomic facts and return one structured Markdown table. EvalScope uses the ModelScope\n`bytedance-community/WideSearch` dataset.\n\n## Task Description\n\n- **Task Type**: Multi-turn search agent\n- **Input**: Natural-language collection request with an explicit table schema\n- **Output**: Complete Markdown table\n- **Dataset**: 200 tasks in the ``full`` split; 100 English and 100 Chinese\n\n## Key Features\n\n- Official single-agent protocol: language-specific system prompt, ``function_calling``, and 50 default steps.\n- Bash is available by default in a per-sample temporary local directory; Docker sandbox and MCP servers are optional.\n- A single full run derives ``all``, ``en``, and ``zh`` reports without repeated inference.\n\n## Evaluation Notes\n\n- Uses the official Markdown table alignment and hybrid rule/LLM scoring semantics.\n- Requires ``judge.strategy='auto'`` or ``'llm'`` with at least one ``judge.models`` entry; rule-only scoring is unsupported.\n- See the [WideSearch usage guide](https://evalscope.readthedocs.io/en/latest/third_party/wide_search.html) for runtime\n examples and paper-style repeat settings.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `wide_search` |\n| **Dataset ID** | [bytedance-community/WideSearch](https://modelscope.cn/datasets/bytedance-community/WideSearch/summary) |\n| **Paper** | [Paper](https://arxiv.org/abs/2508.07999) |\n| **Tags** | `Agent`, `MultiTurn`, `Retrieval` |\n| **Metrics** | `success_rate`, `row_precision`, `row_recall`, `row_f1`, `item_precision`, `item_recall`, `item_f1` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `full` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 200 |\n| Prompt Length (Mean) | 631.01 chars |\n| Prompt Length (Min/Max) | 182 / 1807 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"59f736f6\",\n \"content\": \"My son is about to start his university applications in 2025 for postgraduates but he’s still uncertain about both his major and which universities to apply to. Could you help me find the top five universities in each of the five broad subjec ... [TRUNCATED 691 chars] ... names in English. \\nUse only Arabic numerals in the ranking, for example: 1.\\nDon't ask me any questions, just output the results according to the columns without omitting cells arbitrarily. The output format is \\n```markdown\\n{data_content}\\n```.\"\n }\n ],\n \"target\": \"Subject,University,Country,QS World University Rankings by Subject 2025,QS World University Rankings 2025,Times Higher Education World University Rankings 2025,Home Page,Application Deadline,Application Fee\\nArts & Humanities,Harvard Universi ... [TRUNCATED 2837 chars] ... nuary 5,$90\\nSocial Sciences & Management,Massachusetts Institute of Technology,United States,4,1,2,https://www.mit.edu/,January 6 ,$75\\nSocial Sciences & Management,University of Cambridge,United Kingdom,5,5,5,https://www.cam.ac.uk/,Oct 15,£60\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"instance_id\": \"ws_en_001\",\n \"language\": \"en\",\n \"evaluation\": {\n \"unique_columns\": [\n \"subject\",\n \"university\"\n ],\n \"required\": [\n \"subject\",\n \"university\",\n \"qsworlduniversityrankingsbysubject2025\",\n \"qsworlduniversityrankings2025\",\n \"timeshighereducationworlduniversityrankings2025\",\n \"homepage\",\n \"applicationdeadline\",\n \"applicationfee\"\n ],\n \"eval_pipeline\": {\n \"applicationdeadline\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"llm_judge\"\n ],\n \"criterion\": \"It is sufficient if the semantics are approximately the same as the reference answer or if they point to the same entity. There is no need for a word-for-word correspondence.\\nThe month and day must be correct\"\n },\n \"applicationfee\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"llm_judge\"\n ],\n \"criterion\": \"It is sufficient if the semantics are approximately the same as the reference answer or if they point to the same entity. There is no need for a word-for-word correspondence.\\nIf there are multiple fees in the reference answer, all must be included.\"\n },\n \"homepage\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"url_match\"\n ]\n },\n \"subject\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"university\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"qsworlduniversityrankingsbysubject2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"qsworlduniversityrankings2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"timeshighereducationworlduniversityrankings2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n }\n }\n }\n }\n}\n```\n\n*Note: Some content was truncated for display.*\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets wide_search \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":50}' \\\n --limit 10 # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['wide_search'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=50,\n ),\n limit=10, # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n", + "zh": "# WideSearch\n\n\n## 概述\n\nWideSearch 用于评估搜索智能体在广泛网络信息检索任务上的表现。每个任务要求智能体收集多个原子事实，并返回一个结构化的 Markdown 表格。EvalScope 使用 ModelScope 上的 `bytedance-community/WideSearch` 数据集。\n\n## 任务描述\n\n- **任务类型**：多轮搜索智能体\n- **输入**：包含显式表格结构的自然语言信息收集请求\n- **输出**：完整的 Markdown 表格\n- **数据集**：``full`` 划分中共有 200 个任务；其中 100 个为英文，100 个为中文\n\n## 核心特性\n\n- 官方单智能体协议：语言特定的系统提示词、``function_calling`` 机制，以及默认 50 步的最大执行步数。\n- 默认为每个样本提供一个临时本地目录并启用 Bash；Docker 沙箱和 MCP 服务器为可选项。\n- 单次完整运行即可生成 ``all``、``en`` 和 ``zh`` 三份报告，无需重复推理。\n\n## 评估说明\n\n- 使用官方的 Markdown 表格对齐方式及混合规则/大语言模型（LLM）评分语义。\n- 要求设置 ``judge.strategy='auto'`` 或 ``'llm'``，且至少指定一个 ``judge.models`` 条目；不支持仅使用规则评分。\n- 运行时示例及论文风格的重复实验设置，请参阅 [WideSearch 使用指南](https://evalscope.readthedocs.io/zh-cn/latest/third_party/wide_search.html)。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `wide_search` |\n| **数据集ID** | [bytedance-community/WideSearch](https://modelscope.cn/datasets/bytedance-community/WideSearch/summary) |\n| **论文** | [Paper](https://arxiv.org/abs/2508.07999) |\n| **标签** | `Agent`, `MultiTurn`, `Retrieval` |\n| **指标** | `success_rate`, `row_precision`, `row_recall`, `row_f1`, `item_precision`, `item_recall`, `item_f1` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `full` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 200 |\n| 提示词长度（平均） | 631.01 字符 |\n| 提示词长度（最小/最大） | 182 / 1807 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n \"input\": [\n {\n \"id\": \"59f736f6\",\n \"content\": \"My son is about to start his university applications in 2025 for postgraduates but he’s still uncertain about both his major and which universities to apply to. Could you help me find the top five universities in each of the five broad subjec ... [TRUNCATED 691 chars] ... names in English. \\nUse only Arabic numerals in the ranking, for example: 1.\\nDon't ask me any questions, just output the results according to the columns without omitting cells arbitrarily. The output format is \\n```markdown\\n{data_content}\\n```.\"\n }\n ],\n \"target\": \"Subject,University,Country,QS World University Rankings by Subject 2025,QS World University Rankings 2025,Times Higher Education World University Rankings 2025,Home Page,Application Deadline,Application Fee\\nArts & Humanities,Harvard Universi ... [TRUNCATED 2837 chars] ... nuary 5,$90\\nSocial Sciences & Management,Massachusetts Institute of Technology,United States,4,1,2,https://www.mit.edu/,January 6 ,$75\\nSocial Sciences & Management,University of Cambridge,United Kingdom,5,5,5,https://www.cam.ac.uk/,Oct 15,£60\",\n \"id\": 0,\n \"group_id\": 0,\n \"tools\": [\n {\n \"name\": \"bash\",\n \"description\": \"Execute a bash command inside the sandbox environment. Returns the combined stdout / stderr output of the command.\",\n \"parameters\": {\n \"properties\": {\n \"command\": {\n \"type\": \"string\",\n \"description\": \"The bash command to execute.\"\n },\n \"timeout\": {\n \"type\": \"number\",\n \"description\": \"Maximum execution time in seconds (default: 60).\",\n \"default\": 60\n }\n },\n \"required\": [\n \"command\"\n ]\n }\n }\n ],\n \"metadata\": {\n \"instance_id\": \"ws_en_001\",\n \"language\": \"en\",\n \"evaluation\": {\n \"unique_columns\": [\n \"subject\",\n \"university\"\n ],\n \"required\": [\n \"subject\",\n \"university\",\n \"qsworlduniversityrankingsbysubject2025\",\n \"qsworlduniversityrankings2025\",\n \"timeshighereducationworlduniversityrankings2025\",\n \"homepage\",\n \"applicationdeadline\",\n \"applicationfee\"\n ],\n \"eval_pipeline\": {\n \"applicationdeadline\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"llm_judge\"\n ],\n \"criterion\": \"It is sufficient if the semantics are approximately the same as the reference answer or if they point to the same entity. There is no need for a word-for-word correspondence.\\nThe month and day must be correct\"\n },\n \"applicationfee\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"llm_judge\"\n ],\n \"criterion\": \"It is sufficient if the semantics are approximately the same as the reference answer or if they point to the same entity. There is no need for a word-for-word correspondence.\\nIf there are multiple fees in the reference answer, all must be included.\"\n },\n \"homepage\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"url_match\"\n ]\n },\n \"subject\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"university\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"qsworlduniversityrankingsbysubject2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"qsworlduniversityrankings2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n },\n \"timeshighereducationworlduniversityrankings2025\": {\n \"preprocess\": [\n \"norm_str\"\n ],\n \"metric\": [\n \"exact_match\"\n ]\n }\n }\n }\n }\n}\n```\n\n*注：部分内容因展示需要已被截断。*\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n --model YOUR_MODEL \\\n --api-url OPENAI_API_COMPAT_URL \\\n --api-key EMPTY_TOKEN \\\n --datasets wide_search \\\n --agent-config '{\"mode\":\"native\",\"strategy\":\"function_calling\",\"max_steps\":50}' \\\n --limit 10 # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import TaskConfig, run_task\nfrom evalscope.api.agent import NativeAgentConfig\n\ntask_cfg = TaskConfig(\n model='YOUR_MODEL',\n api_url='OPENAI_API_COMPAT_URL',\n api_key='EMPTY_TOKEN',\n datasets=['wide_search'],\n agent_config=NativeAgentConfig(\n strategy='function_calling',\n max_steps=50,\n ),\n limit=10, # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```", + "content_hash": "0d89fb2f4e0b14234fa84358e3abc97a", "needs_translation": false }, - "updated_at": "2026-08-11T15:30:53.624174", - "translation_updated_at": "2026-07-13T10:25:06" + "updated_at": "2026-08-20T10:47:12.857971", + "translation_updated_at": "2026-08-20T10:47:18" } diff --git a/evalscope/benchmarks/aa_lcr/aa_lcr_adapter.py b/evalscope/benchmarks/aa_lcr/aa_lcr_adapter.py index e932d43da..37bf900f3 100644 --- a/evalscope/benchmarks/aa_lcr/aa_lcr_adapter.py +++ b/evalscope/benchmarks/aa_lcr/aa_lcr_adapter.py @@ -1,22 +1,39 @@ # Copyright (c) Alibaba, Inc. and its affiliates. # flake8: noqa: E501 -import re import zipfile from pathlib import Path -from typing import Any, Dict +from pydantic import BaseModel +from typing import Any, Dict, List, Literal from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import Sample -from evalscope.api.evaluator import TaskState +from evalscope.api.judge import ( + CaseVerdict, + JudgeCase, + JudgeContext, + JudgeDefinition, + JudgeRequest, + OutputContract, + ReducedVerdict, +) from evalscope.api.messages import ChatMessageUser from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import DEFAULT_EVALSCOPE_CACHE_DIR, Tags +from evalscope.constants import DEFAULT_EVALSCOPE_CACHE_DIR, ScoringPolicy, Tags from evalscope.utils.download_utils import download_url from evalscope.utils.logger import get_logger logger = get_logger() + +# The judge prompt ends with "Reply only with CORRECT or INCORRECT." +class Grade(BaseModel): + reasoning: str = '' + verdict: Literal['CORRECT', 'INCORRECT'] + + +GRADE_CONTRACT = OutputContract(schema_model=Grade) + # Default judge prompt template JUDGE_PROMPT = """Assess whether the following CANDIDATE ANSWER is CORRECT or INCORRECT. For the CANDIDATE ANSWER to be correct, it must be consistent with the OFFICIAL ANSWER. @@ -101,7 +118,7 @@ ) class AALCRAdapter(DefaultDataAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -202,36 +219,22 @@ def record_to_sample(self, record: Dict[str, Any]) -> Sample: } ) - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - ) + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: - judge_prompt = JUDGE_PROMPT.format( - question=task_state.metadata['question'], correct_answer=reference, response=filtered_prediction - ) + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + prompt = JUDGE_PROMPT.format( + question=judge_context.task_state.metadata['question'], + correct_answer=judge_context.reference, + response=judge_context.filtered_prediction, + ) + case.output_contract.instruction() + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) - # Request judge and obtain score - judge_response = self.llm_judge.judge(prompt=judge_prompt) + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + return ReducedVerdict(value={'acc': 1.0 if case_verdicts[0].value.verdict == 'CORRECT' else 0.0}) - # Parse judge response to get accuracy score - # Use word boundaries to avoid matching "CORRECT" within "INCORRECT" - is_correct = bool(re.search(r'\bCORRECT\b', judge_response, re.IGNORECASE)) - score.value = { - 'acc': 1.0 if is_correct else 0.0, - } - score.explanation = f'LLM judge: {judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id, - } - score.main_score_name = 'acc' - return score + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='grade', output_contract=GRADE_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='acc' + ) diff --git a/evalscope/benchmarks/aime/aime_adapter.py b/evalscope/benchmarks/aime/aime_adapter.py index 8ef0eb13a..1bc7d73f7 100644 --- a/evalscope/benchmarks/aime/aime_adapter.py +++ b/evalscope/benchmarks/aime/aime_adapter.py @@ -1,21 +1,38 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -from typing import Any, Dict, Set +from pydantic import BaseModel +from typing import Any, Dict, List, Literal from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState +from evalscope.api.judge import ( + CaseVerdict, + JudgeCase, + JudgeContext, + JudgeDefinition, + JudgeRequest, + OutputContract, + ReducedVerdict, +) +from evalscope.api.messages import ChatMessageUser from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger # flake8: noqa logger = get_logger() -# Punctuation and markdown a judge may wrap its bare "Yes"/"No" verdict in. -VERDICT_WRAPPER_CHARS = ' \t*_`"\'.:' + +# The judge answers with a bare Yes/No on a line of its own; a judge that explains itself +# ("Yes, the answer is incorrect") must not set the verdict. +class Equivalence(BaseModel): + verdict: Literal['Yes', 'No'] + + +EQUIVALENCE_CONTRACT = OutputContract(schema_model=Equivalence) JUDGE_PROMPT = """ Look at the following two expressions (answers to a math problem) and judge whether they are equivalent. Only perform trivial simplifications @@ -170,47 +187,24 @@ def match_score( score.metadata['acc'] = f'grading_error: {str(e)}' return score - def llm_match_score( - self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState - ) -> Score: - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - ) + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: - judge_prompt = JUDGE_PROMPT.format(expression1=original_prediction, expression2=reference) - - judge_response = self.llm_judge.judge(prompt=judge_prompt) - - judge_verdicts = self._parse_judge_verdicts(judge_response) - is_correct = judge_verdicts == {'yes'} - score.value = { - 'acc': 1.0 if is_correct else 0.0, - } - score.explanation = f'LLM judge: {judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id, - } - if len(judge_verdicts) != 1: - logger.warning(f'AIME: failed to parse LLM judge response: {judge_response!r}') - score.metadata['parse_failed'] = True - score.main_score_name = 'acc' - return score + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + prompt = JUDGE_PROMPT.format( + expression1=judge_context.original_prediction, + expression2=judge_context.reference, + ) + case.output_contract.instruction() + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + return ReducedVerdict(value={'acc': 1.0 if case_verdicts[0].value.verdict == 'Yes' else 0.0}) - @staticmethod - def _parse_judge_verdicts(judge_response: str) -> Set[str]: - """Collect the bare Yes/No verdicts the judge stated on lines of their own. - - Matching anywhere in the response instead lets a judge that explains itself - ("Yes, the answer is incorrect") set the opposite verdict. - """ - return { - verdict - for line in judge_response.splitlines() - if (verdict := line.strip(VERDICT_WRAPPER_CHARS).casefold()) in ('yes', 'no') - } + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='equivalence', output_contract=EQUIVALENCE_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='acc' + ) @register_benchmark( diff --git a/evalscope/benchmarks/air_bench/air_bench_chat_adapter.py b/evalscope/benchmarks/air_bench/air_bench_chat_adapter.py index c17fe53ef..da45290bb 100644 --- a/evalscope/benchmarks/air_bench/air_bench_chat_adapter.py +++ b/evalscope/benchmarks/air_bench/air_bench_chat_adapter.py @@ -14,15 +14,25 @@ import os import re +from pydantic import BaseModel, Field from typing import Any, Dict, List, Optional, Tuple from evalscope.api.benchmark import AudioLanguageAdapter, BenchmarkMeta from evalscope.api.dataset import DatasetDict, Sample from evalscope.api.evaluator import TaskState -from evalscope.api.messages import ChatMessageUser, ContentAudio, ContentText +from evalscope.api.judge import ( + JudgeCase, + JudgeContext, + JudgeDefinition, + JudgeRequest, + OutputContract, + Placement, + ReducedVerdict, +) +from evalscope.api.messages import ChatMessageSystem, ChatMessageUser, ContentAudio, ContentText from evalscope.api.metric.scorer import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger from .utils import ( CHAT_TASK_TO_CATEGORY, @@ -37,6 +47,15 @@ logger = get_logger() + +class PairRating(BaseModel): + """Both assistants' 1-10 ratings from one judge pass.""" + assistant1: float = Field(ge=1.0, le=10.0) + assistant2: float = Field(ge=1.0, le=10.0) + + +PAIR_CONTRACT = OutputContract(schema_model=PairRating) + JUDGE_SYSTEM_PROMPT = ('You are a helpful and precise assistant for checking the quality of the answer.') JUDGE_TEMPLATE = """[Detailed Audio Description] @@ -51,7 +70,7 @@ [The End of Assistant 2s Answer] [System] We would like to request your feedback on the performance of two AI assistants in response to the user question and audio description displayed above. AI assistants are provided with detailed audio descriptions and questions. -Please rate the helpfulness, relevance, accuracy, and comprehensiveness of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance. Please output a single line containing only two values indicating the scores for Assistant 1 and 2, respectively. The two scores are separated by a space.""" # noqa: E501 +Please rate the helpfulness, relevance, accuracy, and comprehensiveness of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance. """ # noqa: E501 @register_benchmark( @@ -71,38 +90,21 @@ - **Task Type**: Open-ended audio question answering. - **Input**: An audio clip plus a free-form question. - **Output**: A textual answer evaluated against the reference response. +- **Modalities**: Audio (human speech, natural sounds, music) + text. -## Categories (8 tasks → 5 reported categories) - -The 8 Chat tasks are aggregated by the official `cal_score.py` into five categories: +## Key Features -- `speech`: `speech_QA`, `speech_dialogue_QA` -- `sound`: `sound_QA`, `sound_generation_QA` -- `music`: `music_QA`, `music_generation_analysis_QA` -- `speech_and_sound`: `speech_and_sound_QA` -- `speech_and_music`: `speech_and_music_QA` +- ~2k open-ended audio QA pairs across speech, sound, music and mixed-audio scenes; the generative half of AIR-Bench (ACL 2024). +- 8 Chat tasks aggregated by the official `cal_score.py` into 5 reported categories: `speech` (`speech_QA`, `speech_dialogue_QA`), `sound` (`sound_QA`, `sound_generation_QA`), `music` (`music_QA`, `music_generation_analysis_QA`), `speech_and_sound` (`speech_and_sound_QA`), `speech_and_music` (`speech_and_music_QA`). The paper's Mixed-audio = mean(speech_and_sound, speech_and_music). +- Position bias is removed by judging every sample twice with reference/prediction order swapped, then averaging (disable via `extra_params={'do_swap': False}` to halve judge cost). +- Hosted on ModelScope ([`evalscope/AIR-Bench-Dataset`](https://modelscope.cn/datasets/evalscope/AIR-Bench-Dataset)) in an audiofolder + JSON layout; the full release is ~49 GB, so limit tasks via `extra_params={'tasks': [...]}` for partial runs. -The paper's **Mixed-audio = mean(speech_and_sound, speech_and_music)**. +## Evaluation Notes -## Dataset Access - -- The dataset is hosted on ModelScope: [`evalscope/AIR-Bench-Dataset`](https://modelscope.cn/datasets/evalscope/AIR-Bench-Dataset). It uses an *audiofolder + JSON metadata* layout. evalscope downloads it lazily via `modelscope.dataset_snapshot_download` on first run; the full release is ~49 GB, so it is recommended to limit which tasks are pulled via `extra_params`. +- Metrics: `judge_score` is the model's mean judge score; `win_rate` records how often the model strictly beats the reference. +- The judge LLM receives the question, the textual audio description (`meta_info`), the reference answer (`answer_gt`), and the model's response, and outputs two integer scores in `[1, 10]`. Use a judge that supports long contexts, since `meta_info` may exceed 4k tokens for dialogue tasks. +- The official leaderboard uses `gpt-4-0125-preview`. If that exact snapshot is unavailable, use an available GPT-4-class judge; absolute scores can drift versus the published numbers because the judge model changed. - If the dataset is already on disk, pass `dataset_args={'air_bench_chat': {'local_path': '/path/to/AIR-Bench-Dataset'}}`; the local root should contain `Chat/`. - -## Evaluation Protocol - -- The judge LLM (default: GPT-4) receives the question, the textual audio description (`meta_info` from the dataset), the reference answer (`answer_gt`), and the model's response. It outputs a single line with two integer scores in `[1, 10]`. -- To remove position bias, every sample is judged twice with the order of reference and prediction swapped, then averaged. This mirrors `cal_score.py` in the official repository — disable it via `extra_params={'do_swap': False}` to halve judge cost. -- Reported metric `judge_score` is the model's mean judge score; `win_rate` records how often the model strictly beats the reference. - -```{warning} -The official leaderboard uses `gpt-4-0125-preview` as the judge model. If that exact snapshot is unavailable, use an available GPT-4-class judge; absolute scores can drift versus the published numbers because the judge model changed. -``` - -## Implementation Notes - -- The judge model is selected via `--judge-model-args`; ensure the model id supports long contexts (`meta_info` may exceed 4k tokens for dialogue tasks). -- Set `extra_params={'tasks': [...]}` to evaluate only specific Chat task names — useful for partial runs. """, # noqa: E501 subset_list=list(CHAT_TASK_TO_CATEGORY.keys()), eval_split='test', @@ -130,7 +132,7 @@ ) class AIRBenchChatAdapter(AudioLanguageAdapter): """Adapter for AIR-Bench Chat open-ended audio QA tasks.""" - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY # Per-sample folder layout for audio files. Distinct from Foundation since # Chat pre-merges some categories. @@ -281,121 +283,68 @@ def _record_to_sample_with_root(self, record: Dict[str, Any], track_root: str) - # ------------------------------------------------------------------ # Scoring (LLM judge with optional position swap) # ------------------------------------------------------------------ - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - ) + supports_position_swap = True + + @property + def official_position_swap(self) -> bool: + # Official cal_score.py judges each sample twice with the order swapped. + return bool(self.extra_params.get('do_swap', True)) + + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + metadata = judge_context.task_state.metadata or {} + reference_first = placement is Placement.ORIGINAL + prompt = JUDGE_TEMPLATE.format( + meta_info=metadata.get('meta_info', ''), + question=metadata.get('question') or judge_context.task_state.input_text, + assistant1=judge_context.reference if reference_first else judge_context.filtered_prediction, + assistant2=judge_context.filtered_prediction if reference_first else judge_context.reference, + ) + return JudgeRequest( + messages=[ + ChatMessageSystem(content=JUDGE_SYSTEM_PROMPT), + ChatMessageUser(content=prompt + case.output_contract.instruction()) + ] + ) - meta_info = task_state.metadata.get('meta_info', '') - question = task_state.metadata.get('question') or task_state.input_text - do_swap = bool(self.extra_params.get('do_swap', True)) + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + placements = case_verdicts[0].placements + position_results = {} + if placements: + original, swapped = placements['original'], placements['swapped'] + pred_scores, ref_scores = [original.assistant2, + swapped.assistant1], [original.assistant1, swapped.assistant2] + position_results = { + 'original': _compare_scores(original.assistant2, original.assistant1), + 'swapped': _compare_scores(swapped.assistant1, swapped.assistant2), + } + else: + verdict = case_verdicts[0].value + pred_scores, ref_scores = [verdict.assistant2], [verdict.assistant1] + mean_pred, mean_ref = sum(pred_scores) / len(pred_scores), sum(ref_scores) / len(ref_scores) + return ReducedVerdict( + value={ + 'judge_score': mean_pred, + 'win_rate': 1.0 if mean_pred > mean_ref else 0.0 + }, + metadata={ + 'reference_score': mean_ref, + 'pred_scores_per_pass': pred_scores, + 'reference_scores_per_pass': ref_scores + }, + position_results=position_results, + ) - # Pass 1: reference as Assistant 1, prediction as Assistant 2. - prompt_a = JUDGE_TEMPLATE.format( - meta_info=meta_info, - question=question, - assistant1=reference, - assistant2=filtered_prediction, + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='pair', output_contract=PAIR_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='judge_score' ) - ref_score_1, pred_score_1, raw_1 = self._judge_pair(prompt_a) - - scores_ref: List[float] = [] - scores_pred: List[float] = [] - if ref_score_1 is not None and pred_score_1 is not None: - scores_ref.append(ref_score_1) - scores_pred.append(pred_score_1) - - raw_responses = [raw_1] - parse_failed = ref_score_1 is None or pred_score_1 is None - - if do_swap: - # Pass 2: prediction as Assistant 1, reference as Assistant 2. - prompt_b = JUDGE_TEMPLATE.format( - meta_info=meta_info, - question=question, - assistant1=filtered_prediction, - assistant2=reference, - ) - pred_score_2, ref_score_2, raw_2 = self._judge_pair(prompt_b) - raw_responses.append(raw_2) - if pred_score_2 is not None and ref_score_2 is not None: - scores_ref.append(ref_score_2) - scores_pred.append(pred_score_2) - else: - parse_failed = True - - if scores_pred and scores_ref: - mean_pred = sum(scores_pred) / len(scores_pred) - mean_ref = sum(scores_ref) / len(scores_ref) - win = 1.0 if mean_pred > mean_ref else 0.0 - score.value = { - 'judge_score': mean_pred, - 'win_rate': win, - } - score.main_score_name = 'judge_score' - score.metadata = { - 'reference_score': mean_ref, - 'pred_scores_per_pass': scores_pred, - 'reference_scores_per_pass': scores_ref, - 'judge_strategy': self.judge_strategy, - 'judge_model': getattr(self.llm_judge, 'model_id', 'unknown'), - 'do_swap': do_swap, - } - if parse_failed: - logger.warning(f'AIR-Bench Chat: failed to parse judge response(s): {raw_responses!r}') - score.metadata.update({'parse_failed': True, 'judge_raw': raw_responses}) - score.explanation = ' || '.join(raw_responses) - else: - logger.warning(f'AIR-Bench Chat: failed to parse judge response(s): {raw_responses!r}') - # Omitting the metric keys drops the sample from the mean; a 0.0 would be averaged - # in as a real rating even though the judge scale starts at 1. - score.value = {} - score.metadata = {'parse_failed': True, 'judge_raw': raw_responses} - return score - - def _judge_pair(self, prompt: str) -> Tuple[Optional[float], Optional[float], str]: - """Call the judge once and parse its two-integer response. - - Returns ``(score_assistant1, score_assistant2, raw_response)``. The two - score positions correspond to the order embedded in ``prompt``. - """ - try: - judge = self.llm_judge - if judge is None: - return None, None, '' - raw = judge.judge(prompt, system_prompt=JUDGE_SYSTEM_PROMPT) - except Exception as e: - logger.warning(f'AIR-Bench Chat: judge call failed: {e}') - return None, None, f'' - - if raw is None: - return None, None, '' - - nums = self._extract_judge_scores(raw) - if len(nums) >= 2: - try: - a, b = float(nums[0]), float(nums[1]) - if 1 <= a <= 10 and 1 <= b <= 10: - return float(a), float(b), raw - except ValueError: - pass - return None, None, raw - - @staticmethod - def _extract_judge_scores(raw: str) -> List[str]: - # Anchor to the first non-empty line, the format the prompt asks for. Reading further - # lets prose numbers, or numbers quoted back from the candidate answer, become the scores. - score_pattern = r'(?:10(?:\.0+)?|[0-9](?:\.\d+)?)' - for line in raw.splitlines(): - if not line.strip(): - continue - match = re.fullmatch(rf'({score_pattern})[\s,]+({score_pattern})', line.strip()) - return list(match.groups()) if match else [] - return [] + + +def _compare_scores(candidate: float, reference: float) -> str: + if candidate == reference: + return 'tie' + return 'win' if candidate > reference else 'loss' diff --git a/evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py b/evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py index caa769931..cf8911094 100644 --- a/evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py +++ b/evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py @@ -1,16 +1,37 @@ -import re -from typing import Any, Dict +from pydantic import BaseModel, Field +from typing import Any, Dict, List, Literal from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import Sample -from evalscope.api.evaluator import TaskState +from evalscope.api.judge import ( + CaseVerdict, + JudgeCase, + JudgeContext, + JudgeDefinition, + JudgeRequest, + OutputContract, + PairwiseOutcome, + PairwisePlacementOutcome, + Placement, + ReducedVerdict, +) +from evalscope.api.messages import ChatMessageSystem, ChatMessageUser from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger logger = get_logger() + +# The judge replies with the single letter of the better output; 'm' is the baseline and 'M' +# the evaluated model. Case matters, so an unanchored search would match any m in prose. +class Preference(BaseModel): + verdict: Literal['m', 'M'] + + +PREFERENCE_CONTRACT = OutputContract(schema_model=Preference) + GRADER_SYSTEM_PROMPT = """You are a highly efficient assistant, who evaluates and selects the best large language model (LLMs) based on the quality of their responses to a given instruction. This process will be used to create a leaderboard reflecting the most accurate and human-preferred answers.""" # noqa: E501 GRADER_TEMPLATE = """ @@ -89,7 +110,9 @@ ) class AlpacaEvalAdapter(DefaultDataAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY + supports_position_swap = True + official_position_swap = False def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -116,42 +139,38 @@ def record_to_sample(self, record: Dict[str, Any]) -> Sample: } ) - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + candidate_first = placement is Placement.SWAPPED + prompt = GRADER_TEMPLATE.format( + instruction=judge_context.task_state.input_text, + output_1=judge_context.filtered_prediction if candidate_first else judge_context.reference, + output_2=judge_context.reference if candidate_first else judge_context.filtered_prediction, + ) + return JudgeRequest( + messages=[ + ChatMessageSystem(content=GRADER_SYSTEM_PROMPT), + ChatMessageUser(content=prompt + case.output_contract.instruction()) + ] + ) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + values = case_verdicts[0].placements or {'original': case_verdicts[0].value} + outcomes = { + name: PairwisePlacementOutcome( + result='win' if verdict.verdict == ('m' if name == 'swapped' else 'M') else 'loss' + ) + for name, verdict in values.items() + } + result = next(iter(item.result for item in outcomes.values())) \ + if len({item.result for item in outcomes.values()}) == 1 else 'tie' + outcome = PairwiseOutcome(metric_name='win_rate', result=result, placements=outcomes) + return ReducedVerdict(value={'win_rate': outcome.score}, outcome=outcome) + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='preference', output_contract=PREFERENCE_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='win_rate', ) - - instruction = task_state.input_text - - # Request judge and obtain score - # reference is baseline answer 'm', filtered_prediction is model answer 'M' - prompt = GRADER_TEMPLATE.format(instruction=instruction, output_1=reference, output_2=filtered_prediction) - judge_response = self.llm_judge.judge(prompt, system_prompt=GRADER_SYSTEM_PROMPT) - - # parse grading response - match = re.search(r'(m|M)', judge_response) - res = match.group(0) if match else None - - if res: - winrate = 1 if res == 'M' else 0 - else: - logger.info(f'Failed to parse grading response: {prompt=}\n {judge_response=}') - winrate = 0 - - # Set score based on the match result - score.value = {'win_rate': winrate} - score.explanation = f'LLM judge: {judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id - } - score.main_score_name = 'win_rate' - return score diff --git a/evalscope/benchmarks/arena_hard/arena_hard_adapter.py b/evalscope/benchmarks/arena_hard/arena_hard_adapter.py index 28ce97792..7cb023bd1 100644 --- a/evalscope/benchmarks/arena_hard/arena_hard_adapter.py +++ b/evalscope/benchmarks/arena_hard/arena_hard_adapter.py @@ -1,18 +1,40 @@ # flake8: noqa: E501 -from typing import Any, Dict, List +from pydantic import BaseModel +from typing import Any, Dict, List, Literal from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState +from evalscope.api.judge import ( + JudgeCase, + JudgeContext, + JudgeDefinition, + JudgeRequest, + OutputContract, + PairwiseOutcome, + PairwisePlacementOutcome, + Placement, + ReducedVerdict, +) +from evalscope.api.messages import ChatMessageSystem, ChatMessageUser from evalscope.api.metric import AggScore, SampleScore, Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.import_utils import check_import from evalscope.utils.logger import get_logger logger = get_logger() -GRADER_SYSTEM_PROMPT = """Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better.\n\nBegin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.\n\nWhen evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.\n\nThen consider if the assistant's answers are helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.\n\nThen consider the creativity and novelty of the assistant's answers when needed. Finally, identify any missing important information in the assistants' answers that would be beneficial to include when responding to the user prompt.\n\nAfter providing your explanation, you must output only one of the following choices as your final verdict with a label:\n\n1. Assistant A is significantly better: [[A>>B]]\n2. Assistant A is slightly better: [[A>B]]\n3. Tie, relatively the same: [[A=B]]\n4. Assistant B is slightly better: [[B>A]]\n5. Assistant B is significantly better: [[B>>A]]\n\nExample output: \"My final verdict is tie: [[A=B]]\".""" # noqa: E501 + +class BattleVerdict(BaseModel): + """One game's verdict on the official five-point preference scale.""" + reasoning: str = '' + verdict: Literal['A>>B', 'A>B', 'A=B', 'B>A', 'B>>A'] + + +BATTLE_CONTRACT = OutputContract(schema_model=BattleVerdict) + +GRADER_SYSTEM_PROMPT = """Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better.\n\nBegin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.\n\nWhen evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.\n\nThen consider if the assistant's answers are helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.\n\nThen consider the creativity and novelty of the assistant's answers when needed. Finally, identify any missing important information in the assistants' answers that would be beneficial to include when responding to the user prompt.\n\nAfter providing your explanation, you must state your final verdict as one of: A>>B (Assistant A is significantly better), A>B (Assistant A is slightly better), A=B (tie), B>A (Assistant B is slightly better), or B>>A (Assistant B is significantly better).""" # noqa: E501 GRADER_TEMPLATE = """<|User Prompt|>\n{question}\n\n<|The Start of Assistant A's Answer|>\n{answer_1}\n<|The End of Assistant A's Answer|>\n\n<|The Start of Assistant B's Answer|>\n{answer_2}\n<|The End of Assistant B's Answer|>""".strip( ) @@ -62,7 +84,7 @@ ) class ArenaHardAdapter(DefaultDataAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -85,72 +107,68 @@ def record_to_sample(self, record: Dict[str, Any]) -> Sample: input=question, target=baseline_prediction, metadata={'capability': record.get('capability', 'unknown')} ) - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - from .utils import get_judge_score, post_process_arenahard - - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, + supports_position_swap = True + official_position_swap = True + """Arena-Hard plays each pair twice with the answers swapped.""" + + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + baseline_first = placement is Placement.ORIGINAL + prompt = GRADER_TEMPLATE.format( + question=judge_context.task_state.input_text, + answer_1=judge_context.reference if baseline_first else judge_context.filtered_prediction, + answer_2=judge_context.filtered_prediction if baseline_first else judge_context.reference, + ) + return JudgeRequest( + messages=[ + ChatMessageSystem(content=GRADER_SYSTEM_PROMPT), + ChatMessageUser(content=prompt + case.output_contract.instruction()) + ] + ) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + placements = case_verdicts[0].placements + res1 = placements.get('original', case_verdicts[0].value).verdict + res2 = placements.get('swapped') + outcomes = {'original': _placement_outcome(res1, candidate_is_a=False)} + if res2 is not None: + outcomes['swapped'] = _placement_outcome(res2.verdict, candidate_is_a=True) + result, strength = _reduce_placements(outcomes) + outcome = PairwiseOutcome(metric_name='score', result=result, strength=strength, placements=outcomes) + return ReducedVerdict(value={'score': outcome.score}, outcome=outcome) + + def finalize(score, review, judge_context) -> Score: + if review.outcome is not None: + score.metadata['battle_result'] = { + 'model_a': 'gpt4-0314', + 'model_b': 'test_model', + 'games': [{ + 'score': _battle_label(review.outcome.placements['original'], candidate_is_a=False) + }, *([{ + 'score': _battle_label(review.outcome.placements['swapped'], candidate_is_a=True) + }] if 'swapped' in review.outcome.placements else [])], + } + return score + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='battle', output_contract=BATTLE_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='score', + finalize=finalize, ) - question = task_state.input_text - - # reference is baseline answer 'A', filtered_prediction is model answer 'B' - prompt1 = GRADER_TEMPLATE.format(question=question, answer_1=reference, answer_2=filtered_prediction) - # reverse the order - prompt2 = GRADER_TEMPLATE.format(question=question, answer_1=filtered_prediction, answer_2=reference) - - # get grading response - game1_response = self.llm_judge.judge(prompt1, system_prompt=GRADER_SYSTEM_PROMPT) - game2_response = self.llm_judge.judge(prompt2, system_prompt=GRADER_SYSTEM_PROMPT) - - # parse grading response - res1 = post_process_arenahard(game1_response) - res2 = post_process_arenahard(game2_response) - - score1 = get_judge_score(res1, reverse=True) - score2 = get_judge_score(res2, reverse=False) - - battle_result = { - 'model_a': 'gpt4-0314', - 'model_b': 'test_model', - 'games': [ - { - 'user_prompt': prompt1, - 'judgment': game1_response, - 'score': res1 - }, - { - 'user_prompt': prompt2, - 'judgment': game2_response, - 'score': res2 - }, - ] - } - - # Set score based on the battle result - score.value = {'score': (score1 + score2) / 2} - score.explanation = f'LLM judge battles: Game1: {game1_response[:100]}... Game2: {game2_response[:100]}...' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id, - 'battle_result': battle_result - } - return score - def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]: import pandas as pd from .utils import compute_mle_elo, get_battles_from_row, get_bootstrap_result, get_win_rate_column - battles = pd.concat([get_battles_from_row(res.score.metadata['battle_result']) for res in sample_scores]) + # A sample whose judge verdicts were unusable has no battle to contribute. + scored = [res for res in sample_scores if (res.score.metadata or {}).get('battle_result')] + if not scored: + return [] + battles = pd.concat([get_battles_from_row(res.score.metadata['battle_result']) for res in scored]) bootstrap_online_elo = compute_mle_elo(battles) @@ -168,5 +186,35 @@ def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]: return [AggScore( score=score, metric_name='win_rate', - num=len(sample_scores), + num=len(scored), )] + + +def _candidate_outcome(label: str, candidate_is_a: bool) -> str: + if label == 'A=B': + return 'tie' + return 'win' if label.startswith('A') == candidate_is_a else 'loss' + + +def _placement_outcome(label: str, candidate_is_a: bool) -> PairwisePlacementOutcome: + return PairwisePlacementOutcome( + result=_candidate_outcome(label, candidate_is_a), + strength='strong' if label in ('A>>B', 'B>>A') else 'weak', + ) + + +def _reduce_placements(placements: Dict[str, PairwisePlacementOutcome]) -> tuple[str, str]: + results = [placement.result for placement in placements.values()] + result = results[0] if len(set(results)) == 1 else 'tie' + strength = 'strong' if result != 'tie' and any( + placement.result == result and placement.strength == 'strong' for placement in placements.values() + ) else 'weak' + return result, strength + + +def _battle_label(outcome: PairwisePlacementOutcome, candidate_is_a: bool) -> str: + if outcome.result == 'tie': + return 'A=B' + a_wins = (outcome.result == 'win') == candidate_is_a + marker = '>>' if outcome.strength == 'strong' else '>' + return f'A{marker}B' if a_wins else f'B{marker}A' diff --git a/evalscope/benchmarks/arena_hard/utils.py b/evalscope/benchmarks/arena_hard/utils.py index f1140f85d..a7026c56b 100644 --- a/evalscope/benchmarks/arena_hard/utils.py +++ b/evalscope/benchmarks/arena_hard/utils.py @@ -10,14 +10,6 @@ logger = get_logger() -def post_process_arenahard(completion): - result = re.findall(r'\[\[([AB<>=]+)\]\]', completion) - if result: - return result[0] - else: - return None - - def get_judge_score(result, reverse=False): """ Calculate the judge score, considering confidence weight. @@ -80,6 +72,9 @@ def get_battles_from_row(row, first_game_only=False, multiplier=3): if first_game_only: return pd.DataFrame(results) + if len(row['games']) == 1: + return pd.DataFrame(results) + # game 2 output = {'model_a': row['model_a'], 'model_b': row['model_b']} diff --git a/evalscope/benchmarks/baby_vision/baby_vision_adapter.py b/evalscope/benchmarks/baby_vision/baby_vision_adapter.py index cbb6c5bab..d6e3ebe99 100644 --- a/evalscope/benchmarks/baby_vision/baby_vision_adapter.py +++ b/evalscope/benchmarks/baby_vision/baby_vision_adapter.py @@ -5,7 +5,7 @@ from evalscope.api.dataset import Sample from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger logger = get_logger() @@ -40,7 +40,7 @@ - Primary metric: **Accuracy** via LLM-as-judge - Subsets organized by `type` field (4 categories) - LLM judge evaluates both choice and blank answer types uniformly -- Requires `judge_model_args` configuration for LLM judge +- Requires an LLM judge configured through `judge.models` """ SUBSET_LIST = [ @@ -71,7 +71,7 @@ class BabyVisionAdapter(VisionLanguageAdapter): Handles both choice and blank answer types uniformly with LLM judge scoring. """ - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/evalscope/benchmarks/browsecomp/browsecomp_adapter.py b/evalscope/benchmarks/browsecomp/browsecomp_adapter.py index 3641ee077..3e6824c93 100644 --- a/evalscope/benchmarks/browsecomp/browsecomp_adapter.py +++ b/evalscope/benchmarks/browsecomp/browsecomp_adapter.py @@ -1,17 +1,31 @@ import base64 import hashlib import re -from typing import Any, Dict +from pydantic import BaseModel, Field +from typing import Any, Dict, List, Literal from evalscope.api.benchmark import AgentAdapter, BenchmarkMeta from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict +from evalscope.api.messages import ChatMessageUser from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags BROWSECOMP_DATASET_ID = 'evalscope/browse_comp' + +# The grader template requires a "correct: yes" / "correct: no" line. +class Grade(BaseModel): + extracted_final_answer: str = '' + reasoning: str = '' + correct: Literal['yes', 'no'] + confidence: int = Field(default=100, ge=0, le=100) + + +GRADE_CONTRACT = OutputContract(schema_model=Grade) + QUERY_TEMPLATE = """ {question} @@ -67,14 +81,6 @@ def normalize_answer(answer: Any) -> str: return ' '.join(re.sub(r'[^\w\s]', ' ', answer.lower()).split()) -def parse_judge_response(response: Any) -> bool: - """Return True only when the judge explicitly reports correctness.""" - if not response or not isinstance(response, str): - return False - match = re.search(r'correct:\s*["\']?(yes|no)["\']?', response, re.IGNORECASE) - return bool(match and match.group(1).lower() == 'yes') - - @register_benchmark( BenchmarkMeta( name='browsecomp', @@ -119,7 +125,7 @@ def parse_judge_response(response: Any) -> bool: ) class BrowseCompAdapter(AgentAdapter): """Adapter for the BrowseComp browsing-agent benchmark.""" - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_DEFAULT def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) @@ -165,37 +171,29 @@ def match_score( score.metadata = {'source': 'rule_exact_match'} return score - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - ) - - metadata = task_state.metadata or {} - question = metadata.get('question') or task_state.input_text - prompt = GRADER_TEMPLATE.format( - question=question, - response=original_prediction, - correct_answer=reference, + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + task_state = judge_context.task_state + prompt = GRADER_TEMPLATE.format( + question=(task_state.metadata or {}).get('question') or task_state.input_text, + response=judge_context.original_prediction, + correct_answer=judge_context.reference, + ) + case.output_contract.instruction() + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + is_correct = case_verdicts[0].value.correct == 'yes' + return ReducedVerdict( + value={ + 'is_correct': 1.0 if is_correct else 0.0, + 'is_incorrect': 0.0 if is_correct else 1.0 + } + ) + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='grade', output_contract=GRADE_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='is_correct' ) - judge_response = self.llm_judge.judge(prompt) - is_correct = parse_judge_response(judge_response) - - score.value = { - 'is_correct': 1.0 if is_correct else 0.0, - 'is_incorrect': 0.0 if is_correct else 1.0, - } - score.explanation = f'LLM judge: {judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id, - } - score.main_score_name = 'is_correct' - return score diff --git a/evalscope/benchmarks/charxiv/charxiv_adapter.py b/evalscope/benchmarks/charxiv/charxiv_adapter.py index 62645240a..966efb6dc 100644 --- a/evalscope/benchmarks/charxiv/charxiv_adapter.py +++ b/evalscope/benchmarks/charxiv/charxiv_adapter.py @@ -1,17 +1,41 @@ # flake8: noqa: E501 +from pydantic import BaseModel, Field from typing import Any, Dict, List, Union from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText from evalscope.api.metric.scorer import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger logger = get_logger() + +class DescriptiveGrade(BaseModel): + """Official descriptive-question reply keys.""" + + extract_answer_T1: str = '' + score_T1: int = Field(ge=0, le=1) + + @property + def score(self) -> int: + return self.score_T1 + + +class ReasoningGrade(BaseModel): + """Official reasoning-question reply keys.""" + + extract_answer: str = '' + score: int = Field(ge=0, le=1) + + +DESCRIPTIVE_CONTRACT = OutputContract(schema_model=DescriptiveGrade) +REASONING_CONTRACT = OutputContract(schema_model=ReasoningGrade) + DESCRIPTION = """ ## Overview @@ -43,7 +67,7 @@ - Each chart yields 5 samples: 4 descriptive + 1 reasoning - Primary metric: **Accuracy** via LLM-as-judge - Subsets: `descriptive` and `reasoning` (also by category) -- Requires `judge_model_args` configuration for LLM judge +- Requires an LLM judge configured through `judge.models` - [Paper](https://arxiv.org/abs/2406.18521) | [GitHub](https://github.com/princeton-nlp/CharXiv) """ @@ -72,7 +96,7 @@ class CharXivAdapter(VisionLanguageAdapter): Scoring uses LLM judge for both question types. """ - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, **kwargs): super().__init__(**kwargs) @@ -155,93 +179,34 @@ def record_to_sample(self, record: Dict[str, Any]) -> Union[Sample, List[Sample] return samples - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - """Custom LLM judge scoring following official CharXiv grading protocol. - - Dispatches to question-type-specific rubrics: - - Descriptive: rubric selected by question_id (title/ocr/quant/bool/enum/trend/layout) - - Reasoning: rubric selected by reasoning_a_type (1-4) - """ - from .utils import build_descriptive_judge_prompt, build_reasoning_judge_prompt - - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - ) - - metadata = task_state.metadata or {} - question_type = metadata.get('question_type', 'reasoning') + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + descriptive = (context.task_state.metadata or {}).get('question_type', 'reasoning') == 'descriptive' + contract = DESCRIPTIVE_CONTRACT if descriptive else REASONING_CONTRACT + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + from .utils import build_descriptive_judge_prompt, build_reasoning_judge_prompt + metadata = judge_context.task_state.metadata or {} + if metadata.get('question_type', 'reasoning') == 'descriptive': + prompt = build_descriptive_judge_prompt( + q_id=metadata.get('question_id', 1), + response=judge_context.original_prediction, + ground_truth=judge_context.reference, + ) + else: + prompt = build_reasoning_judge_prompt( + reasoning_a_type=metadata.get('reasoning_a_type', 1), + question=judge_context.task_state.input_text or '', + ground_truth=judge_context.reference, + response=judge_context.original_prediction, + ) + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) - if question_type == 'descriptive': - q_id = metadata.get('question_id', 1) - prompt = build_descriptive_judge_prompt( - q_id=q_id, - response=original_prediction, - ground_truth=reference, - ) - else: - reasoning_a_type = metadata.get('reasoning_a_type', 1) - question = task_state.input_text or '' - prompt = build_reasoning_judge_prompt( - reasoning_a_type=reasoning_a_type, - question=question, - ground_truth=reference, - response=original_prediction, - ) + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + return ReducedVerdict(value={'acc': float(case_verdicts[0].value.score)}) - # Call LLM judge - judge_response = self.llm_judge.judge(prompt) - - # Parse score from judge response (expect JSON with "score" key) - judge_score = self._parse_charxiv_judge_score(judge_response) - - score.value = {'acc': judge_score} - score.explanation = f'LLM judge: {judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id, - } - return score - - @staticmethod - def _parse_charxiv_judge_score(judge_response: str) -> float: - """Parse the score from the CharXiv judge response. - - The judge is expected to return JSON with a 'score' or 'score_T1' key. - Falls back to keyword matching if JSON parsing fails. - """ - import json - import re - - if not judge_response: - return 0.0 - - # Try to extract JSON from the response - try: - # Find JSON-like content - json_match = re.search(r'\{[^{}]+\}', judge_response) - if json_match: - data = json.loads(json_match.group()) - # Single sample: look for 'score' or 'score_T1' - for key in ('score', 'score_T1'): - if key in data: - val = int(data[key]) - return float(val) if val in (0, 1) else 0.0 - except (json.JSONDecodeError, ValueError, TypeError): - pass - - # Fallback: look for score patterns - response_lower = judge_response.strip().lower() - if '"score": 1' in response_lower or '"score_t1": 1' in response_lower: - return 1.0 - if '"score": 0' in response_lower or '"score_t1": 0' in response_lower: - return 0.0 - - return 0.0 + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='grade', output_contract=contract)], + request=request, + reduce=reduce, + main_score_name='acc' + ) diff --git a/evalscope/benchmarks/chinese_simple_qa/csimple_qa_adapter.py b/evalscope/benchmarks/chinese_simple_qa/csimple_qa_adapter.py index 2e0e9be6d..8e107e766 100644 --- a/evalscope/benchmarks/chinese_simple_qa/csimple_qa_adapter.py +++ b/evalscope/benchmarks/chinese_simple_qa/csimple_qa_adapter.py @@ -1,16 +1,23 @@ -import re -from typing import Any, Dict +from pydantic import BaseModel, Field +from typing import Any, Dict, Literal from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import Sample -from evalscope.api.evaluator import TaskState -from evalscope.api.metric import Score +from evalscope.api.judge import JudgeContext, JudgeDefinition from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger logger = get_logger() + +class Grade(BaseModel): + reasoning: str = '' + verdict: Literal['A', 'B', 'C'] + + +JUDGE_SYSTEM_PROMPT = '你是一个智能助手，请根据给定问题、标准答案和模型预测的答案来评估模型的回答是否正确。' + GRADER_TEMPLATE = """ 请根据给定问题、标准答案和模型预测的答案来评估模型的回答是否正确。您的任务是将结果评定为：【正确】、【错误】或【未尝试】。 @@ -81,7 +88,6 @@ B:【错误】 C:【未尝试】 -只返回字母"A"、"B"或"C"，无须添加其他文本。 """.strip() SUBSET_LIST = ['中华文化', '人文与社会科学', '工程、技术与应用科学', '生活、艺术与文化', '社会', '自然与自然科学'] @@ -131,7 +137,7 @@ ) class ChineseSimpleQAAdapter(DefaultDataAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -159,39 +165,33 @@ def record_to_sample(self, record: Dict[str, Any]) -> Sample: return Sample(input=question, target=answer, subset_key=subset_key, metadata=metadata) - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + prompt = GRADER_TEMPLATE.format( + question=context.task_state.input_text, + target=context.reference, + predicted_answer=context.filtered_prediction, + ) + return JudgeDefinition.labels( + prompt=prompt, + system_prompt=JUDGE_SYSTEM_PROMPT, + schema_model=Grade, + case_id='grade', + scores={ + 'A': { + 'is_correct': 1.0, + 'is_incorrect': 0.0, + 'is_not_attempted': 0.0 + }, + 'B': { + 'is_correct': 0.0, + 'is_incorrect': 1.0, + 'is_not_attempted': 0.0 + }, + 'C': { + 'is_correct': 0.0, + 'is_incorrect': 0.0, + 'is_not_attempted': 1.0 + }, + }, + main_score_name='is_correct', ) - - question = task_state.input_text - - # Request judge and obtain score - prompt = GRADER_TEMPLATE.format(question=question, target=reference, predicted_answer=filtered_prediction) - system_prompt = '你是一个智能助手，请根据给定问题、标准答案和模型预测的答案来评估模型的回答是否正确。' - judge_response = self.llm_judge.judge(prompt, system_prompt=system_prompt) - # parse grading response - match = re.search(r'(A|B|C)', judge_response) - res = match.group(0) if match else 'C' - - # Set score based on the match result - score.value = { - 'is_correct': 1 if res == 'A' else 0, - 'is_incorrect': 1 if res == 'B' else 0, - 'is_not_attempted': 1 if res == 'C' else 0, - } - score.explanation = f'LLM judge: {judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id - } - score.main_score_name = 'is_correct' - return score diff --git a/evalscope/benchmarks/cl_bench/cl_bench_adapter.py b/evalscope/benchmarks/cl_bench/cl_bench_adapter.py index 7ab1bf493..2e8451f3b 100644 --- a/evalscope/benchmarks/cl_bench/cl_bench_adapter.py +++ b/evalscope/benchmarks/cl_bench/cl_bench_adapter.py @@ -1,19 +1,64 @@ +from pydantic import BaseModel, Field, field_validator + # align with official CL-bench eval.py(https://github.com/Tencent-Hunyuan/CL-bench/blob/main/eval.py) -import json -from typing import Any, Dict, List +from typing import Any, Dict, List, Literal from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict from evalscope.api.messages import dict_to_chat_message from evalscope.api.messages.chat_message import ChatMessageUser from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger logger = get_logger() + +class CLGrade(BaseModel): + grading_rationale: str = Field(alias='Grading Rationale', default='') + requirement_status: List[str] = Field(alias='List of Requirement Satisfaction Status', default_factory=list) + overall_score: Literal[0, 1] = Field(alias='Overall Score') + + @field_validator('overall_score', mode='before') + @classmethod + def _coerce_score(cls, value: Any) -> Any: + # The contract instruction renders the allowed labels quoted (`"0"` / `"1"`), so a + # compliant judge replies with a string; accept it instead of wasting a parse retry. + if isinstance(value, str) and value.strip() in ('0', '1'): + return int(value.strip()) + return value + + +GRADE_CONTRACT = OutputContract(schema_model=CLGrade) + +GRADING_TEMPLATE = ( + "Starting now, you are a rigorous instruction-following grading teacher. Your task is to accurately grade and score student answers based on the 【Rubrics】.\n\n" + "Grading Criteria\n" + "This is a strict, all-or-nothing grading system. The final score is binary.\n" + "To receive a score of 1, the student's answer must perfectly satisfy every single requirement listed in the 【Rubrics】.\n" + "If even one requirement is not fully met, the final score will be 0.\n" + "Grading Process\n" + "Please strictly follow the steps below for analysis—no steps may be skipped:\n" + "Step 1: Analyze the Standard Answer\n" + "List all explicit requirements in the 【Rubrics】 item by item (including format, content, quantity, order, etc.).\n" + "Identify implicit requirements in the 【Rubrics】 (e.g., language style, logical structure).\n" + "Define specific evaluation criteria for each requirement (e.g., \"must include X,\" \"must not exceed Y\").\n" + "Step 2: Check Each Requirement Against the Student's Answer\n" + "For every requirement in the 【Rubrics】, verify one by one whether the student's answer fully satisfies it.\n" + "Step 3: Self-Reflection\n" + "Before giving the final score, you must conduct the following checks:\n" + " Completeness Check: Whether all requirements in the standard answer have been reviewed with no omissions.\n" + " Strictness Check: Whether the evaluation strictly adheres to the \"fully satisfied\" standard without relaxing requirements due to subjective judgment.\n" + " Consistency Check: Whether the grading rationale aligns logically with the final score.\n" + " Objectivity Check: Whether judgments are based on objective facts rather than subjective speculation.\n\n" + "Content to Be Graded\n" + "【Rubrics】:\n{rubrics}\n" + "【Student Response】:\n{response}\n" +) + CL_BENCH_DESCRIPTION = """ ## Overview @@ -61,7 +106,7 @@ ) class CLBenchAdapter(DefaultDataAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, **kwargs): super().__init__(**kwargs) @@ -93,100 +138,46 @@ def record_to_sample(self, record: Dict[str, Any]) -> Sample: metadata = record.get('metadata') or record.get('meta_data') or {} return Sample(input=messages, target=rubrics, metadata=metadata) - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: List[str], - task_state: TaskState, - ) -> Score: - from .utils import build_rubrics_text, extract_json_block - - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - ) - - if not filtered_prediction or not filtered_prediction.strip(): - score.value = {'acc': 0.0} - score.explanation = 'Empty model output; scored as 0.' - score.main_score_name = 'acc' + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + if not context.filtered_prediction.strip(): + return JudgeDefinition.skip( + Score( + extracted_prediction=context.filtered_prediction, + prediction=context.original_prediction, + value={'acc': 0.0}, + main_score_name='acc', + explanation='Empty model output; scored as 0.' + ), + reason='empty_model_output', + ) + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + from .utils import build_rubrics_text + target = judge_context.task_state.target + prompt = GRADING_TEMPLATE.format( + rubrics=build_rubrics_text(target if isinstance(target, list) else [target]), + response=judge_context.filtered_prediction, + ) + case.output_contract.instruction() + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + grade = case_verdicts[0].value + return ReducedVerdict( + value={'acc': float(grade.overall_score)}, + metadata={ + 'requirement_status': grade.requirement_status, + 'grading_rationale': grade.grading_rationale, + } + ) + + def finalize(score, review, judge_context) -> Score: + score.explanation = review.metadata.get('grading_rationale', '') return score - rubrics_text = build_rubrics_text(reference) - grading_prompt = ( - "Starting now, you are a rigorous instruction-following grading teacher. Your task is to accurately grade and score student answers based on the 【Rubrics】.\n\n" - "Grading Criteria\n" - "This is a strict, all-or-nothing grading system. The final score is binary.\n" - "To receive a score of 1, the student's answer must perfectly satisfy every single requirement listed in the 【Rubrics】.\n" - "If even one requirement is not fully met, the final score will be 0.\n" - "Grading Process\n" - "Please strictly follow the steps below for analysis—no steps may be skipped:\n" - "Step 1: Analyze the Standard Answer\n" - "List all explicit requirements in the 【Rubrics】 item by item (including format, content, quantity, order, etc.).\n" - "Identify implicit requirements in the 【Rubrics】 (e.g., language style, logical structure).\n" - "Define specific evaluation criteria for each requirement (e.g., \"must include X,\" \"must not exceed Y\").\n" - "Step 2: Check Each Requirement Against the Student's Answer\n" - "For every requirement in the 【Rubrics】, verify one by one whether the student's answer fully satisfies it.\n" - "Step 3: Self-Reflection\n" - "Before giving the final score, you must conduct the following checks:\n" - " Completeness Check: Whether all requirements in the standard answer have been reviewed with no omissions.\n" - " Strictness Check: Whether the evaluation strictly adheres to the \"fully satisfied\" standard without relaxing requirements due to subjective judgment.\n" - " Consistency Check: Whether the grading rationale aligns logically with the final score.\n" - " Objectivity Check: Whether judgments are based on objective facts rather than subjective speculation.\n" - "Output Format Requirements\n" - "【Grading Rationale】: xxx\n" - "【List of Requirement Satisfaction Status】: [x₁, x₂, …, xᵢ, …, xₙ] (where n is the total number of requirements in the 【Rubrics】, and xᵢ indicates whether the student's answer meets the i-th requirement, with values \"yes\"/\"no\")\n" - "【Overall Score】: x points (x is an integer, either 0 or 1.)\n\n" - "Content to Be Graded\n" - f"【Rubrics】:\n{rubrics_text}\n" - f"【Student Response】:\n{filtered_prediction}\n" - "\nPlease strictly output ONLY the following JSON format (do not output any other content):\n" - "{\n" - ' "Grading Rationale": "Your detailed grading rationale",\n' - ' "List of Requirement Satisfaction Status": ["yes", "no", ...],\n' - ' "Overall Score": 0 or 1\n' - "}\n" + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='grade', output_contract=GRADE_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='acc', + finalize=finalize ) - messages = [ChatMessageUser(content=grading_prompt)] - - judge_response = self.llm_judge.judge(messages=messages) - - parsed = None - json_block = extract_json_block(judge_response) - if json_block: - try: - parsed = json.loads(json_block) - except json.JSONDecodeError as exc: - logger.warning(f'Failed to parse CL-bench judge response: {exc}') - - if not isinstance(parsed, dict): - score.value = {'acc': 0.0} - score.explanation = f'LLM judge parsing failed: {judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id, - 'judge_response': judge_response, - } - score.main_score_name = 'acc' - return score - - overall_score = parsed.get('Overall Score', 0) - try: - overall_score = int(overall_score) - except (TypeError, ValueError): - overall_score = 0 - overall_score = 1 if overall_score == 1 else 0 - - score.value = {'acc': float(overall_score)} - score.explanation = parsed.get('Grading Rationale', '') - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id, - 'requirement_status': parsed.get('List of Requirement Satisfaction Status', []), - 'judge_response': parsed, - } - score.main_score_name = 'acc' - return score diff --git a/evalscope/benchmarks/cl_bench/utils.py b/evalscope/benchmarks/cl_bench/utils.py index bedb11d3b..0385f1d0a 100644 --- a/evalscope/benchmarks/cl_bench/utils.py +++ b/evalscope/benchmarks/cl_bench/utils.py @@ -16,16 +16,3 @@ def build_rubrics_text(rubrics: Optional[Iterable[Any]]) -> str: if criteria: lines.append(f'{index}. {criteria}') return '\n'.join(lines) if lines else 'No specific rubrics provided.' - - -def extract_json_block(result_text: str) -> Optional[str]: - # Remove code block wrapper if present - if result_text.startswith('```json'): - result_text = result_text[7:] - if result_text.startswith('```'): - result_text = result_text[3:] - if result_text.endswith('```'): - result_text = result_text[:-3] - result_text = result_text.strip() - - return result_text diff --git a/evalscope/benchmarks/claw_eval/claw_eval_adapter.py b/evalscope/benchmarks/claw_eval/claw_eval_adapter.py index 5352f658b..94afa25d3 100644 --- a/evalscope/benchmarks/claw_eval/claw_eval_adapter.py +++ b/evalscope/benchmarks/claw_eval/claw_eval_adapter.py @@ -189,9 +189,10 @@ def _on_inference(self, model: Model, sample: Sample) -> InferenceResult: trace_root = output_root / 'traces' api_key = self._resolve_api_key(model) base_url = self._resolve_base_url(model) - judge_args = self._task_config.judge_model_args or {} + judge_args = self._task_config.judge.models[0].model_dump(exclude={'judge_id'}, exclude_none=True) \ + if self._task_config.judge.models else {} judge_model = judge_args.get('model_id') - no_judge = self._task_config.judge_strategy == JudgeStrategy.RULE + no_judge = self._task_config.judge.strategy == JudgeStrategy.RULE official_config = self._build_official_config( model=model, tasks_dir=tasks_dir, diff --git a/evalscope/benchmarks/cmmu/cmmu_adapter.py b/evalscope/benchmarks/cmmu/cmmu_adapter.py index cb18885f5..8cf41d486 100644 --- a/evalscope/benchmarks/cmmu/cmmu_adapter.py +++ b/evalscope/benchmarks/cmmu/cmmu_adapter.py @@ -1,20 +1,31 @@ -import ast -import re +from pydantic import BaseModel, Field from typing import Any, Dict, List from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState -from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict +from evalscope.api.messages import ChatMessageSystem, ChatMessageUser, Content, ContentImage, ContentText from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.io_utils import bytes_to_base64 from evalscope.utils.logger import get_logger from evalscope.utils.multi_choices import MultipleChoiceTemplate, answer_character, parse_answers_zh, prompt logger = get_logger() + +class CmmuGrade(BaseModel): + """The judge's required reply: a proportion of correctly answered blanks, plus its reasoning.""" + + correct: float = Field(ge=0.0, le=1.0) + analysis: str = '' + + +# The judge prompt requires exactly this JSON object as the whole reply. +GRADE_CONTRACT = OutputContract(schema_model=CmmuGrade) + SUBSET_LIST = ['biology', 'chemistry', 'geography', 'history', 'math', 'physics', 'politics'] MULT_CHOICE_PROMPT = MultipleChoiceTemplate.CHINESE_SINGLE_ANSWER_TEMPLATE_COT @@ -78,7 +89,7 @@ ) class CMMUAdapter(VisionLanguageAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_DEFAULT def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -130,50 +141,46 @@ def extract_answer(self, prediction, task_state): else: return prediction.strip() - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - ) + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + if context.task_state.metadata['type'] in (MULTI_CHOICE_TYPE, MULTIPLE_RESPONSE_TYPE): + return JudgeDefinition.skip( + Score( + extracted_prediction=context.filtered_prediction, + prediction=context.original_prediction, + value={'acc': 1.0 if context.filtered_prediction == context.reference else 0.0}, + main_score_name='acc' + ), + reason='deterministic_choice_scoring', + ) - question = task_state.input_text - question_type = task_state.metadata['type'] - if question_type in [MULTI_CHOICE_TYPE, MULTIPLE_RESPONSE_TYPE]: - score.value = {'acc': 1 if filtered_prediction == reference else 0} - else: + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: from .prompt import EVALUATION_SYSTEM_PROMPT, EVALUATION_USER_TEMPLATE - - prompt = EVALUATION_USER_TEMPLATE.format( - question=question, target=reference, predicted_answer=original_prediction + prompt_text = EVALUATION_USER_TEMPLATE.format( + question=judge_context.task_state.input_text, + target=judge_context.reference, + predicted_answer=judge_context.original_prediction, ) - judge_response = self.llm_judge.judge(prompt, system_prompt=EVALUATION_SYSTEM_PROMPT) - try: - ans_parsed = ast.literal_eval(judge_response) - correctness = ans_parsed.get('correct', 0) - analysis = ans_parsed.get('analysis', '') - except Exception: - pattern = re.compile(r'"correct"\s*:\s*1') - match = re.search(pattern, judge_response) - correctness = 1 if match else 0 - - score.value = { - 'acc': correctness, - } - score.explanation = f'LLM judge: {judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'analysis': analysis, - 'model': self.llm_judge.model_id - } - score.main_score_name = 'acc' - return score + return JudgeRequest( + messages=[ChatMessageSystem(content=EVALUATION_SYSTEM_PROMPT), + ChatMessageUser(content=prompt_text)] + ) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + grade = case_verdicts[0].value + return ReducedVerdict(value={'acc': grade.correct}, metadata={'analysis': grade.analysis}) + + def finalize(score, review, judge_context) -> Score: + for observation in review.valid_observations: + score.metadata.update(observation.reduced.metadata) + return score + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='grade', output_contract=GRADE_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='acc', + finalize=finalize + ) @staticmethod def create_content_and_answers_list(record: Dict[str, Any]) -> tuple[List[Content], List[str]]: diff --git a/evalscope/benchmarks/count_qa/count_qa_adapter.py b/evalscope/benchmarks/count_qa/count_qa_adapter.py index 7981c005e..c621906cb 100644 --- a/evalscope/benchmarks/count_qa/count_qa_adapter.py +++ b/evalscope/benchmarks/count_qa/count_qa_adapter.py @@ -44,7 +44,7 @@ rule the paper states for its rewriter LLM. A reply with no digit scores 0, so `max_tokens` must leave the model room to reach its answer; a model that narrates its count ("row 1 has 3 ...") is scored on the first number it mentions rather than on its stated total -- Scoring is deterministic arithmetic and needs no LLM judge: keep `judge_strategy` at `rule` or +- Scoring is deterministic arithmetic and needs no LLM judge: keep `judge.strategy` at `rule` or `auto`, since `llm` replaces both metrics with a generic judge score. To read a different number out of a model that ignores the output format, prepend a per-run filter such as `filters={'regex': {'regex_pattern': '(\\d+)', 'group_select': -1}}` (last number) via diff --git a/evalscope/benchmarks/data_collection/data_collection_adapter.py b/evalscope/benchmarks/data_collection/data_collection_adapter.py index b35f8274c..9affd0816 100644 --- a/evalscope/benchmarks/data_collection/data_collection_adapter.py +++ b/evalscope/benchmarks/data_collection/data_collection_adapter.py @@ -16,6 +16,19 @@ logger = get_logger() +SAMPLE_COLUMNS = [ + 'task_type', + 'categories', + 'dataset_name', + 'subset_name', + 'tags', + 'sample_id', + 'metric', + 'score', + 'sample_weight', +] +"""Columns of the per-sample aggregation frame, declared so an all-excluded run still has them.""" + @register_benchmark( BenchmarkMeta( @@ -206,6 +219,11 @@ def _build_sample_dataframe(self, sample_scores: List[SampleScore]): main_metric = sample_score.score.main_score_name sample_weight = float(collection_info.get('weight', 1.0)) + # A sample whose judge review was unusable has no value and no main score; it is + # excluded from every aggregate rather than counted as 0. + if main_score is None: + continue + # Each row represents one sample records.append({ 'task_type': collection_info['task_type'], @@ -219,7 +237,9 @@ def _build_sample_dataframe(self, sample_scores: List[SampleScore]): 'sample_weight': sample_weight, }) # NOTE: All sample weights are assumed (as per new requirement) to sum to ~1 globally. - return pd.DataFrame(records) + # The columns are declared so that an all-excluded run still yields a frame every groupby + # and the report generator can read, rather than a column-less frame that raises KeyError. + return pd.DataFrame(records, columns=SAMPLE_COLUMNS) def _group_and_compute(self, df, group_cols, macro_child: Optional[str] = None): """ @@ -280,6 +300,9 @@ def _build_category_level_report(self, df): Category-level hierarchical aggregation using sample-level weights. Macro is the mean of subset-level micro averages. """ + if df.empty: + # No sample means no category depth to expand, and grouping on no column raises. + return [] df_categories = df.copy() max_depth = df_categories['categories'].apply(len).max() for level in range(max_depth): diff --git a/evalscope/benchmarks/deepsearchqa/deepsearchqa_adapter.py b/evalscope/benchmarks/deepsearchqa/deepsearchqa_adapter.py index 31af7cc14..41bf26bfd 100644 --- a/evalscope/benchmarks/deepsearchqa/deepsearchqa_adapter.py +++ b/evalscope/benchmarks/deepsearchqa/deepsearchqa_adapter.py @@ -3,10 +3,18 @@ from evalscope.api.benchmark import AgentLoopAdapter, BenchmarkMeta from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, ReducedVerdict +from evalscope.api.messages import ChatMessageUser from evalscope.api.metric import AggScore, SampleScore, Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags -from .utils import aggregate_official_scores, build_grader_prompt, parse_judge_response, rule_fallback_score +from evalscope.constants import ScoringPolicy, Tags +from .utils import ( + GRADE_CONTRACT, + aggregate_official_scores, + build_grader_prompt, + metrics_from_grade, + rule_fallback_score, +) DEEPSEARCHQA_DATASET_ID = 'google/deepsearchqa' @@ -64,7 +72,7 @@ ) class DeepSearchQAAdapter(AgentLoopAdapter): """Adapter for the DeepSearchQA deep-research benchmark.""" - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_DEFAULT def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) @@ -107,46 +115,41 @@ def match_score( main_score_name='f1', ) - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - answer_type = task_state.metadata['answer_type'] - prompt = build_grader_prompt( - question=task_state.input_text, - reference=reference, - answer_type=answer_type, - response=filtered_prediction, - ) - - if not filtered_prediction: - judge_response = '' - value, judge_metadata = {}, { - 'empty_model_response': True, - 'error_message': 'AI response was empty.', - } - else: - judge_response = self.llm_judge.judge(prompt) - value, judge_metadata = parse_judge_response(judge_response) - - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - value=value, - explanation=f'LLM judge: {judge_response}', - main_score_name='f1', + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + if not context.filtered_prediction: + return JudgeDefinition.skip( + Score( + extracted_prediction=context.filtered_prediction, + prediction=context.original_prediction, + value={}, + main_score_name='f1', + metadata={ + 'empty_model_response': True, + 'error_message': 'AI response was empty.' + } + ), + reason='empty_model_output', + ) + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + prompt = build_grader_prompt( + question=judge_context.task_state.input_text, + reference=judge_context.reference, + answer_type=judge_context.task_state.metadata['answer_type'], + response=judge_context.filtered_prediction, + ) + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + value, metadata = metrics_from_grade(case_verdicts[0].value) + return ReducedVerdict(value=value, metadata=metadata) + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='grade', output_contract=GRADE_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='f1' ) - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id, - 'rating_prompt': prompt, - **judge_metadata, - } - return score def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]: return aggregate_official_scores(sample_scores) diff --git a/evalscope/benchmarks/deepsearchqa/utils.py b/evalscope/benchmarks/deepsearchqa/utils.py index e3828c9fa..61577773e 100644 --- a/evalscope/benchmarks/deepsearchqa/utils.py +++ b/evalscope/benchmarks/deepsearchqa/utils.py @@ -1,7 +1,8 @@ -import json import re -from typing import Any, Dict, List, Optional, Tuple +from pydantic import BaseModel, ConfigDict, Field +from typing import Any, Dict, List, Tuple +from evalscope.api.judge import OutputContract from evalscope.api.metric import AggScore, SampleScore DEEPSEARCH_QA_PROMPT = """ @@ -98,41 +99,35 @@ def rule_fallback_score(prediction: str, reference: str, answer_type: str) -> Tu } -def parse_judge_response(response: str) -> Tuple[Dict[str, float], Dict[str, Any]]: - if not response: - return {}, { - 'empty_auto_rater_response': True, - 'error_message': 'Auto-rater response was empty.', - 'judge_raw': response, - } +class AnswerCorrectness(BaseModel): + """The judge's ``Answer Correctness`` node, using the official DeepSearchQA keys.""" - parsed = _extract_json_object(response) - if parsed is None: - return _invalid_score('Invalid JSON response from auto-rater.', response) + model_config = ConfigDict(populate_by_name=True) - answer_correctness = parsed.get('Answer Correctness') - if not isinstance(answer_correctness, dict): - return _invalid_score("Missing or malformed 'Answer Correctness' node.", response) + explanation: str = Field(alias='Explanation') + correctness_details: Dict[str, bool] = Field(alias='Correctness Details') + excessive_answers: List[str] = Field(default_factory=list, alias='Excessive Answers') - explanation = answer_correctness.get('Explanation') - if not isinstance(explanation, str): - return _invalid_score("Missing or malformed 'Explanation' in Answer Correctness.", response) - details = answer_correctness.get('Correctness Details') - if not isinstance(details, dict) or not all(isinstance(k, str) for k in details.keys()) \ - or not all(isinstance(v, bool) for v in details.values()): - return _invalid_score("Invalid 'Correctness Details' in Answer Correctness.", response) +class Grade(BaseModel): + model_config = ConfigDict(populate_by_name=True) - excessive_answers = answer_correctness.get('Excessive Answers', []) - if not isinstance(excessive_answers, list) or not all(isinstance(item, str) for item in excessive_answers): - return _invalid_score("Invalid 'Excessive Answers' in Answer Correctness.", response) + answer_correctness: AnswerCorrectness = Field(alias='Answer Correctness') + +GRADE_CONTRACT = OutputContract(schema_model=Grade) + + +def metrics_from_grade(grade: Grade) -> Tuple[Dict[str, float], Dict[str, Any]]: + """Fold the judge's correctness verdict into precision/recall/f1 plus diagnostic metadata.""" + details = grade.answer_correctness.correctness_details + excessive_answers = grade.answer_correctness.excessive_answers expected = len(details) correct = sum(1 for value in details.values() if value) excessive = len([answer for answer in excessive_answers if answer.strip()]) metadata = { - 'answer_correctness_explanation': explanation, + 'answer_correctness_explanation': grade.answer_correctness.explanation, 'correct': correct, 'expected': expected, 'excessive': excessive, @@ -149,19 +144,17 @@ def aggregate_official_scores(sample_scores: List[SampleScore]) -> List[AggScore valid_scores = [] empty_model = 0 - empty_auto_rater = 0 - invalid_auto_rater = 0 + judge_failure = 0 for sample_score in sample_scores: metadata = sample_score.score.metadata or {} if metadata.get('empty_model_response'): empty_model += 1 continue - if metadata.get('empty_auto_rater_response'): - empty_auto_rater += 1 - continue - if metadata.get('invalid_auto_rater_response'): - invalid_auto_rater += 1 + # A judge that could not produce a usable verdict (empty or malformed) is excluded from the + # means and reported as a single failure rate, rather than the old empty/invalid split. + if not sample_score.score.status.is_usable: + judge_failure += 1 continue valid_scores.append(sample_score) @@ -193,8 +186,7 @@ def aggregate_official_scores(sample_scores: List[SampleScore]) -> List[AggScore for metric_name, count in { 'empty_model_response': empty_model, - 'empty_auto_rater_response': empty_auto_rater, - 'invalid_auto_rater_response': invalid_auto_rater, + 'judge_parse_failure': judge_failure, }.items(): agg_scores.append(AggScore( score=count / total, @@ -219,27 +211,6 @@ def _split_reference(answer: str) -> List[str]: return [part.strip() for part in re.split(r',|;|\n', answer_text) if part.strip()] -def _extract_json_object(text: str) -> Optional[Dict[str, Any]]: - if not text: - return None - - json_str = text.strip() - start_marker = '```json' - start_idx = json_str.find(start_marker) - if start_idx != -1: - json_str = json_str[start_idx + len(start_marker):].strip() - end_idx = json_str.rfind('```') - if end_idx != -1: - json_str = json_str[:end_idx].strip() - - try: - parsed = json.loads(json_str) - except json.JSONDecodeError: - return None - - return parsed if isinstance(parsed, dict) else None - - def _calculate_metric(true_positives: int, false_positives: int, false_negatives: int) -> Dict[str, float]: precision = true_positives / (true_positives + false_positives) if true_positives + false_positives else 0.0 recall = true_positives / (true_positives + false_negatives) if true_positives + false_negatives else 0.0 @@ -257,11 +228,3 @@ def _score_from_counts(correct: int, expected: int, excessive: int) -> Dict[str, false_positives=excessive, false_negatives=expected - correct, ) - - -def _invalid_score(error_message: str, response: str) -> Tuple[Dict[str, float], Dict[str, Any]]: - return {}, { - 'invalid_auto_rater_response': True, - 'error_message': error_message, - 'judge_raw': response, - } diff --git a/evalscope/benchmarks/docmath/docmath_adapter.py b/evalscope/benchmarks/docmath/docmath_adapter.py index a98cd62d5..83a5821d4 100644 --- a/evalscope/benchmarks/docmath/docmath_adapter.py +++ b/evalscope/benchmarks/docmath/docmath_adapter.py @@ -1,16 +1,28 @@ -from typing import Any, Dict +from pydantic import BaseModel +from typing import Any, Dict, List, Literal from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState -from evalscope.api.messages import ChatMessageUser +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict +from evalscope.api.messages import ChatMessageSystem, ChatMessageUser from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger logger = get_logger() + +# The judge prompt requires the verdict as "[[YES]]" or "[[NO]]" after its explanation; a bare +# "YES" anywhere in that explanation must not decide the score. +class Equivalence(BaseModel): + reasoning: str = '' + verdict: Literal['YES', 'NO'] + + +EQUIVALENCE_CONTRACT = OutputContract(schema_model=Equivalence) + TEMPLATE_0SHOT = """Please read the following text and answer the question below. @@ -63,7 +75,7 @@ ) class DocMathAdapter(DefaultDataAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_DEFAULT def __init__(self, **kwargs): super().__init__(**kwargs) @@ -125,42 +137,26 @@ def match_score( return score - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - """ - Use LLM judge to evaluate the prediction against the reference. - """ - from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE - - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE + prompt = ORM_USER_TEMPLATE.format( + problem=judge_context.task_state.metadata.get('question', ''), + answer_1=judge_context.reference, + answer_2=judge_context.filtered_prediction, + ) + case.output_contract.instruction() + return JudgeRequest( + messages=[ChatMessageSystem(content=GENERAL_ORM_PROMPT), + ChatMessageUser(content=prompt)] + ) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + return ReducedVerdict(value={'acc': 1.0 if case_verdicts[0].value.verdict == 'YES' else 0.0}) + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='equivalence', output_contract=EQUIVALENCE_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='acc' ) - - question = task_state.metadata.get('question', '') - - # Get grading response - prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=reference, answer_2=filtered_prediction) - orm_response = self.llm_judge.judge(prompt, system_prompt=GENERAL_ORM_PROMPT) - - # Parse grading response - if 'YES' in orm_response: - accuracy = 1.0 - else: - accuracy = 0.0 - - score.value = {'acc': accuracy} - score.explanation = f'LLM judge: {orm_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id - } - score.main_score_name = 'acc' - - return score diff --git a/evalscope/benchmarks/docmath/utils.py b/evalscope/benchmarks/docmath/utils.py index cc3a7ee8d..0ad921b64 100644 --- a/evalscope/benchmarks/docmath/utils.py +++ b/evalscope/benchmarks/docmath/utils.py @@ -14,7 +14,6 @@ Your output must follow the following format: 1) Provide an explanation for why the answers are equivalent or not. -2) Then provide your final answer in the form of: [[YES]] or [[NO]] """ # noqa: E501 ORM_USER_TEMPLATE = """ diff --git a/evalscope/benchmarks/drivelology/drivelology_writing_adapter.py b/evalscope/benchmarks/drivelology/drivelology_writing_adapter.py index f847fc7b5..a66928942 100644 --- a/evalscope/benchmarks/drivelology/drivelology_writing_adapter.py +++ b/evalscope/benchmarks/drivelology/drivelology_writing_adapter.py @@ -1,17 +1,26 @@ -import re +from pydantic import BaseModel, Field from typing import Any, Dict, List from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict from evalscope.api.messages import ChatMessageUser, ContentText from evalscope.api.metric.scorer import AggScore, SampleScore, Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger logger = get_logger() + +class NarrativeRating(BaseModel): + reasoning: str = '' + rating: int = Field(ge=1, le=5) + + +RATING_CONTRACT = OutputContract(schema_model=NarrativeRating) + DESCRIPTION = """ ## Overview @@ -62,7 +71,6 @@ 4 = Good match 5 = Excellent match -Please format your rating strictly as: "Rating: [[X]]" where X is a whole number from 1 to 5. [Candidate Narrative] {candidate} @@ -97,7 +105,7 @@ ) class DrivelologyNarrativeWritingAdapter(DefaultDataAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -144,100 +152,49 @@ def batch_match_score(self, original_predictions, filtered_predictions, referenc scores.append(score) return scores - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - """ - Calculate the match score using LLM judge and BERTScore. - """ - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - ) + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + prompt = NARRATIVE_EVALUATION_TEMPLATE.format( + candidate=judge_context.filtered_prediction, + reference=judge_context.reference, + ) + case.output_contract.instruction() + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) - # Initialize score value dictionary - score.value = {} - - # Use LLM judge to evaluate narrative quality - eval_prompt = NARRATIVE_EVALUATION_TEMPLATE.format(candidate=filtered_prediction, reference=reference) - - judge_response = self.llm_judge.judge(eval_prompt) - logger.info(f'LLM judge response received (first 100 chars): {judge_response[:100]}...') - - # Extract rating using regex pattern - match = re.search(r'Rating:\s*\[\[([1-5])\]\]', judge_response) - if match: - rating = int(match.group(1)) - gpt_score = (rating - 1) / 4.0 # Normalize to 0-1 scale - logger.info(f'Rating extracted: {rating}/5 -> {gpt_score}') - else: - # Try alternative pattern - alt_match = re.search(r'(\[\[|\[)([1-5])(\]\]|\])', judge_response) - if alt_match: - rating = int(alt_match.group(2)) - gpt_score = (rating - 1) / 4.0 - logger.info(f'Rating extracted (alt pattern): {rating}/5 -> {gpt_score}') - else: - # Last resort: standalone digit - number_match = re.search(r'(? {gpt_score}') - else: - gpt_score = 0.0 - logger.warning('No rating found in response, using default 0.0') - - score.value['judge_score'] = gpt_score - score.explanation = f'LLM judge rating: {gpt_score:.2f}' - - score.metadata = { - 'judge_response': judge_response[:300], - 'model': getattr(self.llm_judge, 'model_id', 'unknown') - } - - score.main_score_name = 'judge_score' - return score + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + rating = case_verdicts[0].value.rating + return ReducedVerdict(value={'judge_score': (rating - 1) / 4.0}, metadata={'rating': rating}) + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='rating', output_contract=RATING_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='judge_score' + ) def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]: """ Aggregate scores across all samples. + + Each metric is averaged only over the samples that actually carry it: a sample whose judge + review was unusable has no ``judge_score`` and is excluded from that mean rather than + counted as 0, while its rule-based ``bert_score`` still contributes. """ - if not sample_scores: - return [ - AggScore(metric_name='judge_score', score=0.0, num=0, metadata={}), - AggScore(metric_name='bert_score', score=0.0, num=0, metadata={}) - ] - - # Extract scores - gpt_scores = [ss.score.value.get('judge_score', 0.0) for ss in sample_scores] - bert_scores = [ss.score.value.get('bert_score', 0.0) for ss in sample_scores] - - # Calculate averages - avg_gpt_score = sum(gpt_scores) / len(gpt_scores) if gpt_scores else 0.0 - avg_bert_score = sum(bert_scores) / len(bert_scores) if bert_scores else 0.0 - - return [ - AggScore( - metric_name='judge_score', - score=avg_gpt_score, - num=len(sample_scores), - metadata={ - 'min_score': min(gpt_scores), - 'max_score': max(gpt_scores) - } - ), - AggScore( - metric_name='bert_score', - score=avg_bert_score, - num=len(sample_scores), - metadata={ - 'min_score': min(bert_scores), - 'max_score': max(bert_scores) - } + results: List[AggScore] = [] + for metric_name in ('judge_score', 'bert_score'): + values = [ss.score.value[metric_name] for ss in sample_scores if metric_name in ss.score.value] + if not values: + results.append(AggScore(metric_name=metric_name, score=0.0, num=0, metadata={})) + continue + results.append( + AggScore( + metric_name=metric_name, + score=sum(values) / len(values), + num=len(values), + metadata={ + 'min_score': min(values), + 'max_score': max(values), + }, + ) ) - ] + return results diff --git a/evalscope/benchmarks/frames/frames_adapter.py b/evalscope/benchmarks/frames/frames_adapter.py index b3462fab1..b6c6ee036 100644 --- a/evalscope/benchmarks/frames/frames_adapter.py +++ b/evalscope/benchmarks/frames/frames_adapter.py @@ -1,11 +1,24 @@ -from typing import Any, Dict +from pydantic import BaseModel +from typing import Any, Dict, List, Literal from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import DatasetDict, Sample, load_local_file_dataset, resolve_snapshot_or_local_path from evalscope.api.evaluator import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict +from evalscope.api.messages import ChatMessageSystem, ChatMessageUser from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags + + +# The judge prompt requires the verdict as "[[YES]]" or "[[NO]]" after its explanation; a bare +# "YES" anywhere in that explanation must not decide the score. +class Equivalence(BaseModel): + reasoning: str = '' + verdict: Literal['YES', 'NO'] + + +EQUIVALENCE_CONTRACT = OutputContract(schema_model=Equivalence) TEMPLATE_0SHOT = """Please read the following text and answer the question below. @@ -59,7 +72,7 @@ ) class FramesAdapter(DefaultDataAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_DEFAULT def __init__(self, **kwargs): super().__init__(**kwargs) @@ -146,42 +159,26 @@ def match_score( return score - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - """ - Use LLM judge to evaluate the prediction against the reference. - """ - from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE - - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE + prompt = ORM_USER_TEMPLATE.format( + problem=judge_context.task_state.input_text, + answer_1=judge_context.reference, + answer_2=judge_context.filtered_prediction, + ) + case.output_contract.instruction() + return JudgeRequest( + messages=[ChatMessageSystem(content=GENERAL_ORM_PROMPT), + ChatMessageUser(content=prompt)] + ) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + return ReducedVerdict(value={'acc': 1.0 if case_verdicts[0].value.verdict == 'YES' else 0.0}) + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='equivalence', output_contract=EQUIVALENCE_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='acc' ) - - question = task_state.input_text - - # Get grading response - prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=reference, answer_2=filtered_prediction) - orm_response = self.llm_judge.judge(prompt, system_prompt=GENERAL_ORM_PROMPT) - - # Parse grading response - if 'YES' in orm_response: - accuracy = 1.0 - else: - accuracy = 0.0 - - score.value = {'acc': accuracy} - score.explanation = f'LLM judge: {orm_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id - } - score.main_score_name = 'acc' - - return score diff --git a/evalscope/benchmarks/frames/utils.py b/evalscope/benchmarks/frames/utils.py index 04d8bfbc3..a5423ad25 100644 --- a/evalscope/benchmarks/frames/utils.py +++ b/evalscope/benchmarks/frames/utils.py @@ -27,7 +27,6 @@ def lower(text): Your output must follow the following format: 1) Provide an explanation for why the answers are equivalent or not. -2) Then provide your final answer in the form of: [[YES]] or [[NO]] """ # noqa: E501 ORM_USER_TEMPLATE = """ diff --git a/evalscope/benchmarks/gdpval/gdpval_adapter.py b/evalscope/benchmarks/gdpval/gdpval_adapter.py index 4776f17be..3af4ff375 100644 --- a/evalscope/benchmarks/gdpval/gdpval_adapter.py +++ b/evalscope/benchmarks/gdpval/gdpval_adapter.py @@ -259,25 +259,6 @@ def match_score( ) return score - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - score = self.match_score( - original_prediction=original_prediction, - filtered_prediction=filtered_prediction, - reference=reference, - task_state=task_state, - ) - score.metadata['judge_strategy_note'] = ( - 'GDPval does not use EvalScope local LLM judging. Use OpenAI\'s official GDPval judge on the exported ' - 'submission package.' - ) - return score - def _on_generate_report_end(self, report: Any, output_dir: str, **kwargs: Any) -> None: if is_build_doc(): return diff --git a/evalscope/benchmarks/general_arena/general_arena_adapter.py b/evalscope/benchmarks/general_arena/general_arena_adapter.py index 34d24ea32..e5250d6e1 100644 --- a/evalscope/benchmarks/general_arena/general_arena_adapter.py +++ b/evalscope/benchmarks/general_arena/general_arena_adapter.py @@ -2,22 +2,43 @@ import glob import os from collections import defaultdict -from typing import Any, Dict, List +from pydantic import BaseModel +from typing import Any, Dict, List, Literal from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import DatasetDict, Sample, build_dataset_from_records from evalscope.api.evaluator import TaskState -from evalscope.api.messages.chat_message import ChatMessageUser +from evalscope.api.judge import ( + JudgeCase, + JudgeContext, + JudgeDefinition, + JudgeRequest, + OutputContract, + PairwiseOutcome, + PairwisePlacementOutcome, + Placement, + ReducedVerdict, +) +from evalscope.api.messages.chat_message import ChatMessageSystem, ChatMessageUser from evalscope.api.metric import AggScore, SampleScore, Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.report import Report, ReportKey from evalscope.utils.import_utils import check_import from evalscope.utils.logger import get_logger logger = get_logger() -GRADER_SYSTEM_PROMPT = "Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better.\n\nBegin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.\n\nWhen evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.\n\nThen consider if the assistant's answers are helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.\n\nThen consider the creativity and novelty of the assistant's answers when needed. Finally, identify any missing important information in the assistants' answers that would be beneficial to include when responding to the user prompt.\n\nAfter providing your explanation, you must output only one of the following choices as your final verdict with a label:\n\n1. Assistant A is significantly better: [[A>>B]]\n2. Assistant A is slightly better: [[A>B]]\n3. Tie, relatively the same: [[A=B]]\n4. Assistant B is slightly better: [[B>A]]\n5. Assistant B is significantly better: [[B>>A]]\n\nExample output: \"My final verdict is tie: [[A=B]]\"." # noqa: E501 + +class BattleVerdict(BaseModel): + """One game's verdict on the official five-point preference scale.""" + reasoning: str = '' + verdict: Literal['A>>B', 'A>B', 'A=B', 'B>A', 'B>>A'] + + +BATTLE_CONTRACT = OutputContract(schema_model=BattleVerdict) + +GRADER_SYSTEM_PROMPT = "Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better.\n\nBegin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.\n\nWhen evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.\n\nThen consider if the assistant's answers are helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.\n\nThen consider the creativity and novelty of the assistant's answers when needed. Finally, identify any missing important information in the assistants' answers that would be beneficial to include when responding to the user prompt.\n\nAfter providing your explanation, you must state your final verdict as one of: A>>B (Assistant A is significantly better), A>B (Assistant A is slightly better), A=B (tie), B>A (Assistant B is slightly better), or B>>A (Assistant B is significantly better)." # noqa: E501 GRADER_TEMPLATE = "<|User Prompt|>\n{question}\n\n<|The Start of Assistant A's Answer|>\n{answer_1}\n<|The End of Assistant A's Answer|>\n\n<|The Start of Assistant B's Answer|>\n{answer_2}\n<|The End of Assistant B's Answer|>".strip( ) # noqa: E501 @@ -86,7 +107,7 @@ ) class GeneralArenaAdapter(DefaultDataAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -217,8 +238,8 @@ def _build_pair_wise_data(self, dataset_dict): pairs = [] for model_item, baseline_item in zip(model_data[name], model_data[self.baseline]): # Convert to ReviewResult objects like in get_model_prediction - model_review = ReviewResult.model_validate(model_item) - baseline_review = ReviewResult.model_validate(baseline_item) + model_review = ReviewResult.from_cache_item(model_item) + baseline_review = ReviewResult.from_cache_item(baseline_item) for model_choice, baseline_choice in zip( process_review_item(model_review), process_review_item(baseline_review) @@ -234,71 +255,66 @@ def _build_pair_wise_data(self, dataset_dict): return pairwise_data - def llm_match_score( - self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState - ) -> Score: - """Use LLM as a judge to evaluate the predicted answer against the baseline.""" - from .utils import get_judge_score, post_process_result + supports_position_swap = True + official_position_swap = True + """Each pair is judged twice with the two answers swapped.""" - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - ) + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: - question = task_state.input_text - answer_1 = task_state.metadata['answer_1'] - answer_2 = reference # baseline answer - model_1 = task_state.metadata['model_1'] - model_2 = task_state.metadata['model_2'] - - system_template = self.system_prompt - prompt_template = self.prompt_template - - prompt1 = prompt_template.format(question=question, answer_1=answer_1, answer_2=answer_2) - # reverse the order - prompt2 = prompt_template.format(question=question, answer_1=answer_2, answer_2=answer_1) - - # get grading response - game1_response = self.llm_judge.judge(prompt1, system_prompt=system_template) - game2_response = self.llm_judge.judge(prompt2, system_prompt=system_template) - - # parse grading response - # game1 - res1 = post_process_result(game1_response) - score1 = get_judge_score(res1, reverse=False) - # game2 - res2 = post_process_result(game2_response) - score2 = get_judge_score(res2, reverse=True) - - battle_result = { - 'score': (score1 + score2) / 2, - 'games': [ - { - 'model_a': model_1, - 'model_b': model_2, - 'response': game1_response, - 'judgment': res1 - }, - { - 'model_a': model_2, - 'model_b': model_1, - 'response': game2_response, - 'judgment': res2 - }, - ] - } - - score.value = {'score': battle_result['score']} - score.explanation = f'LLM judge battles: Game1: {game1_response[:100]}... Game2: {game2_response[:100]}...' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': getattr(self, 'judge_strategy', 'default'), - 'model': self.llm_judge.model_id if hasattr(self.llm_judge, 'model_id') else 'unknown', - 'battle_result': battle_result - } - score.main_score_name = 'score' + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + metadata = judge_context.task_state.metadata or {} + candidate_first = placement is Placement.ORIGINAL + prompt = self.prompt_template.format( + question=judge_context.task_state.input_text, + answer_1=metadata['answer_1'] if candidate_first else judge_context.reference, + answer_2=judge_context.reference if candidate_first else metadata['answer_1'], + ) + return JudgeRequest( + messages=[ + ChatMessageSystem(content=self.system_prompt), + ChatMessageUser(content=prompt + case.output_contract.instruction()) + ] + ) - return score + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + placements = case_verdicts[0].placements + res1 = placements.get('original', case_verdicts[0].value).verdict + res2 = placements.get('swapped') + outcomes = {'original': _placement_outcome(res1, candidate_is_a=True)} + if res2 is not None: + outcomes['swapped'] = _placement_outcome(res2.verdict, candidate_is_a=False) + result, strength = _reduce_placements(outcomes) + outcome = PairwiseOutcome(metric_name='score', result=result, strength=strength, placements=outcomes) + return ReducedVerdict(value={'score': outcome.score}, outcome=outcome) + + def finalize(score, review, judge_context) -> Score: + if review.outcome is not None: + metadata = judge_context.task_state.metadata or {} + model_1, model_2 = metadata['model_1'], metadata['model_2'] + score.metadata['battle_result'] = { + 'score': review.value['score'], + 'games': [ + { + 'model_a': model_1, + 'model_b': model_2, + 'judgment': _battle_label(review.outcome.placements['original'], candidate_is_a=True) + }, + *([{ + 'model_a': model_2, + 'model_b': model_1, + 'judgment': _battle_label(review.outcome.placements['swapped'], candidate_is_a=False) + }] if 'swapped' in review.outcome.placements else []), + ], + } + return score + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='battle', output_contract=BATTLE_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='score', + finalize=finalize + ) def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]: """Aggregate scores to compute winrate.""" @@ -307,7 +323,11 @@ def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]: from .utils import compute_mle_elo, get_battles_from_row, get_bootstrap_result, get_win_rate_column - battles = pd.concat([get_battles_from_row(res.score.metadata['battle_result']) for res in sample_scores]) + # A sample whose judge verdicts were unusable has no battle to contribute. + scored = [res for res in sample_scores if (res.score.metadata or {}).get('battle_result')] + if not scored: + return [] + battles = pd.concat([get_battles_from_row(res.score.metadata['battle_result']) for res in scored]) bt_model_coef = compute_mle_elo(battles, baseline_model=self.baseline) @@ -489,3 +509,33 @@ def parse_dataset_key(dataset_key): f.write('\n'.join(leaderboard_outputs)) logger.info(f'Leaderboard results saved to: {leaderboard_file}') + + +def _candidate_outcome(label: str, candidate_is_a: bool) -> str: + if label == 'A=B': + return 'tie' + return 'win' if label.startswith('A') == candidate_is_a else 'loss' + + +def _placement_outcome(label: str, candidate_is_a: bool) -> PairwisePlacementOutcome: + return PairwisePlacementOutcome( + result=_candidate_outcome(label, candidate_is_a), + strength='strong' if label in ('A>>B', 'B>>A') else 'weak', + ) + + +def _reduce_placements(placements: Dict[str, PairwisePlacementOutcome]) -> tuple[str, str]: + results = [placement.result for placement in placements.values()] + result = results[0] if len(set(results)) == 1 else 'tie' + strength = 'strong' if result != 'tie' and any( + placement.result == result and placement.strength == 'strong' for placement in placements.values() + ) else 'weak' + return result, strength + + +def _battle_label(outcome: PairwisePlacementOutcome, candidate_is_a: bool) -> str: + if outcome.result == 'tie': + return 'A=B' + a_wins = (outcome.result == 'win') == candidate_is_a + marker = '>>' if outcome.strength == 'strong' else '>' + return f'A{marker}B' if a_wins else f'B{marker}A' diff --git a/evalscope/benchmarks/general_arena/utils.py b/evalscope/benchmarks/general_arena/utils.py index 38e02e8f6..da133fb47 100644 --- a/evalscope/benchmarks/general_arena/utils.py +++ b/evalscope/benchmarks/general_arena/utils.py @@ -42,14 +42,6 @@ def process_review_item(review_result: ReviewResult) -> list: return [raw_d] -def post_process_result(completion): - result = re.findall(r'\[\[([AB<>=]+)\]\]', completion) - if result: - return result[0] - else: - return None - - def get_judge_score(result, reverse=False): """ Calculate the judge score, considering confidence weight. @@ -113,6 +105,9 @@ def get_battles_from_row(row, first_game_only=False, multiplier=3): if first_game_only: return pd.DataFrame(results) + if len(row['games']) == 1: + return pd.DataFrame(results) + # Dont change the order of model_a and model_b output = {'model_a': game['model_a'], 'model_b': game['model_b']} diff --git a/evalscope/benchmarks/healthbench/healthbench_adapter.py b/evalscope/benchmarks/healthbench/healthbench_adapter.py index 6c2696420..fe1f7d11f 100644 --- a/evalscope/benchmarks/healthbench/healthbench_adapter.py +++ b/evalscope/benchmarks/healthbench/healthbench_adapter.py @@ -1,18 +1,28 @@ import copy import os -from typing import Any, Dict +from pydantic import BaseModel +from typing import Any, Dict, List from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import DatasetDict, Sample, load_local_file_dataset, resolve_snapshot_or_local_path +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict from evalscope.api.messages.chat_message import ChatMessageUser, dict_to_chat_message from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags -from evalscope.utils.function_utils import retry_call +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger logger = get_logger() + +class RubricGrade(BaseModel): + """The judge's reply per rubric item; GRADER_TEMPLATE already asks for exactly these keys.""" + explanation: str = '' + criteria_met: bool + + +RUBRIC_CONTRACT = OutputContract(schema_model=RubricGrade) + GRADER_TEMPLATE = """ Your job is to look at a conversation and a rubric item, and score the last turn (i.e., the last assistant response, or the completion) in the conversation on how well it follows the rubric item. @@ -163,7 +173,7 @@ class HealthBenchAdapter(DefaultDataAdapter): This adapter supports multiple dataset versions and uses LLM judges to evaluate responses against detailed medical criteria. """ - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, *args, **kwargs): """ @@ -226,83 +236,46 @@ def record_to_sample(self, record: Dict[str, Any]) -> Sample: theme = tags[0].split(':')[1].strip() if len(tags) > 0 else 'Unknown' return Sample(input=input_messages, target='', subset_key=theme, metadata=record) - def llm_match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score: - """ - Evaluate AI response using LLM judge against physician-created rubrics. - - Args: - original_prediction: The AI model's original response - filtered_prediction: Filtered/processed version of the response - reference: Reference answer (not used in this evaluation) - task_state: Contains metadata including rubric items - - Returns: - Score: Contains overall score, rubric tag scores, and explanations - """ - from .utils import ( - RubricItem, - calculate_rubric_tag_scores, - calculate_score, - construct_readable_explanation, - parse_json_to_dict, - ) - - # Initialize the score object with prediction details - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + cases = [ + JudgeCase(case_id=f'rubric_{index}', output_contract=RUBRIC_CONTRACT, metadata={'rubric_index': index}) + for index in range(len((context.task_state.metadata or {}).get('rubrics', []))) + ] + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + from .utils import RubricItem + metadata = judge_context.task_state.metadata or {} + rubric = RubricItem.from_dict(metadata['rubrics'][case.metadata['rubric_index']]) + conversation = metadata['prompt'] + [dict(content=judge_context.original_prediction, role='assistant')] + prompt = GRADER_TEMPLATE.replace( + '<>', '\n\n'.join(f"{message['role']}: {message['content']}" for message in conversation) + ).replace('<>', str(rubric)) + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + from .utils import RubricItem, calculate_rubric_tag_scores, calculate_score, construct_readable_explanation + metadata = copy.deepcopy(judge_context.task_state.metadata or {}) + items = [RubricItem.from_dict(item) for item in metadata['rubrics']] + by_case = {verdict.case_id: verdict for verdict in case_verdicts} + responses = [by_case[f'rubric_{index}'].value.model_dump() for index in range(len(items))] + tags, axes = calculate_rubric_tag_scores(items, responses) + return ReducedVerdict( + value={ + 'overall_score': calculate_score(items, responses), + **axes + }, + metadata={ + 'readable_explanation': construct_readable_explanation(items, responses), + 'rubric_tag_scores': tags, + } + ) + + def finalize(score, review, judge_context) -> Score: + judge_context.task_state.target = '**Score Explanation**\n\n' + review.metadata.get( + 'readable_explanation', '' + ) + return score + + return JudgeDefinition.workflow( + cases=cases, request=request, reduce=reduce, main_score_name='overall_score', finalize=finalize ) - - # Extract rubric items and conversation from task metadata - example = copy.deepcopy(task_state.metadata) - rubric_items = [RubricItem.from_dict(d) for d in example['rubrics']] - # Construct full conversation including the AI response - convo_with_response = example['prompt'] + [dict(content=original_prediction, role='assistant')] - # Format conversation as readable string - convo_str = '\n\n'.join([f"{m['role']}: {m['content']}" for m in convo_with_response]) - - # Evaluate response against each rubric item using LLM judge - grading_response_list = [] - for rubric_item in rubric_items: - # Create judge prompt by substituting conversation and rubric item - grader_prompt = GRADER_TEMPLATE.replace('<>', - convo_str).replace('<>', str(rubric_item)) - messages = [ChatMessageUser(content=grader_prompt)] - - def judge_func(): - grading_response = self.llm_judge.judge(messages=messages) - grading_response_dict = parse_json_to_dict(grading_response) - # Validate response format and extract boolean criteria_met field - if 'criteria_met' not in grading_response_dict or not isinstance( - grading_response_dict['criteria_met'], bool - ): - logger.warning('Grading failed due to bad JSON output, retrying...') - raise ValueError('Grading failed due to bad JSON output') - return grading_response_dict - - # Retry logic for robust evaluation - grading_result = retry_call(judge_func, retries=3, sleep_interval=1) - grading_response_list.append(grading_result) - - # Calculate final scores and explanations - overall_score = calculate_score(rubric_items, grading_response_list) # Overall weighted score - rubric_tag_scores, axis_grades = calculate_rubric_tag_scores( - rubric_items, grading_response_list - ) # Scores by category - readable_explanation = construct_readable_explanation( - rubric_items, grading_response_list - ) # Human-readable results - - # Set score values and metadata - score.value = { - 'overall_score': overall_score, - **axis_grades, # Include axis scores at top level - } - score.main_score_name = 'overall_score' - score.metadata = { - 'readable_explanation': readable_explanation, - 'rubric_tag_scores': rubric_tag_scores, - } - # Store explanation in sample target for reference - task_state.target = '**Score Explanation**\n\n' + readable_explanation - return score diff --git a/evalscope/benchmarks/healthbench/utils.py b/evalscope/benchmarks/healthbench/utils.py index 642762c15..2e5b5ed76 100644 --- a/evalscope/benchmarks/healthbench/utils.py +++ b/evalscope/benchmarks/healthbench/utils.py @@ -7,17 +7,6 @@ logger = get_logger() -def parse_json_to_dict(json_string: str) -> dict: - # Remove markdown-style ```json``` markers if present - json_cleaned = re.sub(r'^```json\s*|\s*```$', '', json_string.strip()) - - try: - return json.loads(json_cleaned) - except json.JSONDecodeError as e: - logger.warning(f'JSON decoding failed: {e}') - return {} - - class RubricItem: def __init__(self, criterion: str, points: float, tags: list[str]): diff --git a/evalscope/benchmarks/hipho/hipho_adapter.py b/evalscope/benchmarks/hipho/hipho_adapter.py index a817e1530..e0649f31b 100644 --- a/evalscope/benchmarks/hipho/hipho_adapter.py +++ b/evalscope/benchmarks/hipho/hipho_adapter.py @@ -1,6 +1,7 @@ # flake8: noqa: E501 import json import os +from pydantic import BaseModel, Field, create_model from typing import Any, Dict, List, Optional, Tuple from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter @@ -11,10 +12,11 @@ resolve_snapshot_or_local_path, ) from evalscope.api.evaluator import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger from .utils import ( ANSWER_JUDGE_PROMPT, @@ -25,13 +27,28 @@ extract_boxed_answers, is_chinese_exam, normalize_marking, - parse_judge_correct, - parse_judge_points, strip_boxed, ) logger = get_logger() + +class AnswerVerdict(BaseModel): + """The judge's [Correct]/[Incorrect] answer-level verdict, as JSON.""" + correct: bool + + +ANSWER_CONTRACT = OutputContract(schema_model=AnswerVerdict) + + +def _step_grade_model(max_points: float): + """Per-criterion schema whose upper bound matches the criterion's stated allocation.""" + return create_model( + 'StepGrade', + awarded=(float, Field(ge=0.0, le=float(max_points))), + ) + + # One subset per Olympiad exam paper, keyed by the record ``source`` field. SUBSET_LIST = [ 'APhO_2025', @@ -78,8 +95,8 @@ ## Evaluation Notes -- Requires an LLM judge: run with `judge_strategy='llm'` (or `'auto'`, which enables the judge for this benchmark) - and provide `judge_model_args`. `judge_strategy='rule'` is not supported. +- Requires an LLM judge: set `judge.strategy='llm'` (or `'auto'`, which enables the judge for this benchmark) + and provide `judge.models`. `judge.strategy='rule'` is not supported. - Primary metric: `accuracy`, the per-problem awarded/attainable point ratio in `[0, 1]`, aggregated by mean per subset. For step-level problems the attainable maximum is the sum of the marking criteria; for problems with several official schemes (EuPhO, NBPhO) the highest-scoring scheme is used, matching the paper. @@ -115,7 +132,7 @@ class HiPhOAdapter(VisionLanguageAdapter): without one are graded by matching their boxed final answers. """ - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) @@ -208,102 +225,198 @@ def _load_figure(self, image_ref: str) -> Optional[str]: with open(path, 'rb') as f: return self._image_bytes_to_base64(f.read(), default_format=ext) - def match_score( - self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState - ) -> Score: - raise ValueError( - 'HiPhO is graded against official marking schemes and requires an LLM judge. ' - "Set judge_strategy='llm' (or 'auto') and provide judge_model_args." + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + immediate, skip_reason = self._rule_score( + context.original_prediction, context.filtered_prediction, context.reference, context.task_state + ) + if immediate is not None: + assert skip_reason is not None + return JudgeDefinition.skip(immediate, reason=skip_reason) + return JudgeDefinition.workflow( + cases=self._build_cases(context), + request=self._build_request, + reduce=self._reduce_verdicts, + main_score_name='acc', + finalize=self._finalize_score ) - def llm_match_score( - self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState - ) -> Score: + def _rule_score(self, original_prediction: str, filtered_prediction: str, reference: str, + task_state: TaskState) -> Tuple[Optional[Score], Optional[str]]: + # Both flows short-circuit before touching the judge when there is nothing to grade. metadata = task_state.metadata or {} - score = Score(extracted_prediction=filtered_prediction, prediction=original_prediction) - if metadata.get('marking'): - acc, detail = self._score_step_level(original_prediction, metadata) + if not any(scheme for scheme in metadata['marking']): + return ( + self._empty_score(filtered_prediction, original_prediction, task_state, 'no marking criteria'), + 'no_marking_criteria', + ) else: - acc, detail = self._score_answer_level(original_prediction, metadata) - - score.value = {'acc': acc} - score.main_score_name = 'acc' - score.explanation = detail - score.metadata = { - 'source': 'hipho_llm_judge', - 'grading': 'step_level' if metadata.get('marking') else 'answer_level', - 'judge_model': self.llm_judge.model_id, - } - # Surface the official answer in the review Gold column, which is otherwise - # empty because grading is judge-based rather than reference-based. + if not [strip_boxed(a) for a in metadata['answers']]: + return ( + self._empty_score(filtered_prediction, original_prediction, task_state, 'no ground-truth answer'), + 'missing_ground_truth_answer', + ) task_state.target = self._format_target(metadata) - return score + return None, None + + def _empty_score(self, filtered: str, original: str, task_state: TaskState, reason: str) -> Score: + task_state.target = self._format_target(task_state.metadata or {}) + return Score( + extracted_prediction=filtered, + prediction=original, + value={'acc': 0.0}, + main_score_name='acc', + explanation=reason, + ) - @staticmethod - def _format_target(metadata: Dict[str, Any]) -> str: - """Render the official answer for display in review files and the dashboard.""" - answers = [strip_boxed(a) for a in metadata['answers'] if (a or '').strip()] - if answers: - return ' | '.join(answers) - # Open-ended problems ship no reference answer, only a marking scheme. - criteria = sum(len(scheme) for scheme in metadata['marking']) - return f'graded on {criteria} marking criteria' + def _build_cases(self, context: JudgeContext) -> List[JudgeCase]: + metadata = context.task_state.metadata or {} + if metadata.get('marking'): + return self._build_step_cases(metadata) + return self._build_answer_cases(metadata, context.original_prediction) - def _score_step_level(self, prediction: str, metadata: Dict[str, Any]) -> Tuple[float, str]: - """Grade every criterion of each official scheme and keep the best scheme.""" - question = metadata['question'] + def _build_step_cases(self, metadata: Dict[str, Any]) -> List[JudgeCase]: + cases: List[JudgeCase] = [] + for scheme_idx, scheme in enumerate(metadata['marking']): + for crit_idx, criterion in enumerate(scheme): + max_points = criterion_points(criterion) + if max_points <= 0: + # A criterion with no parseable point allocation cannot be judged; count it as + # zero, as the previous evaluator did, without spending a judge call. + continue + cases.append( + JudgeCase( + case_id=f'step:{scheme_idx}:{crit_idx}', + output_contract=OutputContract(schema_model=_step_grade_model(max_points)), + metadata={ + 'kind': 'step', + 'scheme_idx': scheme_idx, + 'criterion': criterion, + 'max_points': max_points, + }, + ) + ) + return cases + + def _build_answer_cases(self, metadata: Dict[str, Any], prediction: str) -> List[JudgeCase]: + from evalscope.metrics.math import math_equal + + gold_answers = [strip_boxed(a) for a in metadata['answers']] + pred_boxed = extract_boxed_answers(prediction) + aligned = pred_boxed[-len(gold_answers):] if pred_boxed else [] + cases: List[JudgeCase] = [] + for idx, gold in enumerate(gold_answers): + pred = aligned[idx] if idx < len(aligned) else '' + # Rule check first: only ambiguous cases go to the judge, mirroring the old flow. + if pred and math_equal(pred, gold): + continue + if not pred: + continue + cases.append( + JudgeCase( + case_id=f'answer:{idx}', + output_contract=ANSWER_CONTRACT, + metadata={ + 'kind': 'answer', + 'answer_idx': idx, + 'gold': gold, + 'pred': pred, + }, + ) + ) + return cases + + def _build_request(self, case, placement, completed_cases, context) -> JudgeRequest: + metadata = context.task_state.metadata or {} + if case.metadata['kind'] == 'step': + prompt = STEP_JUDGE_PROMPT.format( + question=metadata['question'], + prediction=context.original_prediction, + criterion=case.metadata['criterion'], + ) + else: + prompt = ANSWER_JUDGE_PROMPT.format( + question=metadata['question'], + given_answer=case.metadata['pred'], + ground_truth=case.metadata['gold'], + ) + prompt += case.output_contract.instruction() + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) + + def _reduce_verdicts(self, case_verdicts, context) -> ReducedVerdict: + metadata = context.task_state.metadata or {} + if metadata.get('marking'): + return self._reduce_step(case_verdicts, metadata) + return self._reduce_answer(case_verdicts, metadata, context.original_prediction) + + def _reduce_step(self, case_verdicts, metadata: Dict[str, Any]) -> ReducedVerdict: + verdicts_by_case = {verdict.case_id: verdict for verdict in case_verdicts} best_ratio = 0.0 best_detail = '' for scheme_idx, scheme in enumerate(metadata['marking']): awarded = 0.0 attainable = 0.0 lines: List[str] = [] - for criterion in scheme: + for crit_idx, criterion in enumerate(scheme): max_points = criterion_points(criterion) + if max_points <= 0: + continue attainable += max_points - prompt = STEP_JUDGE_PROMPT.format(question=question, prediction=prediction, criterion=criterion) - response = self.llm_judge.judge(prompt) - points = parse_judge_points(response, max_points) + verdict = verdicts_by_case.get(f'step:{scheme_idx}:{crit_idx}') + # `Field(le=max_points)` bounds the judge's number to the criterion's allocation. + points = float(verdict.value.awarded) if verdict is not None else 0.0 awarded += points lines.append(f'{points:g}/{max_points:g}') if attainable <= 0: - # No criterion stated a point allocation, so the scheme cannot be - # graded. Warn instead of silently reporting a zero score. - logger.warning( - f'HiPhO sample {metadata["id"]} scheme {scheme_idx} has no parseable point ' - f'allocation in its marking criteria; scoring it as 0.' - ) continue ratio = awarded / attainable if ratio >= best_ratio: best_ratio = ratio best_detail = f'scheme {scheme_idx}: {awarded:g}/{attainable:g} [' + ', '.join(lines) + ']' - return best_ratio, best_detail + return ReducedVerdict( + value={'acc': best_ratio}, + metadata={ + 'grading': 'step_level', + 'detail': best_detail + }, + ) - def _score_answer_level(self, prediction: str, metadata: Dict[str, Any]) -> Tuple[float, str]: - """Match each boxed final answer against the ground truth (rule first, judge fallback).""" + def _reduce_answer(self, case_verdicts, metadata: Dict[str, Any], prediction: str) -> ReducedVerdict: from evalscope.metrics.math import math_equal gold_answers = [strip_boxed(a) for a in metadata['answers']] - if not gold_answers: - return 0.0, 'no ground-truth answer' - pred_boxed = extract_boxed_answers(prediction) - # The final answers correspond to the trailing boxed expressions, aligned - # in order with the ground-truth sub-answers. aligned = pred_boxed[-len(gold_answers):] if pred_boxed else [] - + verdicts_by_case = {verdict.case_id: verdict for verdict in case_verdicts} correct = 0 lines: List[str] = [] for idx, gold in enumerate(gold_answers): pred = aligned[idx] if idx < len(aligned) else '' - hit = bool(pred) and (math_equal(pred, gold) or self._judge_answer(metadata, pred, gold)) + if pred and math_equal(pred, gold): + hit = True + else: + verdict = verdicts_by_case.get(f'answer:{idx}') + hit = bool(verdict) and verdict.value.correct correct += int(hit) lines.append(f'{"✓" if hit else "✗"}({pred or "∅"}|{gold})') - return correct / len(gold_answers), ' '.join(lines) + return ReducedVerdict( + value={'acc': correct / len(gold_answers) if gold_answers else 0.0}, + metadata={ + 'grading': 'answer_level', + 'detail': ' '.join(lines) + }, + ) - def _judge_answer(self, metadata: Dict[str, Any], prediction: str, gold: str) -> bool: - """Fall back to the LLM judge for answer equivalence when rules do not match.""" - prompt = ANSWER_JUDGE_PROMPT.format(question=metadata['question'], given_answer=prediction, ground_truth=gold) - return parse_judge_correct(self.llm_judge.judge(prompt)) + def _finalize_score(self, score: Score, review, context) -> Score: + score.explanation = review.metadata.get('detail', '') + return score + + @staticmethod + def _format_target(metadata: Dict[str, Any]) -> str: + """Render the official answer for display in review files and the dashboard.""" + answers = [strip_boxed(a) for a in metadata['answers'] if (a or '').strip()] + if answers: + return ' | '.join(answers) + # Open-ended problems ship no reference answer, only a marking scheme. + criteria = sum(len(scheme) for scheme in metadata['marking']) + return f'graded on {criteria} marking criteria' diff --git a/evalscope/benchmarks/hipho/utils.py b/evalscope/benchmarks/hipho/utils.py index 09d7d0573..981ef8fcf 100644 --- a/evalscope/benchmarks/hipho/utils.py +++ b/evalscope/benchmarks/hipho/utils.py @@ -65,11 +65,8 @@ Question: {question} Output sentence: {given_answer} Correct answer: {ground_truth} -Final Instruction: -You must respond with exactly one of the following: [Correct] or [Incorrect]. -Do NOT include any explanation, reasoning, or additional text. -Any deviation from this format (even a single word) will be considered INVALID. -Judgement: """ +Judge whether the output sentence correctly answers the question. +""" # Step-level judge (HiPhO paper, Appendix B.3). The criterion text itself states the # points to award, so the judge returns the awarded points as a single number. @@ -84,21 +81,14 @@ 1. Analyze the student's solution for physics concepts, mathematical derivations, and calculations. 2. Award points strictly according to the criterion. 3. Consider both conceptual understanding and technical accuracy. -Critical: -1. You MUST respond with ONLY a single number (e.g., 1.0, 0.5, 0.0). -2. NO explanations, NO text, NO reasoning - JUST THE NUMBER. -3. If you provide any text other than the number, your response will be invalid. -Score: """ +Award points strictly according to the criterion. +""" # Every criterion states its own allocation, e.g. "Award 0.1 pt if ..." or "得 0.5 分". _CRITERION_POINTS_RE = re.compile( r'(?:award|給|给|得|扣)\s*\$?\s*([0-9]*\.?[0-9]+)\s*\$?\s*(?:pts?|points?|分)', re.IGNORECASE, ) -_JUDGE_NUMBER_RE = re.compile(r'-?[0-9]*\.?[0-9]+') - -# Sentinel that ``LLMJudge.judge`` returns instead of raising on a failed request. -JUDGE_ERROR_PREFIX = '[ERROR]' def is_chinese_exam(source: str) -> bool: @@ -130,38 +120,6 @@ def criterion_points(criterion: str) -> float: return max(values) if values else 0.0 -def parse_judge_points(response: str, max_points: float) -> float: - """Parse the awarded points from a step-level judge response. - - The judge is instructed to return a bare number; the first number in the - response is used and clamped to ``[0, max_points]`` so a malformed judge - reply can never inflate or deflate a criterion beyond its allocation. - - A failed judge request must score 0, never full credit: ``LLMJudge.judge`` - reports failures as an ``[ERROR] ...`` string that embeds the model id and - endpoint, whose digits would otherwise be parsed as an awarded score. - """ - if not response or response.startswith(JUDGE_ERROR_PREFIX): - return 0.0 - match = _JUDGE_NUMBER_RE.search(response) - if not match: - return 0.0 - return max(0.0, min(float(match.group()), max_points)) - - -def parse_judge_correct(response: str) -> bool: - """Parse a ``[Correct]`` / ``[Incorrect]`` answer-level judge verdict.""" - if not response or response.startswith(JUDGE_ERROR_PREFIX): - return False - text = response.lower() - if '[incorrect]' in text: - return False - if '[correct]' in text: - return True - # Fall back to a bare token when the judge omits the brackets. - return bool(re.search(r'\bcorrect\b', text)) and 'incorrect' not in text - - def extract_boxed_answers(text: str) -> List[str]: """Extract the contents of every complete ``\\boxed{...}`` in order of appearance. diff --git a/evalscope/benchmarks/hle/hle_adapter.py b/evalscope/benchmarks/hle/hle_adapter.py index 9ce6a2c52..527d58985 100644 --- a/evalscope/benchmarks/hle/hle_adapter.py +++ b/evalscope/benchmarks/hle/hle_adapter.py @@ -1,19 +1,38 @@ import re -from typing import Any, Dict, List +from pydantic import BaseModel +from typing import Any, Dict, List, Literal from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState +from evalscope.api.judge import ( + CaseVerdict, + JudgeCase, + JudgeContext, + JudgeDefinition, + JudgeRequest, + OutputContract, + ReducedVerdict, +) from evalscope.api.messages import ChatMessage, ChatMessageSystem, ChatMessageUser, Content, ContentImage, ContentText from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger # flake8: noqa logger = get_logger() + +# The judge prompt requires "GRADE: C" or "GRADE: I" as the final line. +class Grade(BaseModel): + reasoning: str = '' + verdict: Literal['C', 'I'] + + +GRADE_CONTRACT = OutputContract(schema_model=Grade) + SUBSET_LIST = [ 'Biology/Medicine', 'Chemistry', @@ -94,7 +113,7 @@ ) class HLEAdapter(DefaultDataAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -136,48 +155,34 @@ def sample_filter(self, sample): return False return True - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - ) - - confidence = 100 - if task_state.output and task_state.output.completion: - confidence_match = re.search(r'confidence:\s*(\d+)', task_state.output.completion, re.IGNORECASE) - if confidence_match: - confidence = int(confidence_match.group(1)) - - judge_prompt = JUDGE_PROMPT.format( - question=task_state.input_text, response=filtered_prediction, correct_answer=reference + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + prompt = JUDGE_PROMPT.format( + question=judge_context.task_state.input_text, + response=judge_context.filtered_prediction, + correct_answer=judge_context.reference, + ) + case.output_contract.instruction() + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + return ReducedVerdict(value={'acc': 1.0 if case_verdicts[0].value.verdict == 'C' else 0.0}) + + def finalize(score, review, judge_context) -> Score: + score.metadata['confidence'] = self._stated_confidence(judge_context.task_state) + return score + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='grade', output_contract=GRADE_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='acc', + finalize=finalize ) - # Request judge and obtain score - judge_response = self.llm_judge.judge(prompt=judge_prompt) - - # Parse judge response to get accuracy score - accuracy_score = re.search(r'GRADE:\s*([CI])', judge_response, re.IGNORECASE) - if accuracy_score: - grade = accuracy_score.group(1).upper() - score.value = { - 'acc': 1.0 if grade == 'C' else 0.0, - } - else: - score.value = { - 'acc': 0.0, - } - score.explanation = f'LLM judge: {judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id, - 'confidence': confidence, - } - score.main_score_name = 'acc' - return score + @staticmethod + def _stated_confidence(task_state: TaskState) -> int: + """Read the confidence the evaluated model reported; unrelated to the judge verdict.""" + completion = task_state.output.completion if task_state.output else '' + match = re.search(r'confidence:\s*(\d+)', completion or '', re.IGNORECASE) + return int(match.group(1)) if match else 100 diff --git a/evalscope/benchmarks/image_edit/gedit/gedit_adapter.py b/evalscope/benchmarks/image_edit/gedit/gedit_adapter.py index 26f1f7f7d..bf758997b 100644 --- a/evalscope/benchmarks/image_edit/gedit/gedit_adapter.py +++ b/evalscope/benchmarks/image_edit/gedit/gedit_adapter.py @@ -1,15 +1,17 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import copy import os +from pydantic import BaseModel, Field from typing import Any, Dict, List from evalscope.api.benchmark import BenchmarkMeta, ImageEditAdapter from evalscope.api.dataset import Sample from evalscope.api.evaluator.state import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict from evalscope.api.messages import ChatMessage, ChatMessageUser, Content, ContentImage, ContentText from evalscope.api.metric.scorer import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import FileConstants, Tags +from evalscope.constants import FileConstants, ScoringPolicy, Tags from evalscope.utils.io_utils import bytes_to_base64 from evalscope.utils.logger import get_logger @@ -23,6 +25,14 @@ LANGUAGE_LIST = ['en', 'cn'] +class GeditGrade(BaseModel): + score: List[int] = Field(min_length=1) + reasoning: str = '' + + +GEDIT_CONTRACT = OutputContract(schema_model=GeditGrade) + + @register_benchmark( BenchmarkMeta( name='gedit', @@ -75,7 +85,7 @@ ) class GEditAdapter(ImageEditAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, **kwargs): super().__init__(**kwargs) @@ -118,57 +128,44 @@ def sample_filter(self, sample: Sample) -> bool: language = sample.metadata.get('instruction_language', 'en') return super().sample_filter(sample) and language == self.language - def llm_match_score(self, original_prediction, filtered_prediction, reference, task_state: TaskState) -> Score: - import math - - from .utils import mllm_output_to_dict - - metadata = task_state.metadata - text_prompt = metadata['instruction'] - input_image = metadata['input_image'] # base64 image - edited_image = metadata[FileConstants.IMAGE_PATH] # local image path - _SC_prompt = self.SC_prompt.replace('', text_prompt) - - # Initialize the score object with prediction details - score = Score( - extracted_prediction=edited_image, - prediction=edited_image, - ) + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + cases = [ + JudgeCase(case_id='SC', output_contract=GEDIT_CONTRACT, metadata={'kind': 'SC'}), + JudgeCase(case_id='PQ', output_contract=GEDIT_CONTRACT, metadata={'kind': 'PQ'}) + ] - # Build prompts - SC_prompt_final = [ - ChatMessageUser( - content=[ - ContentImage(image=input_image), + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + metadata = judge_context.task_state.metadata or {} + edited_image = metadata[FileConstants.IMAGE_PATH] + if case.metadata['kind'] == 'SC': + content = [ + ContentImage(image=metadata['input_image']), ContentImage(image=edited_image), - ContentText(text=_SC_prompt) + ContentText(text=self.SC_prompt.replace('', metadata['instruction'])) ] + else: + content = [ContentImage(image=edited_image), ContentText(text=self.PQ_prompt)] + content[-1] = ContentText(text=content[-1].text + case.output_contract.instruction()) + return JudgeRequest(messages=[ChatMessageUser(content=content)]) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + import math + by_case = {verdict.case_id: verdict for verdict in case_verdicts} + semantic, perceptual = min(by_case['SC'].value.score), min(by_case['PQ'].value.score) + return ReducedVerdict( + value={ + 'semantic_consistency': float(semantic), + 'perceptual_similarity': float(perceptual), + 'normalized_score': math.sqrt(semantic * perceptual) + } ) - ] - PQ_prompt_final = [ - ChatMessageUser(content=[ContentImage(image=edited_image), - ContentText(text=self.PQ_prompt)]) - ] - guess_if_cannot_parse = True - result_SC = self.llm_judge.judge(messages=SC_prompt_final) - result_PQ = self.llm_judge.judge(messages=PQ_prompt_final) - SC_dict = mllm_output_to_dict(result_SC, give_up_parsing=guess_if_cannot_parse) - PQ_dict = mllm_output_to_dict(result_PQ, give_up_parsing=guess_if_cannot_parse) - - SC_score = min(SC_dict['score']) - PQ_score = min(PQ_dict['score']) - O_score = math.sqrt(SC_score * PQ_score) - - # `normalized_score` is the official Overall: the geometric mean of SC and PQ. - score.value = { - 'semantic_consistency': SC_score, - 'perceptual_similarity': PQ_score, - 'normalized_score': O_score, - } - score.main_score_name = 'normalized_score' - score.metadata = { - 'SC_dict': SC_dict, - 'PQ_dict': PQ_dict, - } - return score + def finalize(score, review, judge_context) -> Score: + image_path = (judge_context.task_state.metadata or {}).get(FileConstants.IMAGE_PATH, '') + score.extracted_prediction = image_path + score.prediction = image_path + return score + + return JudgeDefinition.workflow( + cases=cases, request=request, reduce=reduce, main_score_name='normalized_score', finalize=finalize + ) diff --git a/evalscope/benchmarks/imo_answerbench/imo_answerbench_adapter.py b/evalscope/benchmarks/imo_answerbench/imo_answerbench_adapter.py index 97475e64c..31f1a814f 100644 --- a/evalscope/benchmarks/imo_answerbench/imo_answerbench_adapter.py +++ b/evalscope/benchmarks/imo_answerbench/imo_answerbench_adapter.py @@ -7,7 +7,7 @@ from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger logger = get_logger() @@ -70,7 +70,7 @@ ) class IMOAnswerBenchAdapter(DefaultDataAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_DEFAULT def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/evalscope/benchmarks/job_bench/job_bench_adapter.py b/evalscope/benchmarks/job_bench/job_bench_adapter.py index 9c897adf9..b4cbc2904 100644 --- a/evalscope/benchmarks/job_bench/job_bench_adapter.py +++ b/evalscope/benchmarks/job_bench/job_bench_adapter.py @@ -2,6 +2,7 @@ import shutil from pathlib import Path +from pydantic import BaseModel, Field from typing import Any, Dict, Optional from evalscope.agent.tools.bash import BASH_TOOL_INFO, run_bash @@ -11,11 +12,21 @@ from evalscope.api.benchmark.adapters import AgentLoopAdapter from evalscope.api.dataset import DatasetHub, Sample from evalscope.api.evaluator import TaskState +from evalscope.api.judge import ( + CaseVerdict, + JudgeCase, + JudgeContext, + JudgeDefinition, + JudgeRequest, + OutputContract, + ReducedVerdict, +) +from evalscope.api.messages import ChatMessageSystem, ChatMessageUser, ContentImage, ContentText from evalscope.api.metric import Score from evalscope.api.model import Model from evalscope.api.registry import register_benchmark from evalscope.api.sandbox import merge_sandbox_config_dicts -from evalscope.constants import HubType, JudgeStrategy, Tags +from evalscope.constants import HubType, ScoringPolicy, Tags from evalscope.utils.import_utils import is_build_doc from evalscope.utils.logger import get_logger from .utils import ( @@ -23,12 +34,31 @@ SANDBOX_REFERENCE_DIR, JobBenchArtifactEnvironment, artifact_dir, - evaluate_job_bench_output, + build_judge_prompt, + build_scorecard, + collect_image_attachments, + extract_all_file_contents, + normalize_criteria, parse_rubrics, + rubric_needs_vision, ) logger = get_logger() + +class _CriterionVerdict(BaseModel): + index: int + passed: bool + reasoning: str = '' + evidence: str = '' + + +class _RubricVerdict(BaseModel): + criteria_results: List[_CriterionVerdict] = Field(default_factory=list) + rubric_passed: bool + overall_reasoning: str = '' + + _DATASET_ID = 'evalscope/job-bench' _DEFAULT_DOCKER_IMAGE = 'python:3.11-slim-bookworm' @@ -50,7 +80,7 @@ ## Evaluation Notes - The default evaluation split is `main`. -- Configure `judge_model_args` for rubric scoring. +- Configure `judge.models` for rubric scoring. - `normalized_score` is the primary weighted score (raw total / max score); `pass_rate` is the unweighted proportion of fully passed rubrics; `judge_score` is the raw sum of passed rubric weights, reported as a diagnostic. - Docker runs use `python:3.11-slim-bookworm` by default. For formal evaluation, provide an image with the Office, @@ -86,14 +116,12 @@ class JobBenchAdapter(AgentLoopAdapter): """JobBench adapter using ModelScope data and EvalScope's agent loop.""" - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY strategy_name = 'function_calling' max_steps_default = 250 def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) - if self.judge_strategy == JudgeStrategy.LLM_RECALL: - raise ValueError("JobBench does not support judge_strategy='llm_recall'; use 'auto' or 'llm'.") self._snapshot_dir: Optional[Path] = None def record_to_sample(self, record: Dict[str, Any]) -> Sample: @@ -200,62 +228,105 @@ def _build_docker_environment(self, sample: Sample) -> AgentEnvironment: metadata=sample.metadata, ) - def match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - return Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - value={ - 'normalized_score': 0.0, - 'pass_rate': 0.0, - 'judge_score': 0.0, - }, - metadata={ - 'official_score_computed': False, - 'note': 'JobBench quality scoring requires judge_model_args.', - }, + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + return JudgeDefinition.workflow( + cases=self._build_cases(context), + request=self._build_request, + reduce=self._reduce_verdicts, main_score_name='normalized_score', + finalize=self._finalize_score ) - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - if self.llm_judge is None: - raise ValueError('JobBench requires judge_model_args for LLM-based rubric scoring.') - - evaluation = evaluate_job_bench_output( - output_dir=Path(task_state.metadata['artifact_dir']) / SANDBOX_OUTPUT_DIR, - rubrics=parse_rubrics(task_state.metadata.get('rubric_json')), - judge=self.llm_judge.judge, + def _build_cases(self, context: JudgeContext) -> List[JudgeCase]: + output_dir = Path(context.task_state.metadata['artifact_dir']) / SANDBOX_OUTPUT_DIR + rubrics = parse_rubrics(context.task_state.metadata.get('rubric_json')) + if not rubrics or not output_dir.exists() or not any(path.is_file() for path in output_dir.rglob('*')): + return [] + contents = extract_all_file_contents(output_dir) + if not contents.strip(): + return [] + attachments = collect_image_attachments(output_dir) + return [ + JudgeCase( + case_id=f'rubric:{index}', + output_contract=OutputContract(schema_model=_RubricVerdict), + metadata={ + 'index': index, + 'rubric': rubric, + 'file_contents': contents, + 'image_attachments': attachments, + }, + ) for index, rubric in enumerate(rubrics) + ] + + def _build_request(self, case, placement, completed_cases, context) -> JudgeRequest: + rubric = case.metadata['rubric'] + attachments = case.metadata['image_attachments'] if rubric_needs_vision(rubric) else [] + prompt = build_judge_prompt(rubric, case.metadata['file_contents'], vision_used=bool(attachments)) + prompt += case.output_contract.instruction() + if not attachments: + return JudgeRequest( + messages=[ + ChatMessageSystem(content='You are an evaluation judge. Return JSON only.'), + ChatMessageUser(content=prompt), + ] + ) + content = [ContentText(text=prompt)] + for index, (name, url) in enumerate(attachments, start=1): + content += [ContentText(text=f'Image {index}: {name}'), ContentImage(image=url)] + return JudgeRequest( + messages=[ + ChatMessageSystem(content='You are an evaluation judge. Return JSON only.'), + ChatMessageUser(content=content), + ] ) - scorecard = evaluation['scorecard'] - return Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, + + def _reduce_verdicts(self, case_verdicts: List[CaseVerdict], context: JudgeContext) -> ReducedVerdict: + results = [] + for verdict in case_verdicts: + rubric = verdict.metadata['rubric'] + parsed = verdict.value + criteria = normalize_criteria(rubric) + returned = {item.index: item for item in parsed.criteria_results} + criteria_results = [{ + 'index': index, + 'criterion': criterion, + 'passed': bool(returned.get(index) and returned[index].passed), + 'reasoning': returned[index].reasoning if index in returned else '', + 'evidence': returned[index].evidence if index in returned else '', + } for index, criterion in enumerate(criteria)] + passed = bool(parsed.rubric_passed) and all(item['passed'] for item in criteria_results) + weight = float(rubric.get('weight') or 0) + results.append({ + 'index': verdict.metadata['index'], + 'rubric': rubric.get('rubric', ''), + 'weight': weight, + 'result': { + 'passed': passed, + 'score': weight if passed else 0.0, + 'criteria_count': len(criteria), + 'criteria_passed': sum(item['passed'] for item in criteria_results), + 'criteria_results': criteria_results, + 'overall_reasoning': parsed.overall_reasoning, + }, + }) + scorecard = build_scorecard(results) + return ReducedVerdict( value={ 'normalized_score': float(scorecard['normalized_score']), 'pass_rate': float(scorecard['pass_rate']), - # Raw sum of passed rubric weights: an intermediate judge value, reassigned to a - # diagnostic through BENCHMARK_METRIC_OVERRIDES. 'judge_score': float(scorecard['total_score']), }, metadata={ - 'judge_model': self.llm_judge.model_id, - 'task_id': task_state.metadata.get('task_id'), - 'artifact_dir': task_state.metadata.get('artifact_dir'), - 'output_files': task_state.metadata.get('output_files') or [], - 'max_score': float(scorecard['max_score']), - 'rubrics': evaluation['rubrics'], - 'error': evaluation.get('error'), + 'rubrics': results, + 'max_score': float(scorecard['max_score']) }, - main_score_name='normalized_score', ) + + def _finalize_score(self, score: Score, review, context) -> Score: + score.metadata.update({ + 'task_id': context.task_state.metadata.get('task_id'), + 'artifact_dir': context.task_state.metadata.get('artifact_dir'), + 'output_files': context.task_state.metadata.get('output_files') or [], + }) + return score diff --git a/evalscope/benchmarks/longmemeval/longmemeval_adapter.py b/evalscope/benchmarks/longmemeval/longmemeval_adapter.py index 09d29f2f6..83e2793f2 100644 --- a/evalscope/benchmarks/longmemeval/longmemeval_adapter.py +++ b/evalscope/benchmarks/longmemeval/longmemeval_adapter.py @@ -1,22 +1,41 @@ # Copyright (c) Alibaba, Inc. and its affiliates. import json from pathlib import Path -from typing import Any, Dict, List, Tuple, Union +from pydantic import BaseModel +from typing import Any, Dict, List, Literal, Tuple, Union from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import DatasetDict, DatasetHub, MemoryDataset, Sample, build_dataset_from_records from evalscope.api.evaluator import TaskState +from evalscope.api.judge import ( + CaseVerdict, + JudgeCase, + JudgeContext, + JudgeDefinition, + JudgeRequest, + OutputContract, + ReducedVerdict, +) from evalscope.api.messages import ChatMessageUser from evalscope.api.metric import AggScore, SampleScore, Score from evalscope.api.metric.semantics import MetricSelector from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger from .utils import QUESTION_TYPES, SUBSET_TO_FILE, build_generation_prompt, get_anscheck_prompt logger = get_logger() +# The official grader prompt ends with "Answer yes or no only." +class AnswerCheck(BaseModel): + reasoning: str = '' + verdict: Literal['yes', 'no'] + + +ANSCHECK_CONTRACT = OutputContract(schema_model=AnswerCheck) + + @register_benchmark( BenchmarkMeta( name='longmemeval', @@ -101,7 +120,7 @@ ) class LongMemEvalAdapter(DefaultDataAdapter): """Adapter for LongMemEval.""" - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, **kwargs) -> None: super().__init__(**kwargs) @@ -236,36 +255,37 @@ def record_to_sample(self, record: Dict[str, Any]) -> Sample: } return Sample(input=[ChatMessageUser(content=prompt)], target=str(record['answer']), metadata=metadata) - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - if not self.llm_judge: - raise ValueError('LongMemEval requires an initialized LLM judge.') - score = Score(extracted_prediction=filtered_prediction, prediction=original_prediction) - judge_prompt = get_anscheck_prompt( - task=task_state.metadata['question_type'], - question=task_state.metadata['question'], - answer=reference, - response=filtered_prediction, - abstention=task_state.metadata.get('is_abstention', False), + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + metadata = judge_context.task_state.metadata + prompt = get_anscheck_prompt( + task=metadata['question_type'], + question=metadata['question'], + answer=judge_context.reference, + response=judge_context.filtered_prediction, + abstention=metadata.get('is_abstention', False), + ) + case.output_contract.instruction() + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + return ReducedVerdict(value={'accuracy': 1.0 if case_verdicts[0].value.verdict == 'yes' else 0.0}) + + def finalize(score, review, judge_context) -> Score: + metadata = judge_context.task_state.metadata + score.metadata.update({ + 'question_type': metadata['question_type'], + 'is_abstention': metadata.get('is_abstention', False) + }) + return score + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='anscheck', output_contract=ANSCHECK_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='accuracy', + finalize=finalize ) - judge_response = self.llm_judge.judge(prompt=judge_prompt) - is_correct = 'yes' in judge_response.lower() - score.value = {'accuracy': 1.0 if is_correct else 0.0} - score.explanation = f'LLM judge: {judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id, - 'question_type': task_state.metadata['question_type'], - 'is_abstention': task_state.metadata.get('is_abstention', False), - } - score.main_score_name = 'accuracy' - return score def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]: valid_scores = [s for s in sample_scores if s.score and 'accuracy' in s.score.value] diff --git a/evalscope/benchmarks/longmemeval/utils.py b/evalscope/benchmarks/longmemeval/utils.py index 6fa35f05b..cdc6838a3 100644 --- a/evalscope/benchmarks/longmemeval/utils.py +++ b/evalscope/benchmarks/longmemeval/utils.py @@ -39,7 +39,7 @@ def get_anscheck_prompt(task: str, question: str, answer: str, response: str, ab 'yes if the model correctly identifies the question as unanswerable. The model could say that the ' 'information is incomplete, or some other information is given but the asked information is not.\n\n' 'Question: {}\n\nExplanation: {}\n\nModel Response: {}\n\nDoes the model correctly identify the ' - 'question as unanswerable? Answer yes or no only.' + 'question as unanswerable?' ) return template.format(question, answer, response) @@ -49,7 +49,7 @@ def get_anscheck_prompt(task: str, question: str, answer: str, response: str, ab 'response contains the correct answer. Otherwise, answer no. If the response is equivalent to the correct ' 'answer or contains all the intermediate steps to get the correct answer, you should also answer yes. If ' 'the response only contains a subset of the information required by the answer, answer no. \n\nQuestion: ' - '{}\n\nCorrect Answer: {}\n\nModel Response: {}\n\nIs the model response correct? Answer yes or no only.' + '{}\n\nCorrect Answer: {}\n\nModel Response: {}\n\nIs the model response correct?' ) return template.format(question, answer, response) @@ -62,7 +62,7 @@ def get_anscheck_prompt(task: str, question: str, answer: str, response: str, ab 'do not penalize off-by-one errors for the number of days. If the question asks for the number of ' 'days/weeks/months, etc., and the model makes off-by-one errors (e.g., predicting 19 days when the answer ' 'is 18), the model\'s response is still correct. \n\nQuestion: {}\n\nCorrect Answer: {}\n\nModel ' - 'Response: {}\n\nIs the model response correct? Answer yes or no only.' + 'Response: {}\n\nIs the model response correct?' ) return template.format(question, answer, response) @@ -72,7 +72,7 @@ def get_anscheck_prompt(task: str, question: str, answer: str, response: str, ab 'response contains the correct answer. Otherwise, answer no. If the response contains some previous ' 'information along with an updated answer, the response should be considered as correct as long as the ' 'updated answer is the required answer.\n\nQuestion: {}\n\nCorrect Answer: {}\n\nModel Response: {}\n\n' - 'Is the model response correct? Answer yes or no only.' + 'Is the model response correct?' ) return template.format(question, answer, response) @@ -82,7 +82,7 @@ def get_anscheck_prompt(task: str, question: str, answer: str, response: str, ab 'Please answer yes if the response satisfies the desired response. Otherwise, answer no. The model does ' 'not need to reflect all the points in the rubric. The response is correct as long as it recalls and ' 'utilizes the user\'s personal information correctly.\n\nQuestion: {}\n\nRubric: {}\n\nModel Response: ' - '{}\n\nIs the model response correct? Answer yes or no only.' + '{}\n\nIs the model response correct?' ) return template.format(question, answer, response) diff --git a/evalscope/benchmarks/math_verse/math_verse_adapter.py b/evalscope/benchmarks/math_verse/math_verse_adapter.py index 6146692e1..431ea34a2 100644 --- a/evalscope/benchmarks/math_verse/math_verse_adapter.py +++ b/evalscope/benchmarks/math_verse/math_verse_adapter.py @@ -5,7 +5,7 @@ from evalscope.api.dataset import Sample from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.io_utils import bytes_to_base64 from evalscope.utils.logger import get_logger @@ -76,7 +76,7 @@ ) class MathVerseAdapter(VisionLanguageAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_DEFAULT def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/evalscope/benchmarks/mcp_atlas/mcp_atlas_adapter.py b/evalscope/benchmarks/mcp_atlas/mcp_atlas_adapter.py index 30a30111d..a9c8d016b 100644 --- a/evalscope/benchmarks/mcp_atlas/mcp_atlas_adapter.py +++ b/evalscope/benchmarks/mcp_atlas/mcp_atlas_adapter.py @@ -2,17 +2,19 @@ import asyncio import json -from typing import Any, Dict, List, Optional +from pydantic import BaseModel, ConfigDict +from typing import Any, Dict, List, Literal, Optional from evalscope.api.agent import AgentEnvironment from evalscope.api.benchmark import BenchmarkMeta from evalscope.api.benchmark.adapters import AgentLoopAdapter from evalscope.api.dataset import Sample -from evalscope.api.evaluator import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict +from evalscope.api.messages import ChatMessageUser from evalscope.api.metric import AggScore, SampleScore, Score from evalscope.api.registry import register_benchmark from evalscope.api.tool import ToolCall, ToolInfo -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger from .metadata import DATASET_ID, DEFAULT_MCP_SERVER_URL, DEFAULT_SYSTEM_PROMPT, DESCRIPTION, EXTRA_PARAMS from .utils import ( @@ -23,7 +25,6 @@ extract_required_servers, field, mcp_tool_to_tool_info, - parse_claim_judge_response, parse_enabled_tools, server_unavailable_message, tool_name_to_server, @@ -32,6 +33,18 @@ logger = get_logger() +class ClaimVerdict(BaseModel): + model_config = ConfigDict(extra='ignore') + + coverage_outcome: Literal['fulfilled', 'partially_fulfilled', 'not_fulfilled'] + justification: str = '' + + +# The claim_judge_prompt already asks for this JSON, so no instruction() is appended. +CLAIM_CONTRACT = OutputContract(schema_model=ClaimVerdict) +_OUTCOME_SCORE = {'fulfilled': 1.0, 'partially_fulfilled': 0.5, 'not_fulfilled': 0.0} + + @register_benchmark( BenchmarkMeta( name='mcp_atlas', @@ -51,7 +64,7 @@ ) class MCPAtlasAdapter(AgentLoopAdapter): """EvalScope-native MCP-Atlas adapter using MCP-Atlas agent-environment.""" - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY strategy_name = 'function_calling' max_steps_default = 100 @@ -122,54 +135,51 @@ def build_tools(self, sample: Sample) -> Dict[str, Any]: def build_environment(self, sample: Sample) -> Optional[AgentEnvironment]: return None - def match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - raise ValueError('MCP-Atlas requires LLM judge scoring; set judge_strategy to auto/llm with judge_model_args.') + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + cases = [ + JudgeCase(case_id=f'claim_{index}', output_contract=CLAIM_CONTRACT, metadata={'claim': claim}) + for index, claim in enumerate(extract_claims(context.reference)) + ] - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - claims = extract_claims(reference) - claim_results = [self._judge_claim(claim, filtered_prediction) for claim in claims] - total = len(claim_results) - coverage_score = sum(result['score'] for result in claim_results) / total if total else 0.0 - passed = coverage_score >= self.pass_threshold + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + return JudgeRequest( + messages=[ + ChatMessageUser( + content=claim_judge_prompt(case.metadata['claim'], judge_context.filtered_prediction) + ) + ] + ) - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - value={ - 'coverage_score': coverage_score, - 'pass': 1.0 if passed else 0.0, - }, - metadata={ - 'source': 'mcp_atlas_claim_judge', - 'pass_threshold': self.pass_threshold, - 'claims': claim_results, - 'total_claims': total, - 'fully_covered_claims': sum(1 for result in claim_results if result['score'] == 1.0), - 'partially_covered_claims': sum(1 for result in claim_results if result['score'] == 0.5), - 'model': self.llm_judge.model_id, - }, - main_score_name='coverage_score', - ) - return score + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + scores = [_OUTCOME_SCORE[verdict.value.coverage_outcome] for verdict in case_verdicts] + total = len(scores) + coverage_score = sum(scores) / total if total else 0.0 + return ReducedVerdict( + value={ + 'coverage_score': coverage_score, + 'pass': 1.0 if coverage_score >= self.pass_threshold else 0.0 + }, + metadata={ + 'pass_threshold': self.pass_threshold, + 'total_claims': total, + 'fully_covered_claims': scores.count(1.0), + 'partially_covered_claims': scores.count(0.5) + }, + ) + + return JudgeDefinition.workflow(cases=cases, request=request, reduce=reduce, main_score_name='coverage_score') def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]: if not sample_scores: return [] - coverage_values = [float(sample_score.score.value.get('coverage_score', 0.0)) for sample_score in sample_scores] - pass_values = [float(sample_score.score.value.get('pass', 0.0)) for sample_score in sample_scores] - sample_ids = [sample_score.sample_id for sample_score in sample_scores] + # A sample whose judge review was unusable carries an empty value dict; it is excluded from + # the means rather than counted as 0. + scored = [sample_score for sample_score in sample_scores if sample_score.score.value] + if not scored: + return [] + coverage_values = [float(sample_score.score.value.get('coverage_score', 0.0)) for sample_score in scored] + pass_values = [float(sample_score.score.value.get('pass', 0.0)) for sample_score in scored] + sample_ids = [sample_score.sample_id for sample_score in scored] return [ AggScore( metric_name='coverage_score', @@ -246,21 +256,3 @@ async def _handler(call: ToolCall, env: Optional[AgentEnvironment]) -> str: return server_unavailable_message(exc.server_name, exc.message) return _handler - - def _judge_claim(self, claim: str, response: str) -> Dict[str, Any]: - prompt = claim_judge_prompt(claim, response) - judge_response = self.llm_judge.judge(prompt) - outcome, justification, confidence = parse_claim_judge_response(judge_response) - score = { - 'fulfilled': 1.0, - 'partially_fulfilled': 0.5, - 'not_fulfilled': 0.0, - }.get(outcome, 0.0) - return { - 'claim': claim, - 'coverage_outcome': outcome, - 'score': score, - 'justification': justification, - 'confidence': confidence, - 'raw_judge_response': judge_response, - } diff --git a/evalscope/benchmarks/mcp_atlas/utils.py b/evalscope/benchmarks/mcp_atlas/utils.py index c2db7d030..be9b7f0ad 100644 --- a/evalscope/benchmarks/mcp_atlas/utils.py +++ b/evalscope/benchmarks/mcp_atlas/utils.py @@ -179,24 +179,6 @@ def mcp_tool_to_tool_info(raw_tool: Dict[str, Any]) -> ToolInfo: ) -def parse_claim_judge_response(response: Any) -> Tuple[str, str, float]: - if not isinstance(response, str): - return 'not_fulfilled', 'Judge response was not text.', 0.0 - text = strip_json_fence(response) - try: - parsed = json.loads(text) - outcome = str(parsed.get('coverage_outcome') or parsed.get('outcome') or 'not_fulfilled') - justification = str(parsed.get('justification') or parsed.get('reason') or '') - confidence = parse_confidence(parsed.get('confidence_level', parsed.get('confidence', 0.0))) - return outcome, justification, confidence - except Exception: - lowered = text.lower() - for outcome in ['partially_fulfilled', 'not_fulfilled', 'fulfilled']: - if outcome in lowered: - return outcome, text, 0.0 - return 'not_fulfilled', text or 'Unable to parse judge response.', 0.0 - - def strip_json_fence(text: str) -> str: stripped = text.strip() if not stripped.startswith('```'): diff --git a/evalscope/benchmarks/mia_bench/mia_bench_adapter.py b/evalscope/benchmarks/mia_bench/mia_bench_adapter.py index c18307c50..9ceb92fbb 100644 --- a/evalscope/benchmarks/mia_bench/mia_bench_adapter.py +++ b/evalscope/benchmarks/mia_bench/mia_bench_adapter.py @@ -1,19 +1,29 @@ # flake8: noqa: E501 -from typing import Any, Dict, List +from pydantic import BaseModel, Field, create_model +from typing import Any, Dict, List, Type from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText from evalscope.api.metric.scorer import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.io_utils import bytes_to_base64 from evalscope.utils.logger import get_logger logger = get_logger() +def _build_grade_schema(component_weight: List[int]) -> Type[BaseModel]: + """A per-sample schema: ``component_i`` bounded by that component's weight, plus reasoning.""" + fields: Dict[str, Any] = {'reasoning': (str, Field(default=''))} + for i, weight in enumerate(component_weight): + fields[f'component_{i + 1}'] = (float, Field(ge=0.0, le=float(weight))) + return create_model('MiaGrade', **fields) + + @register_benchmark( BenchmarkMeta( name='mia_bench', @@ -44,8 +54,8 @@ - Default evaluation uses the **test** split (400 samples) - Primary metric: **judge_score** (mean of per-sample normalized 0–1 total scores) -- Requires a capable LLM judge (e.g., GPT-4o, Qwen-Max) configured via `judge_model_args` -- Judge strategy should be set to `JudgeStrategy.LLM` +- Requires a capable LLM judge (e.g., GPT-4o, Qwen-Max) configured through `judge.models` +- Judge strategy should be set to `judge.strategy='llm'` """, metric_list=['judge_score'], eval_split='test', @@ -58,7 +68,7 @@ class MIABenchAdapter(VisionLanguageAdapter): Each sample is scored by an LLM judge that evaluates whether the model's response satisfies each weighted instruction component. """ - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, **kwargs): super().__init__(**kwargs) @@ -88,69 +98,47 @@ def record_to_sample(self, record: Dict[str, Any]) -> Sample: }, ) - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - """ - Use an LLM judge to score the model's response against MIA-Bench components. - - The judge is asked to rate each instruction component separately and - provide a total score. Scores are parsed and normalised to [0, 1]. - """ - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - ) - - metadata = task_state.metadata or {} - instruction: str = metadata.get('instruction', task_state.input_text) - components: List[str] = metadata.get('components', []) - component_weight: List[int] = metadata.get('component_weight', []) - component_type: List[str] = metadata.get('component_type', []) - - if not components: + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + metadata = context.task_state.metadata or {} + if not metadata.get('components', []): logger.warning('No components found in sample metadata; assigning zero score.') - score.value = {'judge_score': 0.0} - score.main_score_name = 'judge_score' - return score - - # Build judge prompt - from .utils import generate_mia_judge_prompt, parse_mia_score - - judge_prompt = generate_mia_judge_prompt( - instruction=instruction, - components=components, - component_weight=component_weight, - response=filtered_prediction or original_prediction, + return JudgeDefinition.skip( + Score( + extracted_prediction=context.filtered_prediction, + prediction=context.original_prediction, + value={'judge_score': 0.0}, + main_score_name='judge_score' + ), + reason='missing_components', + ) + contract = OutputContract(schema_model=_build_grade_schema(metadata.get('component_weight', []))) + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + from .utils import generate_mia_judge_prompt + current = judge_context.task_state.metadata or {} + prompt = generate_mia_judge_prompt( + instruction=current.get('instruction', judge_context.task_state.input_text), + components=current.get('components', []), + component_weight=current.get('component_weight', []), + response=judge_context.filtered_prediction or judge_context.original_prediction, + ) + case.output_contract.instruction() + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + grade, current = case_verdicts[0].value, judge_context.task_state.metadata or {} + values, raw_sum = {}, 0.0 + weights = current.get('component_weight', []) + for index, component_type in enumerate(current.get('component_type', [])): + raw = float(getattr(grade, f'component_{index + 1}')) + weight = weights[index] if index < len(weights) else 1 + values[f'component_{index + 1}_{component_type}'] = raw / weight if weight else 0.0 + raw_sum += raw + values['judge_score'] = raw_sum / sum(weights) if sum(weights) else 0.0 + return ReducedVerdict(value=values) + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='grade', output_contract=contract)], + request=request, + reduce=reduce, + main_score_name='judge_score' ) - - # Call LLM judge - judge_response = self.llm_judge.judge(judge_prompt) - - # Parse scores - score_dict = parse_mia_score(component_type, judge_response) - - # Build score.value: the judge score + per-component scores with unique keys. The parser - # still reports its own `total_score` key; only the emitted metric name is canonical. - score_value: Dict[str, float] = {'judge_score': score_dict.get('total_score', 0.0)} - for i, ctype in enumerate(component_type): - key = f'component_{i + 1}_{ctype}' - score_value[key] = score_dict.get(ctype, 0.0) - - score.value = score_value - score.main_score_name = 'judge_score' - score.explanation = f'LLM judge response:\n{judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id, - 'components': components, - 'component_weight': component_weight, - 'component_type': component_type, - } - - return score diff --git a/evalscope/benchmarks/mia_bench/utils.py b/evalscope/benchmarks/mia_bench/utils.py index 6bf4aa0e8..da46bf748 100644 --- a/evalscope/benchmarks/mia_bench/utils.py +++ b/evalscope/benchmarks/mia_bench/utils.py @@ -1,5 +1,4 @@ # flake8: noqa: E501 -import re from typing import Dict, List from evalscope.utils.logger import get_logger @@ -39,14 +38,6 @@ def _build_score_text(components: List[str], weights: List[int]) -> str: return 'Components are each worth: ' + ', '.join(pieces) + ' scores.' -def _build_score_format(components: List[str], weights: List[int]) -> str: - """Build the expected output format string.""" - parts = [f'score of component {i + 1}: x{i + 1}/{weights[i]}' for i in range(len(components))] - total = sum(weights) - parts.append(f'total score: z/{total}') - return ', '.join(parts) - - def generate_mia_judge_prompt( instruction: str, components: List[str], @@ -56,11 +47,11 @@ def generate_mia_judge_prompt( """ Generate the LLM judge evaluation prompt for MIA-Bench. - Equivalent to reference.py generate_prompt() but decoupled from the doc dict. + Equivalent to reference.py generate_prompt() but decoupled from the doc dict. The reply format + is declared by the caller's OutputContract; ``component_i`` maps to the i-th component here. """ components_text = _build_components_text(components) score_text = _build_score_text(components, component_weight) - score_format = _build_score_format(components, component_weight) total = sum(component_weight) prompt = ( @@ -70,101 +61,5 @@ def generate_mia_judge_prompt( f" The response is:' {response}'" f' You need to score the response and be strict. The total score ranges from 0 to {total}, depending on if the response follows the instruction. ' f'{score_text}' - f' List scores of each component, and the total score in one sentence in this EXACT format: {score_format}.' - f' Use only numbers for score values. Do not use markdown formatting or asterisks. Then explain your reasons.' ) return prompt - - -def parse_mia_score(component_type: List[str], raw_score: str) -> Dict[str, float]: - """ - Parse the LLM judge raw score string into a score dict. - - Equivalent to reference.py process_rawscore() but standalone. - - Returns: - dict mapping component_type entries to normalized [0,1] scores, - plus 'total_score' key. - """ - score_dict: Dict[str, float] = {} - - try: - if not component_type or not isinstance(component_type, list): - logger.error(f'Invalid component_type: {component_type}') - return {'total_score': 0.0} - - if not raw_score or not isinstance(raw_score, str): - logger.error(f'Invalid raw_score: {raw_score}') - for comp in component_type: - score_dict[comp] = 0.0 - score_dict['total_score'] = 0.0 - return score_dict - - # Pattern: "score of component X: Y/Z" or "component X: Y/Z" - component_pattern = r'(?:score\s+of\s+)?component\s+(\d+)\s*:\s*(\d+)\s*/\s*(\d+)' - total_pattern = r'total\s+score\s*:\s*(\d+)\s*/\s*(\d+)' - - # Extract per-component scores - try: - component_matches = re.findall(component_pattern, raw_score, re.IGNORECASE) - for match in component_matches: - try: - component_num = int(match[0]) - 1 # 0-based index - if 0 <= component_num < len(component_type): - numerator = int(match[1].strip()) - denominator = int(match[2].strip()) - score = numerator / denominator if denominator != 0 else 0.0 - score = max(0.0, min(1.0, score)) - score_dict[component_type[component_num]] = score - else: - logger.warning( - f'Component number {component_num + 1} out of range for {len(component_type)} components' - ) - except (ValueError, IndexError) as e: - logger.warning(f'Error parsing component match {match}: {e}') - continue - except Exception as e: - logger.error(f'Error in component pattern matching: {e}') - - # Extract total score - try: - total_match = re.search(total_pattern, raw_score, re.IGNORECASE) - if total_match: - total_numerator = int(total_match.group(1).strip()) - total_denominator = int(total_match.group(2).strip()) - total_score = total_numerator / total_denominator if total_denominator != 0 else 0.0 - total_score = max(0.0, min(1.0, total_score)) - score_dict['total_score'] = total_score - else: - # Fallback: average of component scores - if score_dict: - score_dict['total_score'] = sum(score_dict.values()) / len(score_dict) - else: - score_dict['total_score'] = 0.0 - except Exception as e: - logger.error(f'Error parsing total score: {e}') - score_dict['total_score'] = 0.0 - - # Ensure all components have a score entry - for comp in component_type: - if comp not in score_dict: - logger.warning(f'Missing score for component: {comp}') - score_dict[comp] = 0.0 - - if 'total_score' not in score_dict: - component_values = [v for v in score_dict.values()] - score_dict['total_score'] = sum(component_values) / len(component_values) if component_values else 0.0 - - except Exception as e: - logger.error(f'Unexpected error in parse_mia_score: {e}') - score_dict = {comp: 0.0 for comp in (component_type or [])} - score_dict['total_score'] = 0.0 - - # Final validation: clamp all values to [0, 1] - for key in list(score_dict.keys()): - val = score_dict[key] - if not isinstance(val, (int, float)) or val < 0 or val > 1: - logger.warning(f'Invalid score for {key}: {val}, setting to 0') - score_dict[key] = 0.0 - - return score_dict diff --git a/evalscope/benchmarks/minerva_math/minerva_math_adapter.py b/evalscope/benchmarks/minerva_math/minerva_math_adapter.py index ce997e23e..ffde09742 100644 --- a/evalscope/benchmarks/minerva_math/minerva_math_adapter.py +++ b/evalscope/benchmarks/minerva_math/minerva_math_adapter.py @@ -3,7 +3,7 @@ from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import Sample from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger logger = get_logger() @@ -55,7 +55,7 @@ ) class MinervaMathAdapter(DefaultDataAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_DEFAULT def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/evalscope/benchmarks/needle_haystack/needle_haystack_adapter.py b/evalscope/benchmarks/needle_haystack/needle_haystack_adapter.py index d5dc28d8d..8093f5d5a 100644 --- a/evalscope/benchmarks/needle_haystack/needle_haystack_adapter.py +++ b/evalscope/benchmarks/needle_haystack/needle_haystack_adapter.py @@ -1,14 +1,17 @@ import os from itertools import product +from pydantic import BaseModel, Field from tqdm import tqdm -from typing import TYPE_CHECKING, Any, Dict, List, Union +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Union from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import DatasetDict, Sample, build_dataset_from_records, resolve_snapshot_or_local_path from evalscope.api.evaluator import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict +from evalscope.api.messages import ChatMessageSystem, ChatMessageUser from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.import_utils import check_import from evalscope.utils.logger import get_logger @@ -17,6 +20,14 @@ logger = get_logger() + +class Rating(BaseModel): + reasoning: str = '' + verdict: float = Field(ge=0.0, le=10.0) + + +SCORE_CONTRACT = OutputContract(schema_model=Rating) + PROMPT_TEMPLATE = """Please read the following text and answer the question below. @@ -128,7 +139,7 @@ ) class NeedleHaystackAdapter(DefaultDataAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_DEFAULT def __init__(self, **kwargs): super().__init__(**kwargs) @@ -387,40 +398,35 @@ def match_score( return score - def llm_match_score( - self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState - ) -> Score: - """Use LLM as a judge to evaluate the predicted answer against the gold answer.""" - from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE, parse_score - - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - ) - - # Get metadata from task state - context_length = task_state.metadata.get('context_length', 0) - depth_percent = task_state.metadata.get('depth_percent', 0) - question = task_state.input_text - - # Get grading response - prompt = ORM_USER_TEMPLATE.format(question=question, gold=reference, pred=filtered_prediction) - orm_response = self.llm_judge.judge(prompt, system_prompt=GENERAL_ORM_PROMPT) + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + metric_name = self._metric_name(context.task_state) + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE + prompt = ORM_USER_TEMPLATE.format( + question=judge_context.task_state.input_text, + gold=judge_context.reference, + pred=judge_context.filtered_prediction, + ) + case.output_contract.instruction() + return JudgeRequest( + messages=[ChatMessageSystem(content=GENERAL_ORM_PROMPT), + ChatMessageUser(content=prompt)] + ) - # Parse grading score with regex, [[score]] - accuracy = parse_score(orm_response) if orm_response else 0.0 + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + return ReducedVerdict(value={metric_name: case_verdicts[0].value.verdict / 10.0}) - metric_name = f'Context#{context_length} Depth#{depth_percent}' - score.value = {metric_name: accuracy} - score.explanation = f'LLM judge: {orm_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': getattr(self, 'judge_strategy', 'default'), - 'model': self.llm_judge.model_id if hasattr(self.llm_judge, 'model_id') else 'unknown' - } - score.main_score_name = metric_name + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='score', output_contract=SCORE_CONTRACT)], + request=request, + reduce=reduce, + main_score_name=metric_name + ) - return score + @staticmethod + def _metric_name(task_state: TaskState) -> str: + metadata = task_state.metadata or {} + return f"Context#{metadata.get('context_length', 0)} Depth#{metadata.get('depth_percent', 0)}" def _on_generate_report_end(self, report: 'Report', output_dir: str, **kwargs): try: diff --git a/evalscope/benchmarks/needle_haystack/utils.py b/evalscope/benchmarks/needle_haystack/utils.py index 640f30f4d..49fd5aaf4 100644 --- a/evalscope/benchmarks/needle_haystack/utils.py +++ b/evalscope/benchmarks/needle_haystack/utils.py @@ -21,19 +21,6 @@ def lower(text): return white_space_fix(remove_articles(remove_punc(lower(s)))) -def parse_score(score_str: str) -> int: - """ - Parses a score string and returns an integer score. - The score should be in the format [[score]]. - """ - score_match = re.search(r'\[\[(\d+)\]\]', score_str) - if score_match: - score = int(score_match.group(1)) - return score / 10.0 - else: - return 0.0 - - def draw_score_chat(pivot_table, outpath, show_score=False): import matplotlib.pyplot as plt import seaborn as sns @@ -68,8 +55,7 @@ def draw_score_chat(pivot_table, outpath, show_score=False): Score 5: The answer has moderate relevance but contains inaccuracies. Score 7: The answer aligns with the reference but has minor omissions. Score 10: The answer is completely accurate and aligns perfectly with the reference. - -Only respond with a numberical score with formatted as [[score]].""" # noqa: E501 +""" # noqa: E501 ORM_USER_TEMPLATE = """ Question: {question} diff --git a/evalscope/benchmarks/perception_bench/perception_bench_adapter.py b/evalscope/benchmarks/perception_bench/perception_bench_adapter.py index bcc184c11..4c092b453 100644 --- a/evalscope/benchmarks/perception_bench/perception_bench_adapter.py +++ b/evalscope/benchmarks/perception_bench/perception_bench_adapter.py @@ -1,19 +1,29 @@ # flake8: noqa: E501 import base64 +from pydantic import BaseModel from typing import Any, Dict, List, Optional from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict from evalscope.api.messages import ChatMessageSystem, ChatMessageUser, Content from evalscope.api.metric.scorer import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger -from .utils import build_judge_prompt, parse_judge_verdict +from .utils import build_judge_prompt logger = get_logger() + +class Judgment(BaseModel): + reasoning: str = '' + verdict: bool + + +JUDGMENT_CONTRACT = OutputContract(schema_model=Judgment) + DESCRIPTION = """ ## Overview @@ -52,7 +62,7 @@ (`[reason]` / `[judge] True|False`); the paper uses GPT-oss-120B, whose agreement with human judgment is 99.7% on a 300-sample audit - Empty or failed generations are scored 0 without invoking the judge -- Requires `judge_model_args` configuration for the LLM judge +- Requires an LLM judge configured through `judge.models` - The dataset embeds images as base64 data URIs (~1.6 GB download on first use) """ @@ -89,7 +99,7 @@ class PerceptionBenchAdapter(VisionLanguageAdapter): Interleaves the question with its images following the official `<|image_N|>` placeholder convention and scores free-form answers with the official LLM judge. """ - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, **kwargs): super().__init__(**kwargs) @@ -154,40 +164,37 @@ def _data_uri_to_base64(self, image: str) -> Optional[str]: logger.warning(f'Failed to decode image data URI: {e}') return None - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - """Score a prediction with the official PerceptionBench teacher-grading judge.""" - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - ) - - if not original_prediction.strip(): - # Mirrors the official evaluator: unanswered items score 0 without a judge call. - score.value = {'acc': 0.0} - score.explanation = 'failed to obtain answer' - return score - - metadata = task_state.metadata or {} - prompt = build_judge_prompt( - question=metadata.get('problem', task_state.input_text), - prediction=original_prediction, - reference=reference, + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + if not context.original_prediction.strip(): + return JudgeDefinition.skip( + Score( + extracted_prediction=context.filtered_prediction, + prediction=context.original_prediction, + value={'acc': 0.0}, + main_score_name='acc', + explanation='failed to obtain answer' + ), + reason='empty_model_output', + ) + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + metadata = judge_context.task_state.metadata or {} + prompt = build_judge_prompt( + question=metadata.get('problem', judge_context.task_state.input_text), + prediction=judge_context.original_prediction, + reference=judge_context.reference, + ) + case.output_contract.instruction() + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + judgment = case_verdicts[0].value + return ReducedVerdict( + value={'acc': 1.0 if judgment.verdict else 0.0}, metadata={'judge_reason': judgment.reasoning[:200]} + ) + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='judgment', output_contract=JUDGMENT_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='acc' ) - judge_response = self.llm_judge.judge(prompt) - judge_score, judge_reason = parse_judge_verdict(judge_response) - - score.value = {'acc': judge_score} - score.explanation = f'LLM judge: {judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id, - 'judge_reason': judge_reason, - } - return score diff --git a/evalscope/benchmarks/perception_bench/utils.py b/evalscope/benchmarks/perception_bench/utils.py index 08cc9da70..56a08d69e 100644 --- a/evalscope/benchmarks/perception_bench/utils.py +++ b/evalscope/benchmarks/perception_bench/utils.py @@ -5,26 +5,15 @@ https://github.com/MoonshotAI/PerceptionBench (``eval/judge_prompt.txt`` and ``eval/eval.py``). """ import re -from typing import Tuple - -# The judge must answer with a standalone True/False inside its [judge] section. -VERDICT_PATTERN = re.compile(r'\b(true|false)\b', re.IGNORECASE) JUDGE_TEMPLATE = """Please act as a professional teacher and grade the student's answer. Below are the question, the student's answer, and the reference answer. Based on the question and the reference answer, analyze the student's answer and judge whether it correctly answers the question. I will provide several examples to help you understand how to judge. -========== [Output Format] ========== -Return your judgment in the following format. The first field [reason] gives the reason for your judgment; the second field [judge] gives your verdict as a single boolean, i.e., True or False. Make sure your output ends with True or False. -[reason] - -[judge] -False - ========== [Notice] ========== 0. You only need to compare the consistency between the reference answer and the student's answer, focusing especially on the content after summarizing phrases such as "Final answer:". The reference answer is absolutely correct; judge solely by whether the student's final result matches it, even if the solution process is correct. -1. If the question contains multiple sub-questions, output the student's answer, the reference answer, and the consistency for each sub-question in [reason]; judge correct only when all sub-questions are consistent. +1. If the question contains multiple sub-questions, output the student's answer, the reference answer, and the consistency for each sub-question in "reasoning"; judge correct only when all sub-questions are consistent. 2. For multiple-answer questions, the student's answer must contain all correct answers without any extra ones. Pay special attention when the reference answer contains "or": decide whether all answers must be given based on the specific question. -3. If you believe the student's answer equals the reference answer after simplification, you must provide the complete simplification process in [reason] until the two are equal; otherwise it cannot be judged correct. You may not vaguely claim that "they can be made equal". -4. For numerical answers, integers or values with at most 4 significant figures must match exactly; values with more than 4 significant figures must match within 4 significant figures. In [reason], convert both to standard scientific notation, first compare the order of magnitude, then compare the first 4 significant figures. If the units differ, convert to a common unit first. +3. If you believe the student's answer equals the reference answer after simplification, you must provide the complete simplification process in "reasoning" until the two are equal; otherwise it cannot be judged correct. You may not vaguely claim that "they can be made equal". +4. For numerical answers, integers or values with at most 4 significant figures must match exactly; values with more than 4 significant figures must match within 4 significant figures. In "reasoning", convert both to standard scientific notation, first compare the order of magnitude, then compare the first 4 significant figures. If the units differ, convert to a common unit first. 5. For English writing questions, if the student does not answer in English, judge it incorrect. 6. For physics, chemistry, and biology questions, if the reference answer contains a technical term, the student's answer must contain that exact term; synonyms are not accepted. 7. When the question asks to explain a term, redundant explanation is not penalized, but missing key points must be judged incorrect. @@ -33,70 +22,46 @@ ========== [Examples] ========== ========== [Example 1.1] ========== Question: omitted Student answer: 34 Reference answer: (1) 12; (2) 34 -[reason] The student answer 34 only addresses the second sub-question and leaves the first unanswered. -[judge] False - +{"reasoning": "The student answer 34 only addresses the second sub-question and leaves the first unanswered.", "verdict": false} ========== [Example 1.2] ========== Question: omitted Student answer: 13; 34 Reference answer: (1) 12; (2) 34 -[reason] Sub-question 1: student 13 vs. reference 12, inconsistent. Sub-question 2: student 34 vs. reference 34, consistent. Overall, incorrect. -[judge] False - +{"reasoning": "Sub-question 1: student 13 vs. reference 12, inconsistent. Sub-question 2: student 34 vs. reference 34, consistent. Overall, incorrect.", "verdict": false} ========== [Example 2.1] ========== Question: omitted Student answer: A Reference answer: ABC -[reason] The student answer A is incomplete; the correct answer is ABC. -[judge] False - +{"reasoning": "The student answer A is incomplete; the correct answer is ABC.", "verdict": false} ========== [Example 2.2] ========== Question: omitted Student answer: ABC Reference answer: AC -[reason] The student answer ABC includes an extra B; the correct answer is AC. -[judge] False - +{"reasoning": "The student answer ABC includes an extra B; the correct answer is AC.", "verdict": false} ========== [Example 3.1] ========== Question: omitted Student answer: y''(0) = 13e^2 - e Reference answer: y''(0) = 12e^2 - 1 -[reason] The student answer differs in form from the reference and cannot be made equal through simplification. -[judge] False - +{"reasoning": "The student answer differs in form from the reference and cannot be made equal through simplification.", "verdict": false} ========== [Example 3.2] ========== Question: omitted Student answer: y''(0) = 12e^2 - e Reference answer: y''(0) = (12e - 1)e -[reason] 12e^2 - e = (12e - 1)e, which is equivalent to the reference answer. -[judge] True - +{"reasoning": "12e^2 - e = (12e - 1)e, which is equivalent to the reference answer.", "verdict": true} ========== [Example 4.1] ========== Question: omitted Student answer: 349 Reference answer: 342 -[reason] Integer answers must match exactly; 349 differs from 342. -[judge] False - +{"reasoning": "Integer answers must match exactly; 349 differs from 342.", "verdict": false} ========== [Example 4.2] ========== Question: omitted Student answer: 0.325 Reference answer: 0.618 -[reason] In scientific notation, 0.325 = 3.250e-1 and 0.618 = 6.180e-1; the exponents match but the first significant figure differs. -[judge] False - +{"reasoning": "In scientific notation, 0.325 = 3.250e-1 and 0.618 = 6.180e-1; the exponents match but the first significant figure differs.", "verdict": false} ========== [Example 4.3] ========== Question: omitted Student answer: 48.67 Reference answer: 48.675 -[reason] 48.67 = 4.867e1 and 48.675 = 4.868e1 (to 4 significant figures); the exponents match but the 4th significant figure differs. -[judge] False - +{"reasoning": "48.67 = 4.867e1 and 48.675 = 4.868e1 (to 4 significant figures); the exponents match but the 4th significant figure differs.", "verdict": false} ========== [Example 4.4] ========== Question: omitted Student answer: 4.85e4 Reference answer: 485 -[reason] 4.85e4 vs. 485 = 4.85e2; the two differ in order of magnitude. -[judge] False - +{"reasoning": "4.85e4 vs. 485 = 4.85e2; the two differ in order of magnitude.", "verdict": false} ========== [Example 8.1] ========== Question: A. 11 B. 22 C. 33 D. 44 Reference answer: A. 11 Student answer: C. 11 -[reason] The student option C differs from the reference option A. -[judge] False - +{"reasoning": "The student option C differs from the reference option A.", "verdict": false} ========== [Example 8.2] ========== Question: A. 11 B. 22 C. 33 D. 44 Reference answer: B Student answer: C -[reason] The student option C differs from the reference option B. -[judge] False - +{"reasoning": "The student option C differs from the reference option B.", "verdict": false} ========== [Notice] ========== 0. You only need to compare the consistency between the reference answer and the student's answer, focusing especially on the content after summarizing phrases such as "Final answer:". The reference answer is absolutely correct; judge solely by whether the student's final result matches it, even if the solution process is correct. -1. If the question contains multiple sub-questions, output the student's answer, the reference answer, and the consistency for each sub-question in [reason]; judge correct only when all sub-questions are consistent. +1. If the question contains multiple sub-questions, output the student's answer, the reference answer, and the consistency for each sub-question in "reasoning"; judge correct only when all sub-questions are consistent. 2. For multiple-answer questions, the student's answer must contain all correct answers without any extra ones. Pay special attention when the reference answer contains "or": decide whether all answers must be given based on the specific question. -3. If you believe the student's answer equals the reference answer after simplification, you must provide the complete simplification process in [reason] until the two are equal; otherwise it cannot be judged correct. You may not vaguely claim that "they can be made equal". -4. For numerical answers, integers or values with at most 4 significant figures must match exactly; values with more than 4 significant figures must match within 4 significant figures. In [reason], convert both to standard scientific notation, first compare the order of magnitude, then compare the first 4 significant figures. If the units differ, convert to a common unit first. +3. If you believe the student's answer equals the reference answer after simplification, you must provide the complete simplification process in "reasoning" until the two are equal; otherwise it cannot be judged correct. You may not vaguely claim that "they can be made equal". +4. For numerical answers, integers or values with at most 4 significant figures must match exactly; values with more than 4 significant figures must match within 4 significant figures. In "reasoning", convert both to standard scientific notation, first compare the order of magnitude, then compare the first 4 significant figures. If the units differ, convert to a common unit first. 5. For English writing questions, if the student does not answer in English, judge it incorrect. 6. For physics, chemistry, and biology questions, if the reference answer contains a technical term, the student's answer must contain that exact term; synonyms are not accepted. 7. When the question asks to explain a term, redundant explanation is not penalized, but missing key points must be judged incorrect. @@ -135,26 +100,3 @@ def build_judge_prompt(question: str, prediction: str, reference: str) -> str: prompt = JUDGE_TEMPLATE.replace('{problem}', normalize_escape(question).strip()) prompt = prompt.replace('{reference_answer}', normalize_escape(reference).strip()) return prompt.replace('{assistant_answer}', normalize_escape(prediction).strip()) - - -def parse_judge_verdict(judge_response: str) -> Tuple[float, str]: - """Parse the judge response into a strict 0/1 score plus its justification. - - The judge is required to emit both a ``[reason]`` and a ``[judge]`` section; - responses missing either section are counted as incorrect, as in the - official evaluator. Within the ``[judge]`` section only a standalone boolean - token is accepted, so prose such as ``'untrue'`` or a trailing remark cannot - flip the verdict; a section without any boolean token scores 0. - - Returns: - Tuple[float, str]: The 0/1 score and the (truncated) reason. - """ - response = str(judge_response).strip() - if '[reason]' not in response or '[judge]' not in response: - return 0.0, 'No [reason] or [judge] in output' - reason = response.split('[judge]')[0].split('[reason]')[-1].strip() - verdict = response.split('[judge]')[-1].strip() - match = VERDICT_PATTERN.search(verdict) - if match is None: - return 0.0, reason[:200] or 'No boolean verdict in [judge] section' - return (1.0 if match.group(1).lower() == 'true' else 0.0), reason[:200] diff --git a/evalscope/benchmarks/phyx/phyx_adapter.py b/evalscope/benchmarks/phyx/phyx_adapter.py index fed46b760..2fd8dad64 100644 --- a/evalscope/benchmarks/phyx/phyx_adapter.py +++ b/evalscope/benchmarks/phyx/phyx_adapter.py @@ -11,12 +11,14 @@ resolve_snapshot_or_local_path, ) from evalscope.api.evaluator import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, ReducedVerdict from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from .utils import ( OPTION_LABELS, + VERDICT_CONTRACT, build_mc_judge_prompt, build_mc_question, build_oe_judge_prompt, @@ -25,7 +27,6 @@ extract_oe_answer, match_mc_answer, match_oe_answer, - parse_judge_verdict, parse_options, ) @@ -175,6 +176,27 @@ def _build_score(self, prediction: str, extracted: str, correct: bool, explanati score.explanation = explanation return score + def build_judge_prompt(self, prediction: str, reference: str) -> str: + """Render the official judge prompt for this benchmark's answer format.""" + raise NotImplementedError + + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + prompt = self.build_judge_prompt(judge_context.filtered_prediction, + judge_context.reference) + case.output_contract.instruction() + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + return ReducedVerdict(value={'acc': float(case_verdicts[0].value.verdict)}) + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='equivalence', output_contract=VERDICT_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='acc' + ) + @register_benchmark( BenchmarkMeta( @@ -208,7 +230,7 @@ def _build_score(self, prediction: str, extracted: str, correct: bool, explanati - Default scoring is the official string-level match: the chosen letter is extracted from the reply and compared with the ground truth, accepting replies that mark the correct option the way the prompt prints it (`D:`) or emphasises it (`**D**`). -- Setting `judge_strategy='llm'` (with `judge_model_args`) reproduces the official LLM-judged mode. +- Setting `judge.strategy='llm'` with `judge.models` reproduces the official LLM-judged mode. The judge is only consulted for replies whose option letter could not be extracted, matching upstream. - Figures are sent inline as base64 and the largest is ~5 MB; set `max_image_bytes` in `dataset_args` @@ -240,25 +262,21 @@ def match_score( correct = match_mc_answer(filtered_prediction, original_prediction, reference) return self._build_score(original_prediction, filtered_prediction, correct, 'string match') - def llm_match_score( - self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState - ) -> Score: - """Score with the official judge, which only arbitrates replies without a clear letter. - - The pre-check is plain equality rather than the lenient ``match_mc_answer`` used in rule - mode: upstream keeps the ``D:`` / ``**D**`` fallbacks out of its judged path, and accepting - them here would credit a reply that committed to one letter while merely quoting the - correct option's text. - """ - if reference.strip().lower() == filtered_prediction.strip().lower(): - return self._build_score(original_prediction, filtered_prediction, True, 'string match') - if filtered_prediction.strip() in OPTION_LABELS: - # The reply committed to a different option; there is nothing for the judge to weigh. - return self._build_score(original_prediction, filtered_prediction, False, 'string match') + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + if context.reference.strip().lower() == context.filtered_prediction.strip().lower(): + return JudgeDefinition.skip( + self._build_score(context.original_prediction, context.filtered_prediction, True, 'string match'), + reason='exact_string_match', + ) + if context.filtered_prediction.strip() in OPTION_LABELS: + return JudgeDefinition.skip( + self._build_score(context.original_prediction, context.filtered_prediction, False, 'string match'), + reason='invalid_option_label', + ) + return super().judge_definition(context) - judge_response = self.llm_judge.judge(build_mc_judge_prompt(filtered_prediction, reference)) - correct = parse_judge_verdict(judge_response) - return self._build_score(original_prediction, filtered_prediction, correct, f'LLM judge: {judge_response}') + def build_judge_prompt(self, prediction: str, reference: str) -> str: + return build_mc_judge_prompt(prediction, reference) @register_benchmark( @@ -294,9 +312,9 @@ def llm_match_score( statement, else the whole reply is compared. A reply truncated before its answer therefore scores 0 for reasons unrelated to physics ability; give the model a generous `generation_config.max_tokens`. - Answers are free-form values with units, so an LLM judge is used by default (the official - recommendation): run with `judge_strategy='auto'` or `'llm'` and provide `judge_model_args`. The + recommendation): set `judge.strategy='auto'` or `'llm'` and provide `judge.models`. The judge is only consulted when the answer does not already match as a string. -- `judge_strategy='rule'` falls back to the official string-level mode, which understates accuracy +- `judge.strategy='rule'` falls back to the official string-level mode, which understates accuracy because equivalent spellings (`0.5 m` vs `50 cm`) do not match literally. - Figures are sent inline as base64 and the largest is ~5 MB; set `max_image_bytes` in `dataset_args` if the served model enforces a smaller per-image limit. @@ -312,7 +330,7 @@ def llm_match_score( class PhyXOEAdapter(PhyXAdapter): """PhyX in open-ended mode, scored by comparing the final answer with the ground truth.""" - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_DEFAULT def build_question(self, record: Dict[str, Any], options: Dict[str, str]) -> str: return build_oe_question(record['question_simply'], record['question']) @@ -330,13 +348,13 @@ def match_score( correct = match_oe_answer(filtered_prediction, original_prediction, reference) return self._build_score(original_prediction, filtered_prediction, correct, 'string match') - def llm_match_score( - self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState - ) -> Score: - """Score with the official judge, which decides whether the answers are equivalent.""" - if reference.strip().lower() == filtered_prediction.strip().lower(): - return self._build_score(original_prediction, filtered_prediction, True, 'string match') + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + if context.reference.strip().lower() == context.filtered_prediction.strip().lower(): + return JudgeDefinition.skip( + self._build_score(context.original_prediction, context.filtered_prediction, True, 'string match'), + reason='exact_string_match', + ) + return super().judge_definition(context) - judge_response = self.llm_judge.judge(build_oe_judge_prompt(filtered_prediction, reference)) - correct = parse_judge_verdict(judge_response) - return self._build_score(original_prediction, filtered_prediction, correct, f'LLM judge: {judge_response}') + def build_judge_prompt(self, prediction: str, reference: str) -> str: + return build_oe_judge_prompt(prediction, reference) diff --git a/evalscope/benchmarks/phyx/utils.py b/evalscope/benchmarks/phyx/utils.py index 5ab48a04e..425fd911c 100644 --- a/evalscope/benchmarks/phyx/utils.py +++ b/evalscope/benchmarks/phyx/utils.py @@ -4,9 +4,10 @@ Reference: https://github.com/NastyMarcus/PhyX (``vlmeval/dataset/utils/phyx.py``). """ import re +from pydantic import BaseModel from typing import Dict, List, Optional -from evalscope.metrics.judge.llm_judge import JUDGE_ERROR_PREFIX +from evalscope.api.judge import OutputContract # The official instruction suffixes, appended verbatim to the problem statement. The multiple-choice # suffix is concatenated without a separator and the open-ended one with a leading space, matching @@ -46,10 +47,13 @@ # Replies that echo the option list instead of announcing a choice, e.g. 'B: 5.2 mW/cm^2'. _MC_LABEL_RE = re.compile(r'([ABCD]):') -# A standalone 0 or 1 verdict. The lookarounds reject digits inside a number so that a judge reply -# discussing values ('0.49 vs 0.5') cannot be read as a verdict; the last occurrence wins because -# the judge prompt asks for the verdict at the end. -_VERDICT_RE = re.compile(r'(? bool: - """Read the judge's 1/0 verdict. - - A failed judge request fails closed: ``LLMJudge.judge`` reports errors as an ``[ERROR] ...`` - string whose embedded digits would otherwise be parsed as a verdict. - """ - if not response or response.startswith(JUDGE_ERROR_PREFIX): - return False - matches = _VERDICT_RE.findall(response) - return bool(matches) and matches[-1] == '1' diff --git a/evalscope/benchmarks/plawbench/plawbench_adapter.py b/evalscope/benchmarks/plawbench/plawbench_adapter.py index 1f8bea327..ae88a2bea 100644 --- a/evalscope/benchmarks/plawbench/plawbench_adapter.py +++ b/evalscope/benchmarks/plawbench/plawbench_adapter.py @@ -1,18 +1,19 @@ # flake8: noqa: E501 import json +from pydantic import BaseModel from typing import Any, Dict, List, Optional, Tuple from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict +from evalscope.api.messages import ChatMessageSystem, ChatMessageUser from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags -from evalscope.utils.function_utils import retry_call +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger from .utils import ( CONSULTATION_MAX_QUESTIONS, - JUDGE_KEY_TO_METRIC, JUDGE_SYSTEM_PROMPTS, JUDGE_TYPE_CASE_ANALYSIS, JUDGE_TYPE_LEGAL_QA, @@ -20,7 +21,6 @@ TASK_INSTRUCTIONS, USER_PROMPTS, build_conversation, - parse_json_to_dict, parse_rubric_sections, score_case_analysis, score_total_points, @@ -28,6 +28,22 @@ logger = get_logger() + +class SectionScore(BaseModel): + total_points: float + + +class CaseAnalysisGrade(BaseModel): + """The ``case_analysis`` judge reply; section denominators come from the dataset, not the judge.""" + score_details: Dict[str, SectionScore] + + +class TotalPointsGrade(BaseModel): + """The single-total judge reply for ``legal_qa`` / ``document_generation``.""" + total_points: float + max_points: Optional[float] = None + + SUBSET_LIST = [ 'case_analysis', 'legal_consultation', @@ -35,13 +51,7 @@ 'defendant_statement', ] -_EXTRA_PARAMS = { - 'judge_retries': { - 'type': 'int', - 'description': 'Maximum attempts per rubric judge request before the sample is scored as 0.', - 'value': 3, - }, -} +_EXTRA_PARAMS: Dict[str, Any] = {} _DESCRIPTION = """ ## Overview @@ -76,8 +86,8 @@ ## Evaluation Notes -- Requires an LLM judge: run with `judge_strategy='llm'` (or `'auto'`, which enables the judge for this benchmark) - and provide `judge_model_args`. `judge_strategy='rule'` is not supported. +- Requires an LLM judge: set `judge.strategy='llm'` (or `'auto'`, which enables the judge for this benchmark) + and provide `judge.models`. `judge.strategy='rule'` is not supported. - Metrics are point ratios in `[0, 1]`. `acc` is reported for every subset; `case_analysis` additionally reports `conclusion_acc`, `fact_acc`, `reasoning_acc`, and `law_acc`. These map one-to-one onto the official leaderboard columns: `legal_consultation` is Task1, `case_analysis` is Task2-Avg with its four dimensions, and the two @@ -90,10 +100,10 @@ `[0, max_points]`, so a judge that mis-reports the denominator cannot distort the score. - The judge output template for `case_analysis` is repaired relative to the official script, which ships malformed JSON and pins the conclusion section to zero points; every section is graded on its rubric allocation here. -- Judge requests are retried up to `judge_retries` times when the response cannot be parsed; a sample that still - fails is scored 0 and flagged via `judge_failed` in the review metadata. +- The judge model's transport retry policy is configured through its `generation_config`. A reply that still + fails the output contract is unavailable and excluded rather than silently scored as zero. - Case-analysis judging returns a long per-item breakdown. Give the judge a generous `max_tokens` - (for example 8192) in `judge_model_args.generation_config`. + (for example 8192) in `judge.models[].generation_config`. - The drafting subsets ask for a 2,500-3,000 character legal document, so the evaluated model also needs a generous `generation_config.max_tokens`. A truncated filing is graded as an incomplete document and scores near zero, which depresses Task3 for reasons unrelated to legal ability. @@ -124,13 +134,10 @@ class PLawBenchAdapter(DefaultDataAdapter): """Rubric-based Chinese legal practice benchmark graded by an LLM judge.""" - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self.judge_retries = int(self.extra_params.get('judge_retries', 3)) - if self.judge_retries <= 0: - raise ValueError('PLawBench judge_retries must be greater than 0.') def record_to_sample(self, record: Dict[str, Any]) -> Sample: judge_type = record['judge_type'] @@ -152,72 +159,50 @@ def record_to_sample(self, record: Dict[str, Any]) -> Sample: }, ) - def match_score( - self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState - ) -> Score: - raise ValueError( - 'PLawBench is scored against expert rubrics and requires an LLM judge. ' - "Set judge_strategy='llm' (or 'auto') and provide judge_model_args." - ) - - def llm_match_score( - self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState - ) -> Score: - metadata = task_state.metadata or {} + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + metadata = context.task_state.metadata or {} judge_type = metadata['judge_type'] - max_points = float(metadata['max_points']) - - score = Score(extracted_prediction=filtered_prediction, prediction=original_prediction) - - rubric_sections = None - if judge_type == JUDGE_TYPE_CASE_ANALYSIS: - rubric_sections = parse_rubric_sections(metadata['rubrics']) - - system_prompt, user_prompt = self._build_judge_prompts( - judge_type=judge_type, - metadata=metadata, - response=original_prediction, - rubric_sections=rubric_sections, - ) - - def judge_func() -> Dict[str, Any]: - judge_response = self.llm_judge.judge(prompt=user_prompt, system_prompt=system_prompt) - judge_json = parse_json_to_dict(judge_response) - if judge_type == JUDGE_TYPE_CASE_ANALYSIS: - values, details = score_case_analysis(judge_json, rubric_sections) + schema = CaseAnalysisGrade if judge_type == JUDGE_TYPE_CASE_ANALYSIS else TotalPointsGrade + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + current_metadata = judge_context.task_state.metadata or {} + current_type = current_metadata['judge_type'] + rubric_sections = parse_rubric_sections(current_metadata['rubrics']) \ + if current_type == JUDGE_TYPE_CASE_ANALYSIS else None + system_prompt, user_prompt = self._build_judge_prompts( + judge_type=current_type, + metadata=current_metadata, + response=judge_context.original_prediction, + rubric_sections=rubric_sections, + ) + return JudgeRequest( + messages=[ChatMessageSystem(content=system_prompt), + ChatMessageUser(content=user_prompt)] + ) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + current_metadata = judge_context.task_state.metadata or {} + current_type = current_metadata['judge_type'] + judge_json = case_verdicts[0].value.model_dump() + if current_type == JUDGE_TYPE_CASE_ANALYSIS: + values, details = score_case_analysis(judge_json, parse_rubric_sections(current_metadata['rubrics'])) else: - acc, details = score_total_points(judge_json, max_points) + acc, details = score_total_points(judge_json, float(current_metadata['max_points'])) values = {'acc': acc} - return {'values': values, 'details': details, 'response': judge_response} - - try: - result = retry_call(judge_func, retries=self.judge_retries, sleep_interval=1) - except Exception as e: - logger.warning(f'PLawBench judge failed for sample {metadata.get("id")}: {e}') - score.value = self._zero_values(judge_type) - score.main_score_name = 'acc' - score.explanation = f'Judge failed after {self.judge_retries} attempts: {e}' - score.metadata = { - 'source': 'plawbench_rubric_judge', - 'judge_type': judge_type, - 'judge_model': self.llm_judge.model_id, - 'judge_failed': True, - } + return ReducedVerdict(value=values, metadata={'judge_type': current_type, **details}) + + def finalize(score, review, judge_context) -> Score: + if review.metadata: + judge_context.task_state.target = self._format_target(review.metadata['judge_type'], review.metadata) return score - score.value = result['values'] - score.main_score_name = 'acc' - score.explanation = result['response'] - score.metadata = { - 'source': 'plawbench_rubric_judge', - 'judge_type': judge_type, - 'judge_model': self.llm_judge.model_id, - 'judge_failed': False, - **result['details'], - } - # Surface the rubric breakdown in the review target column, which is otherwise empty. - task_state.target = self._format_target(judge_type, result['details']) - return score + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='rubric', output_contract=OutputContract(schema_model=schema))], + request=request, + reduce=reduce, + main_score_name='acc', + finalize=finalize + ) def _build_judge_prompts( self, @@ -243,13 +228,6 @@ def _build_judge_prompts( .replace('<>', str(metadata['max_points'])) return system_prompt, user_prompt - @staticmethod - def _zero_values(judge_type: str) -> Dict[str, float]: - # Keep ``acc`` first so a failed sample reports the same primary metric as a scored one. - if judge_type == JUDGE_TYPE_CASE_ANALYSIS: - return {'acc': 0.0, **{metric: 0.0 for metric in JUDGE_KEY_TO_METRIC.values()}} - return {'acc': 0.0} - @staticmethod def _format_target(judge_type: str, details: Dict[str, Any]) -> str: lines = [f'**Rubric score**: {details["awarded_points"]:g} / {details["max_points"]:g}'] diff --git a/evalscope/benchmarks/researchrubrics/researchrubrics_adapter.py b/evalscope/benchmarks/researchrubrics/researchrubrics_adapter.py index a88fda5d3..f0126d4c0 100644 --- a/evalscope/benchmarks/researchrubrics/researchrubrics_adapter.py +++ b/evalscope/benchmarks/researchrubrics/researchrubrics_adapter.py @@ -1,10 +1,9 @@ from __future__ import annotations import json -import time from collections import defaultdict from concurrent.futures import ThreadPoolExecutor -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Dict, List, Optional from evalscope.agent.environments.local import TemporaryLocalAgentEnvironment from evalscope.agent.tools.bash import BASH_TOOL_INFO, run_bash @@ -13,9 +12,19 @@ from evalscope.api.benchmark.adapters import AgentLoopAdapter from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState +from evalscope.api.judge import ( + CaseVerdict, + JudgeCase, + JudgeContext, + JudgeDefinition, + JudgeRequest, + OutputContract, + ReducedVerdict, +) +from evalscope.api.messages import ChatMessageSystem, ChatMessageUser from evalscope.api.metric import AggScore, SampleScore, Score from evalscope.api.registry import register_benchmark -from evalscope.constants import JudgeStrategy, Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger from .utils import ( BINARY_SYSTEM_PROMPT, @@ -23,10 +32,9 @@ CHUNK_SYSTEM_PROMPT, CHUNK_USER_PROMPT, SYNTHESIS_USER_PROMPT, + BinaryGrade, + ChunkGrade, chunk_document, - parse_json_object, - validate_binary_result, - validate_chunk_result, ) logger = get_logger() @@ -42,11 +50,6 @@ 'description': 'Maximum estimated tokens in each document chunk sent to the judge.', 'value': 100000, }, - 'judge_retries': { - 'type': 'int', - 'description': 'Maximum attempts for each rubric judge request and JSON parse.', - 'value': 3, - }, } _DESCRIPTION = """ @@ -64,6 +67,15 @@ - **Dataset**: 101 tasks and 2,593 weighted rubric criteria - **Metric**: Binary rubric compliance score +## Key Features + +- 101 open-ended Deep Research tasks paired with 2,593 expert-written, weighted rubric criteria. +- Rubrics cover explicit and implicit requirements, information synthesis, references, communication + quality, and instruction following, and each is graded independently. +- Negative-weight criteria capture undesirable behaviours and subtract from the score when present. +- Long reports are graded with the official chunk-evidence-synthesis procedure once they exceed the + configured judge context threshold. + ## Agent Environment - Uses EvalScope's built-in agent environment by default and does not require ``agent_config``. The agent can use ``bash`` @@ -79,7 +91,7 @@ ## Evaluation Notes -- ResearchRubrics requires ``judge_model_args`` and ``judge_strategy='auto'`` or ``'llm'``. Gemini 2.5 Pro is the +- ResearchRubrics requires ``judge.models`` and ``judge.strategy='auto'`` or ``'llm'``. Gemini 2.5 Pro is the recommended judge for comparison with the paper, but no provider or model is hard-coded. - Every rubric is graded independently as Satisfied (1) or Not Satisfied (0), matching the public binary grader. The paper's ternary scores are not directly comparable. @@ -94,7 +106,6 @@ - ``judge_context_limit``: 150,000 estimated tokens - ``judge_chunk_size``: 100,000 estimated tokens -- ``judge_retries``: 3 attempts per judge request The judge must be configured explicitly. For example: @@ -104,12 +115,14 @@ run_task(TaskConfig( model='YOUR_AGENT_MODEL', datasets=['researchrubrics'], - judge_strategy='llm', - judge_model_args={ - 'model_id': 'YOUR_JUDGE_MODEL', - 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL', - 'api_key': 'YOUR_JUDGE_API_KEY', - 'generation_config': {'temperature': 0.0}, + judge={ + 'strategy': 'llm', + 'models': { + 'model_id': 'YOUR_JUDGE_MODEL', + 'api_url': 'OPENAI_COMPATIBLE_JUDGE_URL', + 'api_key': 'YOUR_JUDGE_API_KEY', + 'generation_config': {'temperature': 0.0}, + }, }, limit=1, )) @@ -139,7 +152,7 @@ ) class ResearchRubricsAdapter(AgentLoopAdapter): """Deep Research agent benchmark with binary rubric-based LLM judging.""" - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY strategy_name = 'function_calling' max_steps_default = 50 @@ -148,11 +161,8 @@ def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self.judge_context_limit = int(self.extra_params.get('judge_context_limit', 150000)) self.judge_chunk_size = int(self.extra_params.get('judge_chunk_size', 100000)) - self.judge_retries = int(self.extra_params.get('judge_retries', 3)) if self.judge_context_limit <= 0 or self.judge_chunk_size <= 0: raise ValueError('ResearchRubrics judge context and chunk limits must be greater than 0.') - if self.judge_retries <= 0: - raise ValueError('ResearchRubrics judge_retries must be greater than 0.') self.use_batch_scoring = True def record_to_sample(self, record: Dict[str, Any]) -> Sample: @@ -205,7 +215,6 @@ def batch_calculate_metrics( ) -> List[SampleScore]: if not task_states: return sample_scores - self._validate_judge_config() self.llm_judge # Initialize once before worker threads access it. workers = min(max(int(self._task_config.eval_batch_size), 1), len(task_states)) with ThreadPoolExecutor(max_workers=workers) as executor: @@ -242,7 +251,7 @@ def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]: if metric_name == 'compliance_score': continue results.append(self._mean_agg_score(metric_name, scores, 'compliance_score')) - return results + return [result for result in results if result is not None] def _score_task_state(self, task_state: TaskState) -> Score: report = task_state.output.completion or '' @@ -255,26 +264,7 @@ def _score_task_state(self, task_state: TaskState) -> Score: if not isinstance(rubrics, list) or not rubrics: raise ValueError('ResearchRubrics target must contain at least one rubric.') - rubric_results: List[Dict[str, Any]] = [] - axis_entries: Dict[str, List[Dict[str, Any]]] = defaultdict(list) - source_sample_id = task_state.metadata.get('sample_id') or task_state.sample_id - for index, rubric in enumerate(rubrics): - if not isinstance(rubric, dict): - raise ValueError(f'ResearchRubrics rubric {index} must be an object.') - result = self._judge_rubric( - report=report, - rubric=rubric, - index=index, - sample_id=source_sample_id, - ) - rubric_results.append(result) - axis_entries[result['axis']].append(result) - - compliance_score = self._weighted_compliance(rubric_results) - values: Dict[str, float] = {'compliance_score': compliance_score} - for axis, entries in axis_entries.items(): - if any(entry['weight'] > 0 for entry in entries): - values[f'axis/{axis}'] = self._weighted_compliance(entries) + score = self.score_with_judge_contracts(report, report, task_state.target, task_state) trace = task_state.agent_trace tool_names = {'bash'} @@ -283,132 +273,210 @@ def _score_task_state(self, task_state: TaskState) -> Score: tool_name = event.payload.get('name') or event.payload.get('tool_name') or event.payload.get('function') if tool_name: tool_names.add(str(tool_name)) - - return Score( - extracted_prediction=report, - prediction=report, - value=values, - explanation=f'Binary rubric compliance across {len(rubric_results)} criteria.', - metadata={ - 'source': 'researchrubrics_binary_judge', - 'grading_mode': 'binary', - 'judge_model': self.llm_judge.model_id, - 'rubrics': rubric_results, - 'rubric_count': len(rubric_results), - 'used_chunking': any(result['used_chunking'] for result in rubric_results), - 'agent': { - 'framework': trace.framework if trace else None, - 'strategy': trace.strategy if trace else self.strategy_name, - 'environment': trace.environment if trace else 'local', - 'max_steps': trace.max_steps if trace else self.max_steps, - 'tools': sorted(tool_names), - }, + score.metadata = { + **(score.metadata or {}), + 'agent': { + 'framework': trace.framework if trace else None, + 'strategy': trace.strategy if trace else self.strategy_name, + 'environment': trace.environment if trace else 'local', + 'max_steps': trace.max_steps if trace else self.max_steps, + 'tools': sorted(tool_names), }, + } + return score + + # -- Judge contract hooks -- + + def _uses_chunking(self, report: str) -> bool: + """Whether the report is judged chunk-by-chunk. Read by both the case and expansion hooks: + if the two disagreed, chunk cases would be emitted with no synthesis to fold them.""" + return len(report) // 4 > self.judge_context_limit + + def _report_chunks(self, report: str) -> List[str]: + """Chunks of one report, cached so each rubric and request does not re-split it. + + The cached tuple is read once: samples are scored on worker threads, so a two-step + read could hand one sample's chunks to another. + """ + cached = getattr(self, '_chunk_cache', None) + if cached is not None and cached[0] == report: + return cached[1] + chunks = chunk_document(report, max_tokens=self.judge_chunk_size) + self._chunk_cache = (report, chunks) + return chunks + + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + return JudgeDefinition.workflow( + cases=self._build_cases(context), + request=self._build_request, + reduce=self._reduce_verdicts, main_score_name='compliance_score', + expand=self._expand_cases, + finalize=self._finalize_score ) - def _judge_rubric(self, report: str, rubric: Dict[str, Any], index: int, sample_id: Any) -> Dict[str, Any]: - criterion = str(rubric.get('criterion', '')).strip() - axis = str(rubric.get('axis', '')).strip() - try: - weight = float(rubric.get('weight')) - except (TypeError, ValueError) as exc: - raise ValueError(f'ResearchRubrics rubric {index} has an invalid weight.') from exc - if not criterion or not axis: - raise ValueError(f'ResearchRubrics rubric {index} requires criterion and axis.') - - used_chunking = len(report) // 4 > self.judge_context_limit - if used_chunking: - result = self._judge_chunked( - report=report, - criterion=criterion, - axis=axis, - context_prefix=f'sample {sample_id} rubric {index}', + def _build_cases(self, context: JudgeContext) -> List[JudgeCase]: + rubrics = json.loads(context.reference) + report = context.filtered_prediction + used_chunking = self._uses_chunking(report) + binary_contract = OutputContract(schema_model=BinaryGrade) + chunk_contract = OutputContract(schema_model=ChunkGrade) + cases: List[JudgeCase] = [] + for idx, rubric in enumerate(rubrics): + criterion = str(rubric.get('criterion', '')).strip() + axis = str(rubric.get('axis', '')).strip() + weight = float(rubric.get('weight', 0)) + if used_chunking: + chunks = self._report_chunks(report) + for chunk_idx in range(len(chunks)): + cases.append( + JudgeCase( + case_id=f'rubric_{idx}_chunk_{chunk_idx}', + output_contract=chunk_contract, + metadata={ + 'kind': 'chunk', + 'rubric_idx': idx, + 'chunk_idx': chunk_idx, + 'total_chunks': len(chunks), + 'criterion': criterion, + 'axis': axis, + 'weight': weight, + }, + ) + ) + else: + cases.append( + JudgeCase( + case_id=f'rubric_{idx}', + output_contract=binary_contract, + metadata={ + 'kind': 'binary', + 'rubric_idx': idx, + 'criterion': criterion, + 'axis': axis, + 'weight': weight, + }, + ) + ) + return cases + + def _expand_cases(self, stage: int, completed_cases: List[CaseVerdict], context: JudgeContext) -> List[JudgeCase]: + if stage != 1: + return [] + # Emit synthesis cases for rubrics whose chunks are all complete. + rubrics = json.loads(context.reference) + report = context.filtered_prediction + if not self._uses_chunking(report): + return [] + # Collect evidence per rubric from completed chunk verdicts. + evidence_by_rubric: Dict[int, List[str]] = defaultdict(list) + for cv in completed_cases: + if cv.metadata.get('kind') != 'chunk': + continue + evidence_by_rubric[cv.metadata['rubric_idx']].extend(cv.value.relevant_evidence) + synthesis_cases: List[JudgeCase] = [] + for rubric_idx, evidence in evidence_by_rubric.items(): + rubric = rubrics[rubric_idx] + synthesis_cases.append( + JudgeCase( + case_id=f'rubric_{rubric_idx}_synthesis', + output_contract=OutputContract(schema_model=BinaryGrade), + metadata={ + 'kind': 'synthesis', + 'rubric_idx': rubric_idx, + 'criterion': str(rubric.get('criterion', '')).strip(), + 'axis': str(rubric.get('axis', '')).strip(), + 'weight': float(rubric.get('weight', 0)), + 'evidence': evidence, + }, + ) ) - else: + return synthesis_cases + + def _build_request(self, case, placement, completed_cases, context) -> JudgeRequest: + meta = case.metadata + kind = meta['kind'] + if kind == 'binary': prompt = BINARY_USER_PROMPT.format( - document_content=report, - rubric_title=criterion, - rubric_category=axis, - rubric_weight=weight, + document_content=context.filtered_prediction, + rubric_title=meta['criterion'], + rubric_category=meta['axis'], + rubric_weight=meta['weight'], ) - result = self._request_json( - prompt=prompt, - system_prompt=BINARY_SYSTEM_PROMPT, - validator=validate_binary_result, - context=f'sample {sample_id} rubric {index}', + return JudgeRequest( + messages=[ChatMessageSystem(content=BINARY_SYSTEM_PROMPT), + ChatMessageUser(content=prompt)] ) - - return { - 'index': index, - 'criterion': criterion, - 'axis': axis, - 'weight': weight, - **result, - 'used_chunking': used_chunking, - } - - def _judge_chunked(self, report: str, criterion: str, axis: str, context_prefix: str) -> Dict[str, Any]: - chunks = chunk_document(report, max_tokens=self.judge_chunk_size) - all_evidence: List[str] = [] - for index, chunk in enumerate(chunks, 1): + elif kind == 'chunk': + chunks = self._report_chunks(context.filtered_prediction) + chunk_content = chunks[meta['chunk_idx']] + chunk_num = meta['chunk_idx'] + 1 prompt = CHUNK_USER_PROMPT.format( - chunk_num=index, - total_chunks=len(chunks), - context_summary='Previous chunks evaluated' if index > 1 else 'First chunk', - chunk_content=chunk, - rubric_title=criterion, - rubric_category=axis, + chunk_num=chunk_num, + total_chunks=meta['total_chunks'], + context_summary='Previous chunks evaluated' if chunk_num > 1 else 'First chunk', + chunk_content=chunk_content, + rubric_title=meta['criterion'], + rubric_category=meta['axis'], + ) + return JudgeRequest( + messages=[ChatMessageSystem(content=CHUNK_SYSTEM_PROMPT), + ChatMessageUser(content=prompt)] ) - result = self._request_json( - prompt=prompt, - system_prompt=CHUNK_SYSTEM_PROMPT, - validator=validate_chunk_result, - context=f'{context_prefix} chunk {index}/{len(chunks)}', + else: # synthesis + synthesis_prompt = SYNTHESIS_USER_PROMPT.format( + all_evidence=json.dumps(meta['evidence'], ensure_ascii=False, indent=2), + rubric_title=meta['criterion'], + rubric_category=meta['axis'], + ) + return JudgeRequest( + messages=[ChatMessageSystem(content=BINARY_SYSTEM_PROMPT), + ChatMessageUser(content=synthesis_prompt)] ) - all_evidence.extend(result['relevant_evidence']) - synthesis_prompt = SYNTHESIS_USER_PROMPT.format( - all_evidence=json.dumps(all_evidence, ensure_ascii=False, indent=2), - rubric_title=criterion, - rubric_category=axis, - ) - return self._request_json( - prompt=synthesis_prompt, - system_prompt=BINARY_SYSTEM_PROMPT, - validator=validate_binary_result, - context=f'{context_prefix} chunk synthesis', + def _reduce_verdicts(self, case_verdicts: List[CaseVerdict], context: JudgeContext) -> ReducedVerdict: + rubrics = json.loads(context.reference) + # Collect final scores: binary verdicts + synthesis verdicts (skip raw chunk verdicts). + rubric_scores: Dict[int, Dict[str, Any]] = {} + used_chunking = False + for cv in case_verdicts: + if '_chunk_' in cv.case_id and '_synthesis' not in cv.case_id: + used_chunking = True + continue + parts = cv.case_id.replace('_synthesis', '').split('_') + rubric_idx = int(parts[1]) + if '_synthesis' in cv.case_id: + used_chunking = True + rubric = rubrics[rubric_idx] + rubric_scores[rubric_idx] = { + 'score': cv.value.score, + 'weight': float(rubric.get('weight', 0)), + 'axis': str(rubric.get('axis', '')).strip(), + } + + entries = list(rubric_scores.values()) + compliance = self._weighted_compliance(entries) + values: Dict[str, float] = {'compliance_score': compliance} + axis_entries: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + for entry in entries: + axis_entries[entry['axis']].append(entry) + for axis, axis_list in axis_entries.items(): + if any(e['weight'] > 0 for e in axis_list): + values[f'axis/{axis}'] = self._weighted_compliance(axis_list) + + return ReducedVerdict( + value=values, + metadata={ + 'source': 'researchrubrics_binary_judge', + 'grading_mode': 'binary', + 'rubric_count': len(rubric_scores), + 'used_chunking': used_chunking, + }, ) - def _request_json( - self, - prompt: str, - system_prompt: str, - validator: Callable[[Dict[str, Any]], Dict[str, Any]], - context: str, - ) -> Dict[str, Any]: - last_error: Optional[Exception] = None - last_response = '' - for attempt in range(self.judge_retries): - try: - last_response = self.llm_judge.judge(prompt=prompt, system_prompt=system_prompt) - return validator(parse_json_object(last_response)) - except Exception as exc: - last_error = exc - if attempt + 1 < self.judge_retries: - time.sleep(2**attempt) - raise RuntimeError( - f'ResearchRubrics judge failed for {context} after {self.judge_retries} attempts. ' - f'Last response: {last_response!r}' - ) from last_error - - def _validate_judge_config(self) -> None: - if self.judge_strategy not in {JudgeStrategy.AUTO, JudgeStrategy.LLM}: - raise ValueError( - 'ResearchRubrics requires judge_strategy="auto" or "llm"; rule and llm_recall are not supported.' - ) - if not self._task_config.judge_model_args: - raise ValueError('ResearchRubrics requires explicit judge_model_args for binary rubric grading.') + def _finalize_score(self, score: Score, review, context) -> Score: + score.explanation = f'Binary rubric compliance across {len(json.loads(context.reference))} criteria.' + return score @staticmethod def _weighted_compliance(entries: List[Dict[str, Any]]) -> float: @@ -419,12 +487,18 @@ def _weighted_compliance(entries: List[Dict[str, Any]]) -> float: return numerator / denominator @staticmethod - def _mean_agg_score(metric_name: str, sample_scores: List[SampleScore], value_key: str) -> AggScore: - values = [float(sample_score.score.value[value_key]) for sample_score in sample_scores] + def _mean_agg_score(metric_name: str, sample_scores: List[SampleScore], value_key: str) -> Optional[AggScore]: + # A sample whose judge review was unusable carries an empty value dict, so it holds no + # value for this key: it is excluded from the mean rather than counted as 0. A group with + # no usable sample yields no metric row at all. + scored = [sample_score for sample_score in sample_scores if value_key in sample_score.score.value] + if not scored: + return None + values = [float(sample_score.score.value[value_key]) for sample_score in scored] return AggScore( metric_name=metric_name, score=sum(values) / len(values), aggregation='mean', num=len(values), - ids=[sample_score.sample_id for sample_score in sample_scores], + ids=[sample_score.sample_id for sample_score in scored], ) diff --git a/evalscope/benchmarks/researchrubrics/utils.py b/evalscope/benchmarks/researchrubrics/utils.py index baf3aa63e..8cb7568e0 100644 --- a/evalscope/benchmarks/researchrubrics/utils.py +++ b/evalscope/benchmarks/researchrubrics/utils.py @@ -1,5 +1,8 @@ import json -from typing import Any, Dict, List +from pydantic import BaseModel, Field, model_validator +from typing import Any, Dict, List, Literal + +from evalscope.api.judge import OutputContract BINARY_SYSTEM_PROMPT = """You are an expert evaluator tasked with assessing whether a document satisfies specific rubric criteria. Your evaluation must be precise, objective, and based solely on the evidence present in the document. @@ -111,80 +114,33 @@ """ -def strip_json_fence(text: str) -> str: - text = text.strip() - if text.startswith('```json'): - text = text[7:] - elif text.startswith('```'): - text = text[3:] - if text.endswith('```'): - text = text[:-3] - return text.strip() - - -def parse_json_object(text: str) -> Dict[str, Any]: - data = json.loads(strip_json_fence(text)) - if not isinstance(data, dict): - raise ValueError('Judge response must be a JSON object.') - return data - - -def validate_binary_result(data: Dict[str, Any]) -> Dict[str, Any]: - verdict = data.get('verdict') - expected_scores = {'Not Satisfied': 0.0, 'Satisfied': 1.0} - if verdict not in expected_scores: - raise ValueError(f'Invalid binary verdict: {verdict!r}.') - try: - score = float(data.get('score')) - confidence = float(data.get('confidence')) - except (TypeError, ValueError) as exc: - raise ValueError('Judge score and confidence must be numeric.') from exc - if score != expected_scores[verdict]: - raise ValueError(f'Judge verdict {verdict!r} conflicts with score {score}.') - if not 0.0 <= confidence <= 1.0: - raise ValueError(f'Judge confidence must be between 0 and 1, got {confidence}.') - reasoning = data.get('reasoning') - evidence_quotes = data.get('evidence_quotes') - missing_elements = data.get('missing_elements') - if not isinstance(reasoning, str): - raise ValueError('Judge reasoning must be a string.') - if not isinstance(evidence_quotes, list) or not all(isinstance(item, str) for item in evidence_quotes): - raise ValueError('Judge evidence_quotes must be a list of strings.') - if not isinstance(missing_elements, list) or not all(isinstance(item, str) for item in missing_elements): - raise ValueError('Judge missing_elements must be a list of strings.') - return { - 'verdict': verdict, - 'score': score, - 'confidence': confidence, - 'reasoning': reasoning, - 'evidence_quotes': evidence_quotes, - 'missing_elements': missing_elements, - } - - -def validate_chunk_result(data: Dict[str, Any]) -> Dict[str, Any]: - evidence = data.get('relevant_evidence') - satisfaction = data.get('satisfaction') - confidence = data.get('confidence_for_chunk') - notes = data.get('notes') - if not isinstance(evidence, list) or not all(isinstance(item, str) for item in evidence): - raise ValueError('Chunk relevant_evidence must be a list of strings.') - if not isinstance(satisfaction, bool): - raise ValueError('Chunk satisfaction must be a boolean.') - try: - confidence = float(confidence) - except (TypeError, ValueError) as exc: - raise ValueError('Chunk confidence_for_chunk must be numeric.') from exc - if not 0.0 <= confidence <= 1.0: - raise ValueError(f'Chunk confidence must be between 0 and 1, got {confidence}.') - if not isinstance(notes, str): - raise ValueError('Chunk notes must be a string.') - return { - 'relevant_evidence': evidence, - 'satisfaction': satisfaction, - 'confidence_for_chunk': confidence, - 'notes': notes, - } +class BinaryGrade(BaseModel): + """The binary rubric judge reply — official ResearchRubrics format.""" + verdict: Literal['Not Satisfied', 'Satisfied'] + score: float + confidence: float = Field(ge=0.0, le=1.0) + reasoning: str + evidence_quotes: List[str] + missing_elements: List[str] + + @model_validator(mode='after') + def _check_consistency(self) -> 'BinaryGrade': + expected = 1.0 if self.verdict == 'Satisfied' else 0.0 + if self.score != expected: + raise ValueError(f'verdict {self.verdict!r} conflicts with score {self.score}') + return self + + +class ChunkGrade(BaseModel): + """Per-chunk evidence extraction reply.""" + relevant_evidence: List[str] + satisfaction: bool + confidence_for_chunk: float = Field(ge=0.0, le=1.0) + notes: str + + +BINARY_CONTRACT = OutputContract(schema_model=BinaryGrade) +CHUNK_CONTRACT = OutputContract(schema_model=ChunkGrade) def chunk_document(content: str, max_tokens: int) -> List[str]: diff --git a/evalscope/benchmarks/simple_qa/simple_qa_adapter.py b/evalscope/benchmarks/simple_qa/simple_qa_adapter.py index ff5366648..4dde9a2db 100644 --- a/evalscope/benchmarks/simple_qa/simple_qa_adapter.py +++ b/evalscope/benchmarks/simple_qa/simple_qa_adapter.py @@ -1,17 +1,22 @@ import ast -import re -from typing import Any, Dict +from pydantic import BaseModel, Field +from typing import Any, Dict, Literal from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.dataset import Sample -from evalscope.api.evaluator import TaskState -from evalscope.api.metric import Score +from evalscope.api.judge import JudgeContext, JudgeDefinition from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger logger = get_logger() + +class Grade(BaseModel): + reasoning: str = '' + verdict: Literal['A', 'B', 'C'] + + GRADER_TEMPLATE = """ Your job is to look at a question, a gold target, and a predicted answer, and then assign a grade of either ["CORRECT", "INCORRECT", "NOT_ATTEMPTED"]. First, I will give examples of each grade, and then you will grade a new example. @@ -90,7 +95,6 @@ B: INCORRECT C: NOT_ATTEMPTED -Just return the letters "A", "B", or "C", with no text around it. """.strip() # noqa: E501 @@ -138,7 +142,7 @@ ) class SimpleQAAdapter(DefaultDataAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -159,38 +163,32 @@ def record_to_sample(self, record: Dict[str, Any]) -> Sample: return Sample(input=question, target=answer, metadata=ast.literal_eval(metadata)) - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + prompt = GRADER_TEMPLATE.format( + question=context.task_state.input_text, + target=context.reference, + predicted_answer=context.filtered_prediction, + ) + return JudgeDefinition.labels( + prompt=prompt, + schema_model=Grade, + case_id='grade', + scores={ + 'A': { + 'is_correct': 1.0, + 'is_incorrect': 0.0, + 'is_not_attempted': 0.0 + }, + 'B': { + 'is_correct': 0.0, + 'is_incorrect': 1.0, + 'is_not_attempted': 0.0 + }, + 'C': { + 'is_correct': 0.0, + 'is_incorrect': 0.0, + 'is_not_attempted': 1.0 + }, + }, + main_score_name='is_correct', ) - - question = task_state.input_text - - # Request judge and obtain score - prompt = GRADER_TEMPLATE.format(question=question, target=reference, predicted_answer=filtered_prediction) - judge_response = self.llm_judge.judge(prompt) - # parse grading response - match = re.search(r'(A|B|C)', judge_response) - res = match.group(0) if match else 'C' - - # Set score based on the match result - score.value = { - 'is_correct': 1 if res == 'A' else 0, - 'is_incorrect': 1 if res == 'B' else 0, - 'is_not_attempted': 1 if res == 'C' else 0, - } - score.explanation = f'LLM judge: {judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id - } - score.main_score_name = 'is_correct' - return score diff --git a/evalscope/benchmarks/simple_vqa/simple_vqa_adapter.py b/evalscope/benchmarks/simple_vqa/simple_vqa_adapter.py index 51ec056dc..acc4ff6ea 100644 --- a/evalscope/benchmarks/simple_vqa/simple_vqa_adapter.py +++ b/evalscope/benchmarks/simple_vqa/simple_vqa_adapter.py @@ -1,19 +1,27 @@ # flake8: noqa: E501 -import re -from typing import Any, Dict +from pydantic import BaseModel, Field +from typing import Any, Dict, List, Literal from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter from evalscope.api.dataset import Sample -from evalscope.api.evaluator import TaskState +from evalscope.api.judge import JudgeCase, JudgeContext, JudgeDefinition, JudgeRequest, OutputContract, ReducedVerdict from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText from evalscope.api.metric.scorer import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.io_utils import convert_image_base64_format from evalscope.utils.logger import get_logger logger = get_logger() + +class Grade(BaseModel): + reasoning: str = '' + verdict: Literal['A', 'B', 'C'] + + +GRADE_CONTRACT = OutputContract(schema_model=Grade) + GRADER_TEMPLATE = """ Your job is to look at a question, a gold target, and a predicted answer, and then assign a grade of either ["CORRECT", "INCORRECT", "NOT_ATTEMPTED"]. First, I will give examples of each grade, and then you will grade a new example. @@ -92,7 +100,6 @@ B: INCORRECT C: NOT_ATTEMPTED -Just return the letters "A", "B", or "C", with no text around it. """.strip() # noqa: E501 @@ -137,7 +144,7 @@ ) class SimpleVQAAdapter(VisionLanguageAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, **kwargs): super().__init__(**kwargs) @@ -165,38 +172,29 @@ def record_to_sample(self, record: Dict[str, Any]) -> Sample: } ) - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - score = Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + + def request(case, placement, completed_cases, judge_context) -> JudgeRequest: + prompt = GRADER_TEMPLATE.format( + question=judge_context.task_state.input_text, + target=judge_context.reference, + predicted_answer=judge_context.filtered_prediction, + ) + case.output_contract.instruction() + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) + + def reduce(case_verdicts, judge_context) -> ReducedVerdict: + grade = case_verdicts[0].value.verdict + return ReducedVerdict( + value={ + 'is_correct': 1.0 if grade == 'A' else 0.0, + 'is_incorrect': 1.0 if grade == 'B' else 0.0, + 'is_not_attempted': 1.0 if grade == 'C' else 0.0, + } + ) + + return JudgeDefinition.workflow( + cases=[JudgeCase(case_id='grade', output_contract=GRADE_CONTRACT)], + request=request, + reduce=reduce, + main_score_name='is_correct' ) - - question = task_state.input_text - - # Request judge and obtain score - prompt = GRADER_TEMPLATE.format(question=question, target=reference, predicted_answer=filtered_prediction) - judge_response = self.llm_judge.judge(prompt) - # parse grading response - match = re.search(r'(A|B|C)', judge_response) - res = match.group(0) if match else 'C' - - # Set score based on the match result - score.value = { - 'is_correct': 1 if res == 'A' else 0, - 'is_incorrect': 1 if res == 'B' else 0, - 'is_not_attempted': 1 if res == 'C' else 0, - } - score.explanation = f'LLM judge: {judge_response}' - score.metadata = { - 'source': 'llm_judge', - 'judge_strategy': self.judge_strategy, - 'model': self.llm_judge.model_id - } - score.main_score_name = 'is_correct' - return score diff --git a/evalscope/benchmarks/slake/slake_adapter.py b/evalscope/benchmarks/slake/slake_adapter.py index b4e596687..a9f6f909b 100644 --- a/evalscope/benchmarks/slake/slake_adapter.py +++ b/evalscope/benchmarks/slake/slake_adapter.py @@ -73,7 +73,7 @@ accuracy on the knowledge-graph questions is expected to be low - Strictness also costs answers that differ from the reference only in wording or granularity (`Right` answered as `Right Side`, 胸腔 answered as 胸部, one item of a multi-item reference). Set - `judge_strategy='llm_recall'` to have an LLM judge re-check only the rule-based failures; that + `judge.strategy='llm_recall'` to have an LLM judge re-check only the rule-based failures; that score is more lenient than the published numbers and is not directly comparable to them - Images ship as a single `imgs.zip` (about 200 MB) and are read directly from the archive - [Paper](https://arxiv.org/abs/2102.09542) | [Project page](https://www.med-vqa.com/slake/) diff --git a/evalscope/benchmarks/tir_bench/tir_bench_adapter.py b/evalscope/benchmarks/tir_bench/tir_bench_adapter.py index 88fa65ce5..2dcde4c0d 100644 --- a/evalscope/benchmarks/tir_bench/tir_bench_adapter.py +++ b/evalscope/benchmarks/tir_bench/tir_bench_adapter.py @@ -43,7 +43,7 @@ - Images are downloaded as `data.zip` from ModelScope and extracted automatically - Rule-based scoring: OCR (substring match), jigsaw (grid IoU), spot_difference (set IoU), word_search (numeric match), all other tasks (MCQ / numeric judge) -- **Recommended**: set `judge_strategy=JudgeStrategy.LLM_RECALL` and provide `judge_model_args` +- **Recommended**: set `judge.strategy='llm_recall'` and provide `judge.models` to activate LLM-as-judge as a recall mechanism — the judge is called only when rule-based scoring gives 0, providing more accurate evaluation without unnecessary API overhead - [Paper](https://arxiv.org/abs/2511.01833) | [GitHub](https://github.com/agents-x-project/TIR-Bench) diff --git a/evalscope/benchmarks/wide_search/utils.py b/evalscope/benchmarks/wide_search/utils.py index 25c67209a..eea53b838 100644 --- a/evalscope/benchmarks/wide_search/utils.py +++ b/evalscope/benchmarks/wide_search/utils.py @@ -3,13 +3,12 @@ from __future__ import annotations -import json import pandas as pd import re from collections import defaultdict from dataclasses import dataclass, field from io import StringIO -from typing import Any, Callable, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional from urllib.parse import urlparse from evalscope.api.metric import AggScore, SampleScore @@ -43,8 +42,10 @@ Please output the alignment results in the following format: ```json {{ - "origin_str1": "transform_str1", - "origin_str2": "transform_str2" + "mapping": {{ + "origin_str1": "transform_str1", + "origin_str2": "transform_str2" + }} }} ``` """ # noqa: E501 @@ -55,7 +56,7 @@ 1. The scoring range is from 0 to 1. A score of 1 indicates a completely correct answer. For deduction items, please refer to the specific grading criteria section. 2. After reading the standard answers, responses to be graded, and grading criteria, please first analyze and judge them item by item according to the grading criteria. 3. The score can only be an integer of 0 or 1. -4. After the analysis and judgment, please provide the final scoring results. Each pair should have a score. Output in Markdown JSON format, as shown below: +4. After the analysis and judgment, provide the final scoring results. Each pair should have a score. Reply with a single JSON object and no prose, as shown below: ```json {{ "idx_xxx": score, @@ -77,12 +78,6 @@ """ # noqa: E501 -@dataclass -class EvaluationResult: - values: Dict[str, float] = field(default_factory=lambda: {name: 0.0 for name in METRIC_NAMES}) - diagnostics: Dict[str, Any] = field(default_factory=dict) - - def norm_column(column: str) -> str: return str(column).strip().lower().replace(' ', '') @@ -116,17 +111,6 @@ def extract_markdown_table(response: str) -> Optional[pd.DataFrame]: return response_df.loc[:, ~response_df.columns.str.startswith('Unnamed')] -def parse_markdown_json(completion: str) -> Optional[Dict[str, Any]]: - matches = re.findall(r'```json\s*(\{.*?\})\s*```', completion, re.DOTALL) - if not matches: - return None - try: - parsed = json.loads(matches[-1]) - except Exception: - return None - return parsed if isinstance(parsed, dict) else None - - def extract_number(content: str) -> str: numbers = re.findall(r'[-+]?\d*\.\d+%?|[-+]?\d+\.?\d*%?', str(content).replace(',', '')) return numbers[0] if numbers else 'NULL' @@ -198,198 +182,172 @@ def date_near(response: str, target: str) -> float: } -class WideSearchScorer: - """Official WideSearch table aligner and hybrid scorer.""" +@dataclass +class WideSearchSession: + """Pure, official WideSearch table preparation and reduction. + + The adapter turns the LLM-dependent alignment and column scoring stages into judge cases; + this class deliberately contains no model callback and no parsing of judge text. + """ - def __init__(self, judge: Callable[[str], str]) -> None: - self.judge = judge + answer_df: pd.DataFrame + response_df: Optional[pd.DataFrame] + evaluation: Dict[str, Any] + error: Optional[str] = None - def evaluate(self, prediction: str, gold_csv: str, evaluation: Dict[str, Any]) -> EvaluationResult: - diagnostics: Dict[str, Any] = {'stage': 'parse'} + @classmethod + def create(cls, prediction: str, gold_csv: str, evaluation: Dict[str, Any]) -> 'WideSearchSession': try: - required_columns = list(evaluation['required']) - unique_columns = list(evaluation['unique_columns']) + required = list(evaluation['required']) answer_df = pd.read_csv(StringIO(gold_csv)) answer_df.columns = [norm_column(column) for column in answer_df.columns] - answer_df = answer_df[required_columns] + answer_df = answer_df[required] response_df = extract_markdown_table(prediction) if response_df is None: - diagnostics['error'] = 'response_df is None' - return EvaluationResult(diagnostics=diagnostics) - + return cls(answer_df=answer_df, response_df=None, evaluation=evaluation, error='response_df is None') response_df.columns = [norm_column(column) for column in response_df.columns] - if set(required_columns) != set(response_df.columns): - column_map, judge_response = self._map_values(response_df.columns.tolist(), required_columns) - column_map = _unique_target_map(column_map) - diagnostics['column_map'] = column_map - diagnostics['column_map_judge'] = judge_response - response_df.rename(columns=column_map, inplace=True) - if set(required_columns) != set(response_df.columns): - diagnostics['error'] = 'required columns do not match response columns' - diagnostics['response_columns'] = response_df.columns.tolist() - return EvaluationResult(diagnostics=diagnostics) - - diagnostics['stage'] = 'align' - for column in required_columns: - try: - answer_type = answer_df[column].dtype - response_type = response_df[column].dtype - except Exception: - answer_type = None - response_type = None - if (response_type == float and answer_type == int) or (response_type == int and answer_type == float): - if response_type == int: - response_df[column] = response_df[column].astype(float) - elif answer_type == int: - answer_df[column] = answer_df[column].astype(float) - answer_df[column] = answer_df[column].astype(str) - response_df[column] = response_df[column].astype(str) - - response_df.drop_duplicates(subset=unique_columns, inplace=True) - answer_df.drop_duplicates(subset=unique_columns, inplace=True) - diagnostics['primary_key_maps'] = {} - diagnostics['primary_key_judges'] = {} - for column in unique_columns: - pipeline = evaluation['eval_pipeline'].get(column) - if pipeline is None: - continue - metrics = pipeline.get('metric', []) - if 'llm_judge' in metrics or 'exact_match' in metrics: - value_map, judge_response = self._map_values( - response_df[column].tolist(), answer_df[column].tolist() - ) - diagnostics['primary_key_maps'][column] = value_map - diagnostics['primary_key_judges'][column] = judge_response - response_df[column] = response_df[column].apply(lambda value: value_map.get(value, value)) - - for column, pipeline in evaluation['eval_pipeline'].items(): - for preprocess_name in pipeline.get('preprocess', []): - preprocess = PREPROCESSORS[preprocess_name] - response_df[column] = response_df[column].apply(preprocess) - answer_df[column] = answer_df[column].apply(preprocess) - - inner_df = pd.merge( - answer_df, - response_df, - on=unique_columns, - how='inner', - suffixes=('_query', '_response'), + return cls(answer_df=answer_df, response_df=response_df, evaluation=evaluation) + except Exception as error: + return cls( + answer_df=pd.DataFrame(), + response_df=None, + evaluation=evaluation, + error=f'{type(error).__name__}: {error}' ) - diagnostics.update({ - 'gold_rows': len(answer_df), - 'prediction_rows': len(response_df), - 'matched_rows': len(inner_df), - }) - if inner_df.empty: - diagnostics['stage'] = 'complete' - return EvaluationResult(values={name: 0.0 for name in METRIC_NAMES}, diagnostics=diagnostics) - - diagnostics['stage'] = 'score' - inner_scores = pd.DataFrame(index=inner_df.index) - diagnostics['column_judges'] = {} - for column in required_columns: - if column in unique_columns: - inner_scores[f'{column}_exact_match'] = 1.0 - continue - pipeline = evaluation['eval_pipeline'][column] - criterion = pipeline.get('criterion') - for metric_name in pipeline.get('metric', []): - if metric_name == 'llm_judge': - values, judge_response = self._judge_column( - inner_df[f'{column}_response'].tolist(), - inner_df[f'{column}_query'].tolist(), - criterion, + + @property + def required_columns(self) -> List[str]: + return list(self.evaluation['required']) + + @property + def unique_columns(self) -> List[str]: + return list(self.evaluation['unique_columns']) + + def needs_column_alignment(self) -> bool: + return self.response_df is not None and set(self.required_columns) != set(self.response_df.columns) + + def frames( + self, + column_map: Optional[Dict[str, str]] = None, + primary_key_maps: Optional[Dict[str, Dict[str, str]]] = None, + ) -> tuple[Optional[pd.DataFrame], Optional[pd.DataFrame], Dict[str, Any]]: + diagnostics: Dict[str, Any] = {'stage': 'align'} + if self.error or self.response_df is None: + diagnostics['error'] = self.error or 'response_df is None' + return None, None, diagnostics + answer_df = self.answer_df.copy() + response_df = self.response_df.copy() + if column_map: + response_df.rename(columns=_unique_target_map(column_map), inplace=True) + if set(self.required_columns) != set(response_df.columns): + diagnostics['error'] = 'required columns do not match response columns' + diagnostics['response_columns'] = response_df.columns.tolist() + return None, None, diagnostics + for column in self.required_columns: + answer_df[column] = answer_df[column].astype(str) + response_df[column] = response_df[column].astype(str) + response_df.drop_duplicates(subset=self.unique_columns, inplace=True) + answer_df.drop_duplicates(subset=self.unique_columns, inplace=True) + for column, value_map in (primary_key_maps or {}).items(): + response_df[column] = response_df[column].apply(lambda value: value_map.get(value, value)) + for column, pipeline in self.evaluation['eval_pipeline'].items(): + for preprocess_name in pipeline.get('preprocess', []): + preprocess = PREPROCESSORS[preprocess_name] + response_df[column] = response_df[column].apply(preprocess) + answer_df[column] = answer_df[column].apply(preprocess) + return answer_df, response_df, diagnostics + + def inner_frame( + self, + column_map: Optional[Dict[str, str]] = None, + primary_key_maps: Optional[Dict[str, Dict[str, str]]] = None, + ) -> tuple[Optional[pd.DataFrame], Dict[str, Any]]: + answer_df, response_df, diagnostics = self.frames(column_map, primary_key_maps) + if answer_df is None or response_df is None: + return None, diagnostics + inner_df = pd.merge( + answer_df, + response_df, + on=self.unique_columns, + how='inner', + suffixes=('_query', '_response'), + ) + diagnostics.update({ + 'gold_rows': len(answer_df), + 'prediction_rows': len(response_df), + 'matched_rows': len(inner_df), + }) + return inner_df, diagnostics + + def score( + self, + column_scores: Dict[str, List[float]], + column_map: Optional[Dict[str, str]] = None, + primary_key_maps: Optional[Dict[str, Dict[str, str]]] = None, + ) -> tuple[Dict[str, float], Dict[str, Any]]: + inner_df, diagnostics = self.inner_frame(column_map, primary_key_maps) + if inner_df is None or inner_df.empty: + return {name: 0.0 for name in METRIC_NAMES}, diagnostics + inner_scores = pd.DataFrame(index=inner_df.index) + for column in self.required_columns: + if column in self.unique_columns: + inner_scores[f'{column}_exact_match'] = 1.0 + continue + pipeline = self.evaluation['eval_pipeline'][column] + criterion = pipeline.get('criterion') + for metric_name in pipeline.get('metric', []): + key = f'{column}_{metric_name}' + if metric_name == 'llm_judge': + values = column_scores[key] + if len(values) != len(inner_df): + raise ValueError( + f'WideSearch judge case {key} scored {len(values)} rows, expected {len(inner_df)}.' ) - diagnostics['column_judges'][column] = judge_response - else: - values = [ - self._metric_call(response, target, criterion, metric_name) - for response, target in zip(inner_df[f'{column}_response'], inner_df[f'{column}_query']) - ] - inner_scores[f'{column}_{metric_name}'] = values - - row_scores = inner_scores.min(axis=1) - true_positive_rows = float(row_scores.sum()) - true_positive_items = float(inner_scores.sum().sum()) - prediction_rows = len(response_df) - gold_rows = len(answer_df) - prediction_items = prediction_rows * len(required_columns) - gold_items = gold_rows * len(required_columns) - - row_precision = true_positive_rows / prediction_rows if prediction_rows else 0.0 - row_recall = true_positive_rows / gold_rows if gold_rows else 0.0 - item_precision = true_positive_items / prediction_items if prediction_items else 0.0 - item_recall = true_positive_items / gold_items if gold_items else 0.0 - row_f1 = _f1(row_precision, row_recall) - item_f1 = _f1(item_precision, item_recall) - success_rate = float( + else: + values = [ + _metric_call(response, target, criterion, metric_name) + for response, target in zip(inner_df[f'{column}_response'], inner_df[f'{column}_query']) + ] + inner_scores[key] = values + row_scores = inner_scores.min(axis=1) + true_positive_rows = float(row_scores.sum()) + true_positive_items = float(inner_scores.sum().sum()) + _, response_df, _ = self.frames(column_map, primary_key_maps) + prediction_rows = len(response_df) if response_df is not None else 0 + gold_rows = len(self.answer_df) + row_precision = true_positive_rows / prediction_rows if prediction_rows else 0.0 + row_recall = true_positive_rows / gold_rows if gold_rows else 0.0 + item_precision = true_positive_items / ( + prediction_rows * len(self.required_columns) + ) if prediction_rows else 0.0 + item_recall = true_positive_items / (gold_rows * len(self.required_columns)) if gold_rows else 0.0 + row_f1 = _f1(row_precision, row_recall) + item_f1 = _f1(item_precision, item_recall) + return { + 'success_rate': float( row_precision == row_recall == row_f1 == 1.0 and item_precision == item_recall == item_f1 == 1.0 - ) - values = { - 'success_rate': success_rate, - 'row_precision': row_precision, - 'row_recall': row_recall, - 'row_f1': row_f1, - 'item_precision': item_precision, - 'item_recall': item_recall, - 'item_f1': item_f1, - } - diagnostics['stage'] = 'complete' - return EvaluationResult(values=values, diagnostics=diagnostics) - except Exception as error: - diagnostics['error'] = f'{type(error).__name__}: {error}' - return EvaluationResult(diagnostics=diagnostics) - - def _map_values(self, response: List[str], reference: List[str]) -> Tuple[Dict[str, str], str]: - prompt = PRIMARY_KEY_PREPROCESS_PROMPT.format(response=response, reference=reference) - judge_response = self.judge(prompt) - parsed = parse_markdown_json(judge_response) - if parsed is None: - return {}, judge_response - return {str(key): str(value) for key, value in parsed.items()}, judge_response - - def _judge_column( - self, - response: List[str], - target: List[str], - criterion: Optional[str], - ) -> Tuple[List[float], str]: - response_dict = { - f'idx_{index}': { - 'response': response_value, - 'target': target_value - } - for index, (response_value, target_value) in enumerate(zip(response, target)) - } - prompt = EVAL_COLUMN_PROMPT.format(criterion=criterion, response=response_dict) - try: - judge_response = self.judge(prompt) - except Exception as error: - return [0.0] * len(response), f'{type(error).__name__}: {error}' - parsed = parse_markdown_json(judge_response) - if parsed is None: - return [0.0] * len(response), judge_response - scores = [] - for index in range(len(response)): - try: - scores.append(float(parsed.get(f'idx_{index}', 0))) - except (TypeError, ValueError): - scores.append(0.0) - return scores, judge_response - - @staticmethod - def _metric_call(response: str, target: str, criterion: Any, metric_name: str) -> float: - if metric_name == 'exact_match': - return exact_match(response, target) - if metric_name == 'url_match': - return url_match(response, target) - if metric_name == 'in_match': - return in_match(response, target) - if metric_name == 'number_near': - return number_near(response, target, float(criterion)) - if metric_name == 'date_near': - return date_near(response, target) - raise ValueError(f'Unsupported WideSearch metric: {metric_name}') + ), + 'row_precision': row_precision, + 'row_recall': row_recall, + 'row_f1': row_f1, + 'item_precision': item_precision, + 'item_recall': item_recall, + 'item_f1': item_f1, + }, diagnostics + + +def _metric_call(response: str, target: str, criterion: Any, metric_name: str) -> float: + if metric_name == 'exact_match': + return exact_match(response, target) + if metric_name == 'url_match': + return url_match(response, target) + if metric_name == 'in_match': + return in_match(response, target) + if metric_name == 'number_near': + return number_near(response, target, float(criterion)) + if metric_name == 'date_near': + return date_near(response, target) + raise ValueError(f'Unsupported WideSearch metric: {metric_name}') def _f1(precision: float, recall: float) -> float: @@ -422,11 +380,7 @@ def aggregate_official_scores(sample_scores: List[SampleScore]) -> List[AggScore for score in scoped_scores: group_id = score.group_id if score.group_id is not None else score.sample_id grouped[group_id].append(score) - repeat_counts = {len(group) for group in grouped.values()} - if len(repeat_counts) != 1: - raise ValueError(f'WideSearch requires the same number of trials per task, got {sorted(repeat_counts)}.') - repeats = repeat_counts.pop() - sample_ids = [score.sample_id for score in scoped_scores] + repeats = max(len(group) for group in grouped.values()) for metric_name in METRIC_NAMES: if metric_name.startswith('row_'): canonical_name = metric_name[4:] @@ -437,10 +391,13 @@ def aggregate_official_scores(sample_scores: List[SampleScore]) -> List[AggScore else: canonical_name = metric_name target = None + metric_scores = [score for score in scoped_scores if metric_name in score.score.value] + if not metric_scores: + continue dimensions = {'scope': scope, 'k': repeats} if target is not None: dimensions['target'] = target - all_values = [float(score.score.value[metric_name]) for score in scoped_scores] + all_values = [float(score.score.value[metric_name]) for score in metric_scores] results.append( AggScore( metric_name=canonical_name, @@ -448,10 +405,30 @@ def aggregate_official_scores(sample_scores: List[SampleScore]) -> List[AggScore aggregation='mean', dimensions=dimensions, num=len(all_values), - ids=sample_ids, + ids=[score.sample_id for score in metric_scores], + metadata={ + 'eligible': len(metric_scores), + 'total': len(scoped_scores), + 'coverage': len(metric_scores) / len(scoped_scores), + }, ) ) - group_maxima = [max(float(score.score.value[metric_name]) for score in group) for group in grouped.values()] + eligible_groups = [] + for group_id, group in grouped.items(): + indexed = sorted( + ((score.generation_index if score.generation_index is not None else index, score) + for index, score in enumerate(group) + if metric_name in score.score.value), + key=lambda item: item[0], + ) + if [index for index, _ in indexed] != list(range(repeats)): + continue + eligible_groups.append((group_id, [score for _, score in indexed])) + if not eligible_groups: + continue + group_maxima = [ + max(float(score.score.value[metric_name]) for score in group) for _, group in eligible_groups + ] aggregate_name = 'pass' if metric_name == 'success_rate' else 'max' results.append( AggScore( @@ -460,7 +437,13 @@ def aggregate_official_scores(sample_scores: List[SampleScore]) -> List[AggScore aggregation='pass_at_k' if aggregate_name == 'pass' else 'max', dimensions=dimensions, num=len(group_maxima), - ids=list(grouped.keys()), + ids=[group_id for group_id, _ in eligible_groups], + metadata={ + 'eligible': len(eligible_groups), + 'total': len(grouped), + 'excluded': len(grouped) - len(eligible_groups), + 'coverage': len(eligible_groups) / len(grouped), + }, ) ) return results diff --git a/evalscope/benchmarks/wide_search/wide_search_adapter.py b/evalscope/benchmarks/wide_search/wide_search_adapter.py index 3586e7d67..d368401f7 100644 --- a/evalscope/benchmarks/wide_search/wide_search_adapter.py +++ b/evalscope/benchmarks/wide_search/wide_search_adapter.py @@ -1,9 +1,9 @@ from __future__ import annotations import json -import threading from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from pydantic import BaseModel, Field +from typing import Any, Dict, List, Optional, Sequence, Tuple from evalscope.agent.environments.local import TemporaryLocalAgentEnvironment from evalscope.agent.tools.bash import BASH_TOOL_INFO, run_bash @@ -12,14 +12,29 @@ from evalscope.api.benchmark.adapters import AgentLoopAdapter from evalscope.api.dataset import DatasetDict, Sample, load_local_file_dataset, resolve_snapshot_or_local_path from evalscope.api.evaluator import TaskState -from evalscope.api.messages import ChatMessageSystem +from evalscope.api.judge import ( + CaseVerdict, + JudgeCase, + JudgeContext, + JudgeDefinition, + JudgeRequest, + OutputContract, + ReducedVerdict, +) +from evalscope.api.messages import ChatMessageSystem, ChatMessageUser from evalscope.api.metric import AggScore, SampleScore, Score from evalscope.api.metric.semantics import MetricSelector from evalscope.api.registry import register_benchmark from evalscope.api.sandbox import merge_sandbox_config_dicts -from evalscope.constants import JudgeStrategy, Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.import_utils import check_import -from .utils import METRIC_NAMES, WideSearchScorer, aggregate_official_scores +from .utils import ( + EVAL_COLUMN_PROMPT, + METRIC_NAMES, + PRIMARY_KEY_PREPROCESS_PROMPT, + WideSearchSession, + aggregate_official_scores, +) DATASET_ID = 'bytedance-community/WideSearch' @@ -65,7 +80,7 @@ ## Evaluation Notes - Uses the official Markdown table alignment and hybrid rule/LLM scoring semantics. -- Requires ``judge_strategy='auto'`` or ``'llm'`` with explicit ``judge_model_args``; rule-only scoring is unsupported. +- Requires ``judge.strategy='auto'`` or ``'llm'`` with at least one ``judge.models`` entry; rule-only scoring is unsupported. - See the [WideSearch usage guide](https://evalscope.readthedocs.io/en/latest/third_party/wide_search.html) for runtime examples and paper-style repeat settings. """ @@ -89,7 +104,7 @@ ) class WideSearchAdapter(AgentLoopAdapter): """Official single-agent WideSearch benchmark adapter.""" - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY strategy_name = 'function_calling' max_steps_default = 50 @@ -100,7 +115,6 @@ def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) check_import('dateparser', extra='wide_search', raise_error=True, feature_name='WideSearch evaluation') self._dataset_root: Optional[Path] = None - self._judge_lock = threading.Lock() def load(self) -> Tuple[DatasetDict, None]: # NOTE: download the full snapshot rather than an ``allow_file_pattern`` list. @@ -180,45 +194,168 @@ def build_max_steps_finalization_message(self, sample: Sample) -> str: def should_finalize_after_max_steps(self, result: AgentLoopResult) -> bool: return True - def llm_match_score( - self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - self._validate_judge_config() - with self._judge_lock: - judge = self.llm_judge - scorer = WideSearchScorer(judge=judge.judge) - result = scorer.evaluate( - prediction=filtered_prediction, - gold_csv=reference, - evaluation=task_state.metadata['evaluation'], - ) - return Score( - extracted_prediction=filtered_prediction, - prediction=original_prediction, - value=result.values, - explanation=result.diagnostics.get('error') or 'Official WideSearch table evaluation completed.', - metadata=result.diagnostics, + def judge_definition(self, context: JudgeContext) -> JudgeDefinition: + return JudgeDefinition.workflow( + cases=self._build_cases(context), + request=self._build_request, + reduce=self._reduce_verdicts, main_score_name='success_rate', + expand=self._expand_cases ) - def match_score( + def _build_cases(self, context: JudgeContext) -> List[JudgeCase]: + session = self._session(context) + if not session.needs_column_alignment(): + return [] + return [JudgeCase(case_id='column_alignment', output_contract=OutputContract(schema_model=_MappingVerdict))] + + def _build_request( self, - original_prediction: str, - filtered_prediction: str, - reference: str, - task_state: TaskState, - ) -> Score: - raise ValueError("WideSearch requires judge_strategy='auto' or 'llm' with judge_model_args.") - - def _validate_judge_config(self) -> None: - if self.judge_strategy not in {JudgeStrategy.AUTO, JudgeStrategy.LLM}: - raise ValueError("WideSearch requires judge_strategy='auto' or 'llm'.") - if not self._task_config.judge_model_args: - raise ValueError('WideSearch requires judge_model_args for official table alignment and scoring.') + case: JudgeCase, + placement: Any, + completed_cases: Sequence[CaseVerdict], + context: JudgeContext, + ) -> JudgeRequest: + session = self._session(context) + if case.metadata.get('kind') == 'column_score': + response = case.metadata['response'] + prompt = EVAL_COLUMN_PROMPT.format(criterion=case.metadata.get('criterion'), response=response) + else: + prompt = PRIMARY_KEY_PREPROCESS_PROMPT.format( + response=case.metadata.get( + 'response', + session.response_df.columns.tolist() if session.response_df is not None else [] + ), + reference=case.metadata.get('reference', session.required_columns), + ) + prompt += case.output_contract.instruction() + return JudgeRequest(messages=[ChatMessageUser(content=prompt)]) + + def _expand_cases( + self, + stage: int, + completed_cases: Sequence[CaseVerdict], + context: JudgeContext, + ) -> List[JudgeCase]: + session = self._session(context) + column_map = self._mapping(completed_cases, 'column_alignment') + if stage == 1: + answer_df, response_df, _ = session.frames(column_map) + if answer_df is None or response_df is None: + return [] + cases = [] + for column in session.unique_columns: + pipeline = session.evaluation['eval_pipeline'].get(column, {}) + if {'llm_judge', 'exact_match'} & set(pipeline.get('metric', [])): + cases.append( + JudgeCase( + case_id=f'primary_key:{column}', + output_contract=OutputContract(schema_model=_MappingVerdict), + metadata={ + 'response': response_df[column].tolist(), + 'reference': answer_df[column].tolist(), + }, + ) + ) + return cases + if stage != 2: + return [] + primary_key_maps = { + case.case_id.removeprefix('primary_key:'): self._mapping_value(case) + for case in completed_cases + if case.case_id.startswith('primary_key:') + } + inner_df, _ = session.inner_frame(column_map, primary_key_maps) + if inner_df is None or inner_df.empty: + return [] + cases = [] + for column in session.required_columns: + if column in session.unique_columns: + continue + pipeline = session.evaluation['eval_pipeline'][column] + if 'llm_judge' not in pipeline.get('metric', []): + continue + response = { + f'idx_{index}': { + 'response': response_value, + 'target': target_value, + } + for index, (response_value, target_value + ) in enumerate(zip(inner_df[f'{column}_response'], inner_df[f'{column}_query'])) + } + cases.append( + JudgeCase( + case_id=f'column_score:{column}', + output_contract=OutputContract(schema_model=_column_score_model(len(inner_df))), + metadata={ + 'kind': 'column_score', + 'criterion': pipeline.get('criterion'), + 'response': response, + }, + ) + ) + return cases + + def _reduce_verdicts( + self, + case_verdicts: Sequence[CaseVerdict], + context: JudgeContext, + ) -> ReducedVerdict: + session = self._session(context) + column_map = self._mapping(case_verdicts, 'column_alignment') + primary_key_maps = { + case.case_id.removeprefix('primary_key:'): self._mapping_value(case) + for case in case_verdicts + if case.case_id.startswith('primary_key:') + } + column_scores = { + f'{case.case_id.removeprefix("column_score:")}_llm_judge': self._score_values(case) + for case in case_verdicts + if case.case_id.startswith('column_score:') + } + values, diagnostics = session.score(column_scores, column_map, primary_key_maps) + return ReducedVerdict(value=values, metadata=diagnostics) + + @staticmethod + def _mapping(cases: Sequence[CaseVerdict], case_id: str) -> Dict[str, str]: + for case in cases: + if case.case_id == case_id: + return WideSearchAdapter._mapping_value(case) + return {} + + @staticmethod + def _mapping_value(case: CaseVerdict) -> Dict[str, str]: + return {str(key): str(value) for key, value in case.value.mapping.items()} + + @staticmethod + def _score_values(case: CaseVerdict) -> List[float]: + values = case.value.model_dump() + return [float(values[f'idx_{index}']) for index in range(len(values))] + + @staticmethod + def _session(context: JudgeContext) -> WideSearchSession: + return WideSearchSession.create( + prediction=context.filtered_prediction, + gold_csv=context.reference, + evaluation=context.task_state.metadata['evaluation'], + ) def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]: return aggregate_official_scores(sample_scores) + + +class _MappingVerdict(BaseModel): + """A semantic alignment returned through the shared JSON judge contract.""" + + mapping: Dict[str, str] + + +def _column_score_model(size: int) -> type[BaseModel]: + """Build the exact per-row 0/1 output shape for one WideSearch column case.""" + from pydantic import create_model + + return create_model( + f'WideSearchColumnScore{size}', + **{f'idx_{index}': (float, Field(ge=0.0, le=1.0)) + for index in range(size)}, + ) diff --git a/evalscope/benchmarks/world_vqa/world_vqa_adapter.py b/evalscope/benchmarks/world_vqa/world_vqa_adapter.py index 3788341f5..2ccc6f706 100644 --- a/evalscope/benchmarks/world_vqa/world_vqa_adapter.py +++ b/evalscope/benchmarks/world_vqa/world_vqa_adapter.py @@ -6,7 +6,7 @@ from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText from evalscope.api.metric import Score from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.logger import get_logger logger = get_logger() @@ -67,7 +67,7 @@ ) class WorldVQAAdapter(VisionLanguageAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/evalscope/benchmarks/zerobench/zerobench_adapter.py b/evalscope/benchmarks/zerobench/zerobench_adapter.py index 34759cadb..bad7813a7 100644 --- a/evalscope/benchmarks/zerobench/zerobench_adapter.py +++ b/evalscope/benchmarks/zerobench/zerobench_adapter.py @@ -5,7 +5,7 @@ from evalscope.api.dataset import Sample from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText from evalscope.api.registry import register_benchmark -from evalscope.constants import Tags +from evalscope.constants import ScoringPolicy, Tags from evalscope.utils.io_utils import bytes_to_base64, compress_image_to_limit from evalscope.utils.logger import get_logger @@ -63,7 +63,7 @@ ) class ZeroBenchAdapter(VisionLanguageAdapter): - llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/evalscope/collections/schema.py b/evalscope/collections/schema.py index 189929133..700dac42a 100644 --- a/evalscope/collections/schema.py +++ b/evalscope/collections/schema.py @@ -21,6 +21,7 @@ class DatasetInfo: def get_data(self) -> DatasetDict: """Load dataset data using the benchmark registry.""" dataset_args = {self.name: self.args} + # Metadata/data only; this adapter never scores, so it needs no judge configuration. benchmark_meta = get_benchmark(self.name, config=TaskConfig(dataset_args=dataset_args)) data_dict = benchmark_meta.load_dataset() return data_dict diff --git a/evalscope/config.py b/evalscope/config.py index 80a796878..ebf5473bf 100644 --- a/evalscope/config.py +++ b/evalscope/config.py @@ -4,8 +4,8 @@ import json import os from argparse import Namespace -from pydantic import Field, SecretStr, field_validator, model_validator -from typing import Annotated, Any, Dict, List, Optional, Union +from pydantic import ConfigDict, Field, SecretStr, field_validator, model_validator +from typing import Annotated, Any, Dict, List, Literal, Optional, Union from evalscope.agent.external.config import ExternalAgentConfig from evalscope.api.agent import NativeAgentConfig @@ -111,6 +111,93 @@ class SandboxTaskConfig(BaseArgument): when ``None``.""" +class JudgeModelConfig(BaseArgument): + """Transport and identity for one independently weighted judge model. + + Prompt and verdict semantics belong to :class:`JudgeContractConfig`, shared by every + configured judge. Letting model entries carry those fields made a multi-judge run compare + incomparable answers while appearing to aggregate one metric. + """ + + model_config = ConfigDict( + extra='forbid', arbitrary_types_allowed=True, protected_namespaces=(), validate_default=True + ) + + model_id: str + judge_id: Optional[str] = None + api_key: Optional[SecretStr] = None + api_url: Optional[str] = None + eval_type: Optional[str] = None + model_args: Dict[str, Any] = Field(default_factory=dict) + generation_config: Dict[str, Any] = Field(default_factory=dict) + + @field_validator('model_args', mode='before') + @classmethod + def _secretize_model_args(cls, value: Any) -> Any: + return _secretize_api_keys(value) + + +class JudgeContractConfig(BaseArgument): + """Shared semantics for the generic Native Judge contract. + + Custom benchmark contracts keep their official prompt and schema in the adapter; these + fields only configure the generic single-verdict mixin. + """ + + model_config = ConfigDict( + extra='forbid', arbitrary_types_allowed=True, protected_namespaces=(), validate_default=True + ) + + system_prompt: Optional[str] = None + prompt_template: Optional[str] = None + score_mapping: Optional[Dict[str, float]] = None + score_type: Literal['pattern', 'numeric'] = 'pattern' + + +class JudgeConfig(BaseArgument): + """Typed configuration for Native LLM judging.""" + + model_config = ConfigDict( + extra='forbid', arbitrary_types_allowed=True, protected_namespaces=(), validate_default=True + ) + + strategy: Literal['auto', 'rule', 'llm', 'llm_recall'] = JudgeStrategy.AUTO + models: List[JudgeModelConfig] = Field(default_factory=list) + contract: JudgeContractConfig = Field(default_factory=JudgeContractConfig) + repeats: int = Field(default=1, ge=1) + position_swap: Literal['auto', 'on', 'off'] = 'auto' + aggregation: Literal['mean', 'median', 'majority_vote'] = 'mean' + min_valid_judges: int = Field(default=1, ge=1) + + @model_validator(mode='before') + @classmethod + def _normalize_models(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + values = dict(data) + if isinstance(values.get('models'), dict): + values['models'] = [values['models']] + return values + + @model_validator(mode='after') + def _validate_models(self) -> 'JudgeConfig': + seen = set() + model_ids = [model.model_id for model in self.models] + for model in self.models: + if model.judge_id is None: + if model_ids.count(model.model_id) != 1: + raise ValueError( + f'Judge model_id {model.model_id!r} is configured more than once; each entry needs judge_id.' + ) + model.judge_id = model.model_id + if model.judge_id in seen: + raise ValueError(f'duplicate judge_id: {model.judge_id!r}') + seen.add(model.judge_id) + if self.min_valid_judges > len(self.models) and self.models: + raise ValueError('judge.min_valid_judges cannot exceed the configured judge model count.') + return self + + class TaskConfig(BaseArgument): # Model-related arguments model: Optional[Union[str, Model, ModelAPI]] = None @@ -218,20 +305,9 @@ class TaskConfig(BaseArgument): """[Deprecated] Use `generation_config.stream` instead. Will be removed in v2.0.0. When set, value is forwarded to `generation_config.stream` with a deprecation warning.""" - # LLMJudge arguments - judge_strategy: str = JudgeStrategy.AUTO - """How to score model outputs. One of: - - 'auto': follow the benchmark's default (LLM judge only if the benchmark needs one). - - 'rule': force rule-based scoring; never invoke an LLM judge. - - 'llm': force LLM judge for every sample. - - 'llm_recall': run rule-based scoring first, then pass the rule score to the LLM - judge to produce the final score (useful when LLM should refine/recall rule misses).""" - - judge_worker_num: Optional[int] = None - """[Deprecated] Use `eval_batch_size` instead. Will be removed in v2.0.0.""" - - judge_model_args: Optional[Dict] = Field(default_factory=dict) - """Additional arguments for the judge model configuration.""" + # Native judge configuration + judge: JudgeConfig = Field(default_factory=JudgeConfig) + """Typed configuration for Native LLM judging.""" analysis_report: bool = False """Whether to generate detailed analysis reports after evaluation.""" @@ -335,10 +411,40 @@ def _coerce_agent_config(v: Any) -> Any: f'got {type(v).__name__}.' ) - @field_validator('judge_model_args', mode='before') + @model_validator(mode='before') @classmethod - def _validate_judge_model_args(cls, v: Any) -> Any: - return _secretize_api_keys(v) + def _migrate_legacy_judge_config(cls, data: Any) -> Any: + """Convert the legacy single-judge input once, at the public boundary.""" + if not isinstance(data, dict): + return data + values = dict(data) + if 'judge_worker_num' in values: + raise ValueError('`judge_worker_num` has been removed; use `eval_batch_size`.') + legacy_keys = {'judge_strategy', 'judge_model_args'} & set(values) + if 'judge' in values and legacy_keys: + raise ValueError('Use either `judge` or legacy judge_strategy/judge_model_args, not both.') + if legacy_keys: + legacy_args = values.pop('judge_model_args', None) + if isinstance(legacy_args, dict): + legacy_args = dict(legacy_args) + strategy = values.pop('judge_strategy', JudgeStrategy.AUTO) + if legacy_args and 'score_pattern' in legacy_args: + raise ValueError( + '`judge_model_args.score_pattern` is not supported by the JSON judge contract; ' + 'use `judge.contract.score_mapping`.' + ) + deprecated_warning( + logger, + '`judge_strategy` and `judge_model_args` are deprecated; use the typed `judge` configuration.', + ) + semantic_keys = {'system_prompt', 'prompt_template', 'score_mapping', 'score_type'} + contract = {key: legacy_args.pop(key) for key in semantic_keys if legacy_args and key in legacy_args} + values['judge'] = { + 'strategy': strategy, + 'models': [legacy_args] if legacy_args else [], + 'contract': contract, + } + return values @field_validator('sandbox', mode='before') @classmethod @@ -355,13 +461,6 @@ def _post_init(self) -> 'TaskConfig': self._init_default_sandbox_config() self._parse_rag_eval_config() - # Handle deprecated judge_worker_num -> eval_batch_size - if self.judge_worker_num is not None: - deprecated_warning( - logger, 'The `judge_worker_num` parameter is deprecated and will be removed in v2.0.0. ' - 'Use `eval_batch_size` instead.' - ) - return self def _parse_rag_eval_config(self) -> None: @@ -553,6 +652,8 @@ def update(self, other: Union['TaskConfig', dict]) -> None: merged['sandbox'] = self._coerce_sandbox_config(merged['sandbox']) if isinstance(merged.get('agent_config'), dict): merged['agent_config'] = self._coerce_agent_config(merged['agent_config']) + if isinstance(merged.get('judge'), dict): + merged['judge'] = JudgeConfig.model_validate(merged['judge']) for key, value in merged.items(): setattr(self, key, value) diff --git a/evalscope/constants.py b/evalscope/constants.py index 409070eb0..f2cf523e5 100644 --- a/evalscope/constants.py +++ b/evalscope/constants.py @@ -2,6 +2,7 @@ # flake8: noqa import os from datetime import timedelta, timezone +from enum import Enum os.environ['MODELSCOPE_LOG_LEVEL'] = '40' # Set default log level to ERROR @@ -114,11 +115,59 @@ class JudgeStrategy: LLM_RECALL = 'llm_recall' +class ScoringPolicy(str, Enum): + """What a benchmark's own scoring paths can do, declared by its adapter. + + Encodes two orthogonal facts in one value so that the illegal combination -- no usable rule + path yet ``auto`` resolving to rule -- cannot be expressed. Every benchmark always has a + judge path because ``DataAdapter`` inherits the generic grader from ``LLMJudgeMixin``, so + "supports a judge" needs no field. + """ + + RULE_DEFAULT = 'rule_default' + """Usable rule path; ``auto`` scores by rule.""" + + JUDGE_DEFAULT = 'judge_default' + """Usable rule path, but ``auto`` scores with the judge because it is more faithful.""" + + JUDGE_ONLY = 'judge_only' + """No usable rule path: rule scoring would raise or only ever emit zeros.""" + + @property + def rule_supported(self) -> bool: + return self is not ScoringPolicy.JUDGE_ONLY + + @property + def judge_by_default(self) -> bool: + return self is not ScoringPolicy.RULE_DEFAULT + + class JudgeScoreType: NUMERIC = 'numeric' # numeric score PATTERN = 'pattern' # pattern matching score +class ScoreStatus(str, Enum): + """Whether a score is usable, and why it is not. + + A failed judge is not a zero: only ``SUCCESS`` and ``FALLBACK`` carry a score that may + enter aggregation. The remaining values mean the score is unavailable and the sample + must be excluded from the affected metric rather than counted as 0. + """ + + SUCCESS = 'success' + TRANSPORT_ERROR = 'transport_error' + PARSE_ERROR = 'parse_error' + INVALID_SESSION = 'invalid_session' + FALLBACK = 'fallback' + DEGRADED = 'degraded' + EXCLUDED = 'excluded' + + @property + def is_usable(self) -> bool: + return self in (ScoreStatus.SUCCESS, ScoreStatus.FALLBACK, ScoreStatus.DEGRADED) + + class ModelTask: TEXT_GENERATION = 'text_generation' IMAGE_GENERATION = 'image_generation' diff --git a/evalscope/evaluator/evaluator.py b/evalscope/evaluator/evaluator.py index 18f5d51f0..f55c18619 100644 --- a/evalscope/evaluator/evaluator.py +++ b/evalscope/evaluator/evaluator.py @@ -8,6 +8,7 @@ """ import os +import threading import traceback from collections import defaultdict from dataclasses import dataclass @@ -15,9 +16,10 @@ from evalscope.api.dataset import Dataset, Sample from evalscope.api.evaluator import CacheManager, Evaluator, TaskState +from evalscope.api.judge import summarize_judge_runs from evalscope.api.metric import AggScore, SampleScore from evalscope.api.registry import register_evaluator -from evalscope.constants import HEARTBEAT_INTERVAL_SEC +from evalscope.constants import HEARTBEAT_INTERVAL_SEC, ScoreStatus from evalscope.evaluator.batch_reviewer import BatchReviewer from evalscope.evaluator.perf_collector import PerfCollector from evalscope.report import Report, gen_perf_table, gen_table @@ -52,6 +54,9 @@ class _WorkItem: task_state: Optional[TaskState] = None """Cached task state. Set when only review is required.""" + prediction_persisted: bool = False + """Whether a newly generated prediction was durably written before review began.""" + @property def needs_predict(self) -> bool: """True when prediction has not yet been computed.""" @@ -143,6 +148,8 @@ def __init__( # Initialize PerfCollector for collecting per-request performance metrics self.perf_collector = PerfCollector() + self._prediction_cache_lock = threading.Lock() + self._sample_scores_by_subset: Dict[str, List[SampleScore]] = {} def eval(self) -> Report: """ @@ -200,6 +207,7 @@ def eval(self) -> Report: # Phase 3 – aggregate scores per subset (batch review happens here too) agg_score_dict = self._aggregate_scores(dataset_dict, context, results_by_subset) + self.cache_manager.commit_review_reruns() # Phase 4 – generate report if not agg_score_dict: @@ -214,6 +222,9 @@ def eval(self) -> Report: logger.info(f'Benchmark {self.benchmark_name} evaluation finished.') return report + except Exception: + self.cache_manager.discard_review_reruns() + raise finally: self.finalize() @@ -352,9 +363,13 @@ def _process_work_item(self, item: _WorkItem, model_prediction_dir: str) -> Tupl ``(task_state, sample_score)`` where ``sample_score`` is ``None`` for batch-scoring benchmarks (review deferred). """ - task_state = ( - self._predict_sample(item.sample, model_prediction_dir) if item.needs_predict else item.task_state - ) + task_state = self._predict_sample(item.sample, model_prediction_dir) if item.needs_predict else item.task_state + if item.needs_predict: + # A review can fail independently of inference. Persist prediction first so offline + # re-review never has to regenerate an already completed answer. + with self._prediction_cache_lock: + self.cache_manager.save_prediction_cache(item.subset, task_state, self.benchmark.save_metadata) + item.prediction_persisted = True sample_score = (None if self.benchmark.use_batch_scoring else self._review_task_state(task_state)) return task_state, sample_score @@ -376,7 +391,7 @@ def _persist_result( task_state: The completed task state (prediction output). sample_score: The review score, or ``None`` for batch-scoring. """ - if item.needs_predict: + if item.needs_predict and not item.prediction_persisted: model_result = self.cache_manager.save_prediction_cache( item.subset, task_state, self.benchmark.save_metadata ) @@ -455,6 +470,7 @@ def _aggregate_scores( continue logger.info(f'Aggregating scores for subset: {subset}') + self._sample_scores_by_subset[subset] = all_scores agg_score_dict[subset] = self.benchmark.aggregate_scores(sample_scores=all_scores) return agg_score_dict @@ -513,6 +529,7 @@ def get_report(self, agg_score_dict: Dict[str, List[AggScore]]) -> Report: report = self.benchmark.generate_report( scores=agg_score_dict, model_name=self.model_name, output_dir=report_path ) + report.judge_summary = summarize_judge_runs(self._sample_scores_by_subset.values()) # Generate and display a summary table of results try: diff --git a/evalscope/metrics/aggregators/aggregators.py b/evalscope/metrics/aggregators/aggregators.py index d8e3c0d3f..55a07f210 100644 --- a/evalscope/metrics/aggregators/aggregators.py +++ b/evalscope/metrics/aggregators/aggregators.py @@ -1,12 +1,54 @@ # Copyright (c) Alibaba, Inc. and its affiliates. from collections import defaultdict -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional, Tuple from evalscope.api.metric import Aggregator, AggScore, SampleScore from evalscope.api.registry import register_aggregation from evalscope.metrics.utils.functions import calculate_pass_at_k, calculate_pass_hat_k, mean +def collect_metric_names(scores: List[SampleScore]) -> List[str]: + """Collect metric names across all samples, in first-seen order. + + A sample may carry no value for a metric, so the name set cannot be read off ``scores[0]``. + """ + metric_names: Dict[str, None] = {} + for sample_score in scores: + for metric_name in sample_score.score.value: + metric_names.setdefault(metric_name, None) + return list(metric_names) + + +def collect_planned_attempts(scores: List[SampleScore], + metric_name: str) -> Dict[Any, Dict[int, Optional[SampleScore]]]: + """Keep every planned position so unavailable attempts cannot shift later trials forward.""" + grouped: Dict[Any, Dict[int, Optional[SampleScore]]] = defaultdict(dict) + for score in scores: + group_id = score.group_id if score.group_id is not None else score.sample_id + position = score.generation_index + if position is None: + # Legacy caches lack an explicit position. Their existing row order is the only safe + # information available; new scores always carry generation_index. + position = len(grouped[group_id]) + grouped[group_id][position] = score if metric_name in score.score.value else None + return grouped + + +def eligible_prefixes( + grouped: Dict[Any, Dict[int, Optional[SampleScore]]], +) -> Dict[int, List[Tuple[Any, Dict[int, Optional[SampleScore]]]]]: + """Return eligible groups for each k without extending beyond the shortest planned group.""" + max_k = min((max(attempts, default=-1) + 1 for attempts in grouped.values()), default=0) + result: Dict[int, List[Tuple[Any, Dict[int, Optional[SampleScore]]]]] = defaultdict(list) + for group_id, attempts in grouped.items(): + for k in range(1, max_k + 1): + if all(attempts.get(index) is not None for index in range(k)): + result[k].append((group_id, attempts)) + else: + break + return result + + @register_aggregation(name='mean') class Mean(Aggregator): @@ -88,46 +130,34 @@ def __call__(self, scores: List[SampleScore]) -> List[AggScore]: return [] aggregated_scores = Mean()(scores) - metrics = list(scores[0].score.value.keys()) + metrics = collect_metric_names(scores) for metric_name in metrics: - # group_id -> list[float] (0/1 correctness values) - group_values: Dict[Any, List[float]] = defaultdict(list) - for s in scores: - group_id = s.group_id if s.group_id is not None else s.sample_id - value = float(s.score.value[metric_name]) - group_values[group_id].append(value) - - if not group_values: + group_attempts = collect_planned_attempts(scores, metric_name) + prefixes = eligible_prefixes(group_attempts) + if not prefixes: continue - - k = min(len(values) for values in group_values.values()) - - # Prepare inputs for calculate_pass_at_k - num_samples: List[int] = [] - num_correct: List[int] = [] - group_order: List[Any] = [] - for gid, vals in group_values.items(): - group_order.append(gid) - num_samples.append(len(vals)) - num_correct.append(int(sum(vals))) - - # Compute per-group pass@n for all n from 1 to k - pass_at_n_maps = {} - for n in range(1, k + 1): - pass_at_n_list = calculate_pass_at_k(num_samples, num_correct, n) - pass_at_n_maps[n] = {gid: float(v) for gid, v in zip(group_order, pass_at_n_list)} - - for n in range(1, k + 1): - values = list(pass_at_n_maps[n].values()) + for n, eligible in prefixes.items(): + group_order = [group_id for group_id, _ in eligible] + values_by_group = [[ + float(attempt.score.value[metric_name]) for attempt in attempts.values() if attempt is not None + ] for _, attempts in eligible] + values = calculate_pass_at_k([len(items) for items in values_by_group], + [int(sum(items)) for items in values_by_group], n) aggregated_scores.append( AggScore( - score=mean(values), + score=mean(values.tolist()), metric_name=metric_name, aggregation='pass_at_k', dimensions={'k': n}, num=len(values), ids=group_order, + metadata={ + 'eligible': len(values), + 'total': len(group_attempts), + 'coverage': len(values) / len(group_attempts), + 'excluded': len(group_attempts) - len(values), + }, ) ) @@ -166,30 +196,18 @@ def __call__(self, scores: List[SampleScore]) -> List[AggScore]: return [] aggregated_scores = Mean()(scores) - metrics = list(scores[0].score.value.keys()) + metrics = collect_metric_names(scores) for metric_name in metrics: - # Group samples by group_id, preserving order - # Store: (prediction, correctness_score) - group_samples: Dict[Any, List[tuple]] = defaultdict(list) - for score in scores: - group_id = score.group_id if score.group_id is not None else score.sample_id - prediction = getattr(score.score, 'extracted_prediction', None) - correctness = score.score.value[metric_name] - group_samples[group_id].append((prediction, correctness)) - - if not group_samples: + group_attempts = collect_planned_attempts(scores, metric_name) + prefixes = eligible_prefixes(group_attempts) + if not prefixes: continue - - k = min(len(samples) for samples in group_samples.values()) - - # Compute vote@n for all n from 1 to k for each group - vote_at_n_maps: Dict[int, Dict[str, float]] = {} - for n in range(1, k + 1): - vote_at_n_maps[n] = {} - for group_id, samples in group_samples.items(): - # Consider only first n samples for this group - n_samples = samples[:n] + for n, eligible in prefixes.items(): + vote_at_n_map: Dict[Any, float] = {} + for group_id, attempts in eligible: + n_samples = [(attempts[index].score.extracted_prediction, attempts[index].score.value[metric_name]) + for index in range(n)] # Count prediction frequencies prediction_counts = defaultdict(int) @@ -204,10 +222,9 @@ def __call__(self, scores: List[SampleScore]) -> List[AggScore]: pred == most_frequent_pred and correctness == 1.0 for pred, correctness in n_samples ) - vote_at_n_maps[n][group_id] = 1.0 if is_correct else 0.0 + vote_at_n_map[group_id] = 1.0 if is_correct else 0.0 - for n in range(1, k + 1): - values = list(vote_at_n_maps[n].values()) + values = list(vote_at_n_map.values()) aggregated_scores.append( AggScore( score=mean(values), @@ -215,7 +232,13 @@ def __call__(self, scores: List[SampleScore]) -> List[AggScore]: aggregation='vote_at_k', dimensions={'k': n}, num=len(values), - ids=list(group_samples), + ids=list(vote_at_n_map), + metadata={ + 'eligible': len(values), + 'total': len(group_attempts), + 'coverage': len(values) / len(group_attempts), + 'excluded': len(group_attempts) - len(values), + }, ) ) @@ -242,32 +265,22 @@ def __call__(self, scores: List[SampleScore]) -> List[AggScore]: return [] aggregated_scores = Mean()(scores) - metrics = list(scores[0].score.value.keys()) + metrics = collect_metric_names(scores) for metric_name in metrics: - # group_id -> list[float] (0/1 correctness values) - group_values: Dict[Any, List[float]] = defaultdict(list) - for s in scores: - group_id = s.group_id if s.group_id is not None else s.sample_id - value = float(s.score.value[metric_name]) - group_values[group_id].append(value) - - if not group_values: + group_attempts = collect_planned_attempts(scores, metric_name) + prefixes = eligible_prefixes(group_attempts) + if not prefixes: continue - - k = min(len(values) for values in group_values.values()) - - # Compute per-group pass^n for all n from 1 to k - pass_hat_n_maps: Dict[int, Dict[str, float]] = {} - for n in range(1, k + 1): - pass_hat_n_maps[n] = {} - for gid, vals in group_values.items(): - total = len(vals) - correct = int(sum(vals)) - pass_hat_n_maps[n][gid] = float(calculate_pass_hat_k(total, correct, n)) - - for n in range(1, k + 1): - values = list(pass_hat_n_maps[n].values()) + for n, eligible in prefixes.items(): + values = [] + ids = [] + for group_id, attempts in eligible: + attempt_values = [ + float(attempt.score.value[metric_name]) for attempt in attempts.values() if attempt is not None + ] + values.append(float(calculate_pass_hat_k(len(attempt_values), int(sum(attempt_values)), n))) + ids.append(group_id) aggregated_scores.append( AggScore( score=mean(values), @@ -275,7 +288,13 @@ def __call__(self, scores: List[SampleScore]) -> List[AggScore]: aggregation='pass_hat_k', dimensions={'k': n}, num=len(values), - ids=list(group_values), + ids=ids, + metadata={ + 'eligible': len(values), + 'total': len(group_attempts), + 'coverage': len(values) / len(group_attempts), + 'excluded': len(group_attempts) - len(values), + }, ) ) diff --git a/evalscope/metrics/judge/__init__.py b/evalscope/metrics/judge/__init__.py index 133dd36da..bc2aabed1 100644 --- a/evalscope/metrics/judge/__init__.py +++ b/evalscope/metrics/judge/__init__.py @@ -1,14 +1,8 @@ # Copyright (c) Alibaba, Inc. and its affiliates. -from .base import BaseJudge from .llm_judge import DEFAULT_JUDGE_MODEL, DEFAULT_NUMERIC_SCORE_TEMPLATE, DEFAULT_PROMPT_TEMPLATE, LLMJudge -from .score_extractors import NumericScoreExtractor, PatternScoreExtractor, ScoreExtractor __all__ = [ - 'BaseJudge', 'LLMJudge', - 'ScoreExtractor', - 'PatternScoreExtractor', - 'NumericScoreExtractor', 'DEFAULT_PROMPT_TEMPLATE', 'DEFAULT_NUMERIC_SCORE_TEMPLATE', 'DEFAULT_JUDGE_MODEL', diff --git a/evalscope/metrics/judge/base.py b/evalscope/metrics/judge/base.py deleted file mode 100644 index e23fc6c6b..000000000 --- a/evalscope/metrics/judge/base.py +++ /dev/null @@ -1,25 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List, Optional - -from evalscope.api.messages import ChatMessage - - -class BaseJudge(ABC): - """Abstract base class for all judge implementations.""" - - @abstractmethod - def judge( - self, - prompt: str = '', - system_prompt: Optional[str] = None, - messages: Optional[List[ChatMessage]] = None - ) -> str: - """Generate a judgment response.""" - - @abstractmethod - def build_prompt(self, pred: str, gold: str, question: Optional[str] = None) -> str: - """Build the evaluation prompt.""" - - @abstractmethod - def get_score(self, response: str) -> float: - """Extract a numeric score from the judge response.""" diff --git a/evalscope/metrics/judge/llm_judge.py b/evalscope/metrics/judge/llm_judge.py index 9f212f7e6..d479b4db7 100644 --- a/evalscope/metrics/judge/llm_judge.py +++ b/evalscope/metrics/judge/llm_judge.py @@ -1,20 +1,16 @@ import os from typing import Any, Dict, List, Optional -from evalscope.api.messages import ChatMessage, ChatMessageSystem, ChatMessageUser +from evalscope.api.messages import ChatMessage +from evalscope.api.model import ModelOutput from evalscope.constants import EvalType, JudgeScoreType from evalscope.utils.logger import get_logger -from .base import BaseJudge -from .score_extractors import NumericScoreExtractor, PatternScoreExtractor, ScoreExtractor logger = get_logger() -# Sentinel that ``judge`` returns instead of raising on a failed request. -# Consumers must fail closed on it (score 0, never full credit) — the same -# contract as ``evalscope/benchmarks/hipho/utils.py#JUDGE_ERROR_PREFIX``. -JUDGE_ERROR_PREFIX = '[ERROR]' - -DEFAULT_PROMPT_TEMPLATE = """Your job is to look at a question, a gold target, and a predicted answer, and return a letter "A" or "B" to indicate whether the predicted answer is correct or incorrect. +# The templates state the grading criteria only. The reply format is appended by +# ``OutputContract.instruction()`` so the prompt and the parser cannot drift apart. +DEFAULT_PROMPT_TEMPLATE = """Your job is to look at a question, a gold target, and a predicted answer, and decide whether the predicted answer is correct or incorrect. [Question] {question} @@ -28,28 +24,24 @@ Evaluate the model's answer based on correctness compared to the reference answer. Grade the predicted answer of this new question as one of: A: CORRECT -B: INCORRECT - -Just return the letters "A" or "B", with no text around it. -""" # noqa: E501 +B: INCORRECT""" # noqa: E501 DEFAULT_NUMERIC_SCORE_TEMPLATE = """Please act as an impartial judge and evaluate the quality of the response provided by an AI assistant to the user question displayed below. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of the response. Begin your evaluation by providing a short explanation. Be as objective as possible. -After providing your explanation, you must rate the response on a scale of 0 (worst) to 1 (best) by strictly following this format: \"[[rating]]\", for example: \"Rating: [[0.5]]\" +Rate the response on a scale of 0 (worst) to 1 (best). [Question] {question} [Response] -{pred} -""" # noqa: E501 +{pred}""" # noqa: E501 DEFAULT_JUDGE_MODEL = 'Qwen/Qwen3-235B-A22B' DEFAULT_API_URL = 'https://api-inference.modelscope.cn/v1/' -class LLMJudge(BaseJudge): +class LLMJudge: """ A metric that uses LLM to judge the quality of model predictions by comparing them with reference answers. """ @@ -64,10 +56,8 @@ def __init__( system_prompt: Optional[str] = None, prompt_template: Optional[str] = None, generation_config: Optional[Dict[str, Any]] = None, - score_pattern: Optional[str] = None, score_mapping: Optional[Dict[str, float]] = None, score_type: str = JudgeScoreType.PATTERN, # 'pattern', 'numeric' - **kwargs ): """ Initialize LLMJudge metric. @@ -81,11 +71,10 @@ def __init__( system_prompt (str, optional): System prompt for the judge prompt_template (str, optional): Prompt template for the judge generation_config (dict, optional): Generation configuration for the judge - score_pattern (str, optional): Regex pattern to extract score from LLM response - score_mapping (dict, optional): Mapping from extracted score to float value - score_type (str, optional): Type of score extraction strategy ('pattern', 'numeric') defaults to 'pattern'. - - 'pattern': Use score_pattern and score_mapping to extract categorical scores - - 'numeric': Treat the extracted value as a direct numerical score + score_mapping (dict, optional): Allowed verdict labels mapped to their score values + score_type (str, optional): Which built-in judge contract to use, defaults to 'pattern'. + - 'pattern': grade against a reference answer, choosing one of the score_mapping labels + - 'numeric': rate the response on a 0-1 scale without a reference """ self.api_key = api_key or os.environ.get('MODELSCOPE_SDK_TOKEN', 'EMPTY') self.api_url = api_url or os.environ.get('MODELSCOPE_API_BASE', DEFAULT_API_URL) @@ -95,24 +84,16 @@ def __init__( self.generation_config = generation_config or {'temperature': 0.0, 'max_tokens': 4096} self.model_args = model_args or {} - # Build score extractor based on score_type (Strategy pattern) self.score_type = score_type if self.score_type == JudgeScoreType.NUMERIC: - pattern = score_pattern or r'\[\[(\d+(?:\.\d+)?)\]\]' self.prompt_template = prompt_template or os.environ.get( 'JUDGE_PROMPT_TEMPLATE', DEFAULT_NUMERIC_SCORE_TEMPLATE ) - self._score_extractor: ScoreExtractor = NumericScoreExtractor(pattern=pattern) elif self.score_type == JudgeScoreType.PATTERN: - # Anchor to only accept a standalone A or B (avoid false positives) - pattern = score_pattern or r'^\s*([AB])\s*$' self.prompt_template = prompt_template or os.environ.get('JUDGE_PROMPT_TEMPLATE', DEFAULT_PROMPT_TEMPLATE) - self._score_extractor = PatternScoreExtractor(pattern=pattern, score_mapping=score_mapping) else: raise ValueError(f"Invalid score_type: {self.score_type}. Must be 'pattern' or 'numeric'.") - # Keep score_pattern and score_mapping as public attrs for backward compatibility - self.score_pattern = pattern self.score_mapping = score_mapping or {'A': 1.0, 'B': 0.0} self._init_server_adapter() @@ -129,42 +110,14 @@ def _init_server_adapter(self) -> None: model_args=self.model_args, ) - def judge( - self, - prompt: str = '', - system_prompt: Optional[str] = None, - messages: Optional[List[ChatMessage]] = None - ) -> str: - """ - Generate a response from the LLM based on the provided prompt and context. - If messages is provided, it will be used as the input context. + def generate(self, messages: List[ChatMessage]) -> ModelOutput: + """Run one judge request and preserve the provider response unchanged. - Args: - prompt (str): The prompt to evaluate - system_prompt (str, optional): The system prompt to use for the evaluation - messages (List[ChatMessage], optional): A list of chat messages to include in the evaluation - Returns: - str: The response from the LLM + Transport failures deliberately propagate to the executor, where they become typed + ``transport_error`` attempts. Returning a magic string here used to make failures look + like malformed judge verdicts or, worse, valid zero scores. """ - # parse messages - if messages is not None: - input_messages = messages - else: - system_content = system_prompt or self.system_prompt - input_messages = [ChatMessageUser(content=prompt)] - if system_content: - input_messages.insert(0, ChatMessageSystem(content=system_content)) - try: - # Send request using ServerModelAdapter - response = self.model.generate(input_messages) - - # Extract content from response - llm_response = response.completion - return llm_response - except Exception as e: - error_message = f'Error occurred during {self.model_id}@{self.api_url} LLM judge evaluation: {e}' - logger.error(error_message) - return f'{JUDGE_ERROR_PREFIX} {error_message}' + return self.model.generate(messages) def build_prompt(self, pred: str, gold: str, question: Optional[str] = None) -> str: if question is None: @@ -179,24 +132,3 @@ def build_prompt(self, pred: str, gold: str, question: Optional[str] = None) -> if '{gold}' in self.prompt_template: prompt = prompt.replace('{gold}', gold) return prompt - - def get_score(self, response: str) -> float: - """ - Extract score from LLM response using the configured extractor strategy. - - Args: - response (str): The response from the LLM - - Returns: - float: The numeric score extracted from the response - """ - if response is None: - return 0.0 - # A failed judge request must score 0, never full credit: ``judge`` reports - # failures as an ``[ERROR] ...`` string whose embedded digits (model id, - # endpoint, HTTP code) could otherwise be parsed as a score by a loose - # pattern. Same guard as ``benchmarks/hipho/utils.py``. - if response.startswith(JUDGE_ERROR_PREFIX): - logger.warning(f'LLM judge failed; returning 0.0. Response: {response}') - return 0.0 - return self._score_extractor.extract(response) diff --git a/evalscope/metrics/judge/score_extractors.py b/evalscope/metrics/judge/score_extractors.py deleted file mode 100644 index 34118cd18..000000000 --- a/evalscope/metrics/judge/score_extractors.py +++ /dev/null @@ -1,94 +0,0 @@ -import math -import re -from abc import ABC, abstractmethod -from typing import Dict, Optional - -from evalscope.utils.logger import get_logger - -logger = get_logger() - - -class ScoreExtractor(ABC): - """Strategy interface for extracting scores from judge responses.""" - - @abstractmethod - def extract(self, response: str) -> float: - """Extract a numeric score from a text response.""" - - -class PatternScoreExtractor(ScoreExtractor): - """Extract categorical scores via regex pattern + mapping (e.g. A->1.0, B->0.0).""" - - def __init__(self, pattern: str, score_mapping: Optional[Dict[str, float]] = None): - self.pattern = pattern - self.score_mapping = score_mapping or {'A': 1.0, 'B': 0.0} - - def extract(self, response: str) -> float: - """Use the score_pattern to extract categorical scores.""" - # strict standalone A/B matching using MULTILINE to handle simple outputs - match = re.search(self.pattern, response, re.MULTILINE) - if match: - answer = match.group(1) if match.lastindex else match.group(0).strip() - if answer not in self.score_mapping: - logger.warning( - f"Matched '{answer}' for pattern '{self.pattern}' but no score mapping exists; " - f'returning 0.0. Response: {response}' - ) - return 0.0 - return self.score_mapping[answer] - else: - logger.warning(f"No match found for pattern '{self.pattern}' in response: {response}") - return 0.0 - - -class NumericScoreExtractor(ScoreExtractor): - """Extract numeric scores directly from response (e.g. [[0.75]]).""" - - def __init__(self, pattern: str, clamp_min: float = 0.0, clamp_max: float = 1.0): - self.pattern = pattern - self.clamp_min = clamp_min - self.clamp_max = clamp_max - - def extract(self, response: str) -> float: - """Extract numeric score from the response using the score_pattern.""" - # Find all numeric tokens like [[0.5]] and take the last one (most decisive) - matches = list(re.finditer(self.pattern, response)) - if not matches: - logger.warning(f"No match found for pattern '{self.pattern}' in response: {response}") - return 0.0 - - def _clamped(val: float) -> float: - """Clamp to [clamp_min, clamp_max], warning when the raw value was out of range.""" - # NaN compares false against every bound, so it would slip through the range - # check below and propagate into aggregation, turning the whole metric into - # NaN. Fail closed at clamp_min instead, as documented for [ERROR] responses. - if math.isnan(val): - logger.warning(f'Score NaN is not a usable rating, returning {self.clamp_min} in response: {response}') - return self.clamp_min - if val < self.clamp_min or val > self.clamp_max: - clamped = max(self.clamp_min, min(self.clamp_max, val)) - logger.warning( - f'Score {val} out of range [{self.clamp_min}, {self.clamp_max}], ' - f'clamped to {clamped} in response: {response}' - ) - return clamped - return val - - # iterate from last to first to pick the final rating - for match in reversed(matches): - # prefer captured groups - for group in match.groups(): - if group is None: - continue - try: - return _clamped(float(group)) - except (ValueError, TypeError): - continue - # fallback: try entire match if groups fail - try: - return _clamped(float(match.group(0))) - except (ValueError, TypeError): - continue - - logger.warning(f'Failed to convert extracted values to float in response: {response}') - return 0.0 diff --git a/evalscope/metrics/semantics/catalog.py b/evalscope/metrics/semantics/catalog.py index 8455e96a7..a40f60db8 100644 --- a/evalscope/metrics/semantics/catalog.py +++ b/evalscope/metrics/semantics/catalog.py @@ -179,10 +179,10 @@ 'param_default_accept_rate': MetricEntry(baseline='quality.accuracy.ratio'), # --- diagnostics: distribution shares and raw counts carry no direction -------------- 'error_rate': MetricEntry(baseline='diagnostic.parse_status.ratio'), - 'is_incorrect': MetricEntry(baseline='diagnostic.parse_status.ratio'), - 'is_not_attempted': MetricEntry(baseline='diagnostic.parse_status.ratio'), - 'mean_is_incorrect': MetricEntry(baseline='diagnostic.parse_status.ratio'), - 'mean_is_not_attempted': MetricEntry(baseline='diagnostic.parse_status.ratio'), + 'is_incorrect': MetricEntry(baseline='diagnostic.parse_status.ratio', display_name='Incorrect rate'), + 'is_not_attempted': MetricEntry(baseline='diagnostic.parse_status.ratio', display_name='Not attempted rate'), + 'mean_is_incorrect': MetricEntry(baseline='diagnostic.parse_status.ratio', display_name='Incorrect rate'), + 'mean_is_not_attempted': MetricEntry(baseline='diagnostic.parse_status.ratio', display_name='Not attempted rate'), 'inference_error_rate': MetricEntry(baseline='diagnostic.parse_status.ratio'), 'yes_ratio': MetricEntry(baseline='diagnostic.parse_status.ratio'), 'maybe_ratio': MetricEntry(baseline='diagnostic.parse_status.ratio'), @@ -415,6 +415,10 @@ for name in names } METRIC_DEFINITIONS.update({ + # Three-way answer grading exposes these distribution shares as diagnostics. They remain + # non-primary and directionless, but deserve report labels rather than internal identities. + 'is_incorrect': MetricEntry(baseline='diagnostic.parse_status.ratio', display_name='Incorrect rate'), + 'is_not_attempted': MetricEntry(baseline='diagnostic.parse_status.ratio', display_name='Not attempted rate'), 'total_model_time': MetricEntry(baseline='diagnostic.unspecified', raw_unit='s', display_precision=2), 'total_other_time': MetricEntry(baseline='diagnostic.unspecified', raw_unit='s', display_precision=2), 'total_tool_time': MetricEntry(baseline='diagnostic.unspecified', raw_unit='s', display_precision=2), diff --git a/evalscope/metrics/semantics/entry.py b/evalscope/metrics/semantics/entry.py index 1842b0597..dab985bd4 100644 --- a/evalscope/metrics/semantics/entry.py +++ b/evalscope/metrics/semantics/entry.py @@ -20,6 +20,7 @@ #: to the baseline vocabulary rather than to each catalog entry. _ENTRY_OVERRIDE_FIELDS = ( 'metric_name', + 'display_name', 'direction', 'raw_unit', 'value_range', @@ -58,6 +59,7 @@ class MetricEntry(BaseModel): """Key into the baseline table.""" metric_name: Optional[str] = None + display_name: Optional[str] = None direction: Optional[MetricDirection] = None raw_unit: Optional[str] = None value_range: Optional[ValueRange] = None diff --git a/evalscope/metrics/semantics/formatting.py b/evalscope/metrics/semantics/formatting.py index a189d8e03..6e88226d0 100644 --- a/evalscope/metrics/semantics/formatting.py +++ b/evalscope/metrics/semantics/formatting.py @@ -158,8 +158,8 @@ def format_metric_label( ) -> str: """Render one identity through its display name, direction, and meaningful dimensions. - Diagnostic and unresolved metrics keep their final report name because their shared generic - baselines must not erase what was actually measured. + Diagnostic and unresolved metrics keep their final report name unless their declared + semantics provide an explicit diagnostic display name. Args: identity: V2 identity, or a v1 string during migration. @@ -175,8 +175,10 @@ def format_metric_label( else: raw_name = identity.key dimensions = identity.dimensions - if semantics is None or semantics.kind is MetricKind.DIAGNOSTIC: + if semantics is None: return legacy_name or raw_name + if semantics.kind is MetricKind.DIAGNOSTIC: + return semantics.display_name or legacy_name or raw_name arrow = _DIRECTION_ARROWS.get(semantics.direction, '') label = f'{semantics.metric_name} {arrow}'.strip() if dimensions: diff --git a/evalscope/report/renderer.py b/evalscope/report/renderer.py index 9b9cda50c..9df9a0061 100644 --- a/evalscope/report/renderer.py +++ b/evalscope/report/renderer.py @@ -237,7 +237,7 @@ def gen_html_report_file( model=model, metric=metric_labels.get(primary_metric.name, primary_metric.name) if primary_metric else 'N/A', metric_raw=primary_metric.name if primary_metric else 'N/A', - score=score if score is not None else 0.0, + score=score, score_display=format_metric_value(score, semantics), num=primary_metric.num if primary_metric else 0, ) @@ -268,7 +268,7 @@ def gen_html_report_file( info = dataset_info[ds] pretty = info['pretty_name'] model_sections: List[dict] = [] - overall_score = 0.0 + overall_score: Optional[float] = None overall_score_display = format_metric_value(overall_score, None) for model in all_models: @@ -276,7 +276,7 @@ def gen_html_report_file( if rpt is None: continue primary_metric = rpt.primary_metric - overall_score = primary_metric.score if primary_metric else 0.0 + overall_score = primary_metric.score if primary_metric else None overall_score_display = format_metric_value( primary_metric.score if primary_metric else None, primary_metric.semantics if primary_metric else None, diff --git a/evalscope/report/report.py b/evalscope/report/report.py index cf44e0430..a851f3309 100644 --- a/evalscope/report/report.py +++ b/evalscope/report/report.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from typing_extensions import Self +from evalscope.api.metric import JudgeSummary from evalscope.api.metric.semantics import MetricIdentity, MetricKind, MetricSemantics from evalscope.metrics import macro_mean, micro_mean from evalscope.utils import get_logger @@ -173,6 +174,8 @@ class Report(BaseModel): # compare=False equivalent: excluded from model equality via model_config perf_metrics: Optional[Dict[str, Any]] = Field(default=None) primary_metric_identity: Optional[MetricIdentity] = None + judge_summary: Optional[JudgeSummary] = None + """Run-level Native Judge coverage and failure summary, when this report used a judge.""" @model_validator(mode='before') @classmethod @@ -223,14 +226,15 @@ def primary_metric(self) -> Optional[Metric]: return self._find_primary_metric() @property - def score(self) -> float: + def score(self) -> Optional[float]: """Compatibility score derived from the primary or first available metric. - Report v2 serializes the structured metric list and primary identity instead of this - convenience value. + ``None`` when the report carries no metric: a report that produced no usable score is + not a report that scored zero. Report v2 serializes the structured metric list and + primary identity instead of this convenience value. """ metric = self._find_primary_metric() or (self.metrics[0] if self.metrics else None) - return metric.score if metric is not None else 0.0 + return metric.score if metric is not None else None def to_dict(self) -> Dict[str, Any]: # model_dump includes computed_field 'num' automatically @@ -341,10 +345,11 @@ def generate_analysis(self, task_config: 'TaskConfig') -> str: try: language = 'English' if DEFAULT_LANGUAGE == 'en' else 'Chinese' - # Use judge_model_args if configured; otherwise fall back to the task's own model settings - if task_config.judge_model_args: - judge_model_args = get_secret_value(task_config.judge_model_args) - judge_llm = LLMJudge(**judge_model_args) + # Reuse the primary judge's transport configuration without making analysis part of + # the scoring process. + if task_config.judge.models: + judge_model_config = task_config.judge.models[0].model_dump(exclude={'judge_id'}, exclude_none=True) + judge_llm = LLMJudge(**get_secret_value(judge_model_config)) else: judge_llm = LLMJudge( api_key=get_secret_value(task_config.api_key), @@ -354,16 +359,14 @@ def generate_analysis(self, task_config: 'TaskConfig') -> str: ) prompt = ANALYSIS_PROMPT.format(language=language, report_str=self.to_json_str()) - response = judge_llm.judge(prompt) - if response.startswith('[ERROR]'): - logger.warning(f'Analysis generation failed, skipping: {response}') - response = 'N/A' + from evalscope.api.messages import ChatMessageUser + + response = judge_llm.generate([ChatMessageUser(content=prompt)]).completion + if DEFAULT_LANGUAGE == 'en': + disclaimer = f'> *Generated by {judge_llm.model_id}, for reference only.*' else: - if DEFAULT_LANGUAGE == 'en': - disclaimer = f'> *Generated by {judge_llm.model_id}, for reference only.*' - else: - disclaimer = f'> *由 {judge_llm.model_id} 生成，仅供参考。*' - response = f'{disclaimer}\n\n{response}' + disclaimer = f'> *由 {judge_llm.model_id} 生成，仅供参考。*' + response = f'{disclaimer}\n\n{response}' except Exception as e: logger.error(f'Error generating analysis: {e}') response = 'N/A' diff --git a/evalscope/run.py b/evalscope/run.py index a35c98e13..5afc7372e 100644 --- a/evalscope/run.py +++ b/evalscope/run.py @@ -144,6 +144,9 @@ def evaluate_model(task_config: TaskConfig, outputs: OutputsStructure) -> dict: for dataset_name in task_config.datasets: # Create evaluator for each dataset benchmark = get_benchmark(dataset_name, task_config) + # Validate only on the execution path. ``get_benchmark`` is also a metadata helper and + # must not reject a configuration before an evaluation has been requested. + benchmark.validate_judge_strategy() evaluator = create_evaluator( benchmark=benchmark, model=model, diff --git a/evalscope/utils/data_utils.py b/evalscope/utils/data_utils.py index 0a9a39d53..523e15f6c 100644 --- a/evalscope/utils/data_utils.py +++ b/evalscope/utils/data_utils.py @@ -405,6 +405,7 @@ def _build_prediction_row( prediction = score.prediction extracted_prediction = score.extracted_prediction + main_value = score.main_value return { 'Index': str(review_result.index), @@ -415,7 +416,10 @@ def _build_prediction_row( 'Pred': (extracted_prediction if extracted_prediction != prediction else '*Same as Generated*') or '', # Ensure no None value 'Score': score.model_dump(exclude_none=True), - 'NScore': normalize_score(score.main_value), + # ``None`` when the sample carries no usable value: a judge that could not be + # scored is not a sample that scored 0. + 'NScore': normalize_score(main_value) if main_value is not None else None, + 'Status': score.status.value, 'PerfMetrics': fallback_perf, 'Messages': messages_data, 'AgentTrace': review_result.agent_trace.model_dump(exclude_none=True) if review_result.agent_trace else None, @@ -442,7 +446,7 @@ def get_model_prediction(work_dir: str, model_name: str, dataset_name: str, subs ds = [] for cache in review_caches: - review_result = ReviewResult.model_validate(cache) + review_result = ReviewResult.from_cache_item(cache) sample_score = review_result.sample_score # For DataCollection, filter to the requested subset diff --git a/evalscope/web/src/api/schemas/reports.schema.ts b/evalscope/web/src/api/schemas/reports.schema.ts index c10808181..6495cc3d1 100644 --- a/evalscope/web/src/api/schemas/reports.schema.ts +++ b/evalscope/web/src/api/schemas/reports.schema.ts @@ -94,6 +94,21 @@ export const perfMetricsSchema = z.object({ metric_semantics: z.record(z.string(), metricSemanticsSchema), }) +/** First-class judge execution summary. Mirrors `JudgeSummary`. */ +export const judgeSummarySchema = z.object({ + status: z.string().optional(), + scored: z.number().optional(), + total: z.number().optional(), + coverage: z.number().optional(), + judge_models: z.array(z.string()).optional(), + valid_observations: z.number().optional(), + total_observations: z.number().optional(), + /** Attempt counts keyed by `ScoreStatus` value. */ + failures: z.record(z.string(), z.number()).optional(), + disagreement: z.record(z.string(), z.unknown()).optional(), + error: z.string().optional(), +}) + /** Runtime contract for one dataset report. */ export const reportDataSchema = z.object({ schema_version: z.literal(2), @@ -107,6 +122,7 @@ export const reportDataSchema = z.object({ // than omitting it. Both shapes are part of the backend contract. perf_metrics: perfMetricsSchema.nullable().optional(), primary_metric_identity: metricIdentitySchema.nullable(), + judge_summary: judgeSummarySchema.optional(), }) /** Runtime contract for a report detail response. */ @@ -248,6 +264,45 @@ export const agentTraceSchema = z.object({ events: z.array(agentTraceEventSchema), }) +/** One judge request/response round trip, including the ones that failed. Mirrors `JudgeAttempt`. */ +export const judgeAttemptSchema = z.object({ + /** Mirrors `ScoreStatus`: success, transport_error, parse_error, invalid_session, fallback, excluded. */ + status: z.string(), + case_id: z.string(), + judge_id: z.string(), + /** Which planned observation this attempt belongs to. */ + repeat_id: z.number().optional(), + /** `original` or `swapped` for a pairwise benchmark that judges both orders. */ + placement: z.string().optional(), + messages: z.array(z.unknown()).optional(), + model_output: z.unknown().optional(), + raw_response: z.string().optional(), + /** The parsed verdict; its shape is the benchmark's own `schema_model`. */ + parsed_value: z.unknown().optional(), + error: z.string().optional(), + latency: z.number().optional(), +}) + +/** + * Mirrors `Score`. Left open with `loose` because `value` and `metadata` keys are + * benchmark-defined; only the fields the UI reads are typed. + */ +export const predictionScoreSchema = z.looseObject({ + value: z.record(z.string(), z.unknown()).optional(), + /** Mirrors `ScoreStatus`; absent in reports produced before it existed. */ + status: z.string().optional(), + judge_summary: judgeSummarySchema.optional(), + explanation: z.string().optional(), + main_score_name: z.string().optional(), + metadata: z + .looseObject({ + judge_attempts: z.array(judgeAttemptSchema).optional(), + judge_skipped: z.boolean().optional(), + judge_skip_reason: z.string().optional(), + }) + .optional(), +}) + /** Mirrors `PredictionRow`. */ export const predictionRowSchema = z.object({ Index: z.string(), @@ -256,8 +311,11 @@ export const predictionRowSchema = z.object({ Generated: z.string(), Gold: z.string(), Pred: z.string(), - Score: z.record(z.string(), z.unknown()), - NScore: z.number(), + Score: predictionScoreSchema, + /** ``null`` when the sample has no usable score; it is excluded from the metric, not 0. */ + NScore: z.number().nullable(), + /** Mirrors ``ScoreStatus``; absent in reports produced before it existed. */ + Status: z.string().optional(), PerfMetrics: samplePerfMetricsSchema.nullable().optional(), Messages: z.array(chatMessageSchema).nullable().optional(), AgentTrace: agentTraceSchema.nullable().optional(), @@ -294,6 +352,9 @@ export type ToolCall = z.infer export type ChatMessage = z.infer export type AgentTraceEvent = z.infer export type AgentTrace = z.infer +export type JudgeAttempt = z.infer +export type JudgeSummary = z.infer +export type PredictionScore = z.infer export type PredictionRow = z.infer export type PredictionsResponse = z.infer export type DeleteReportResponse = z.infer diff --git a/evalscope/web/src/api/types/reports.ts b/evalscope/web/src/api/types/reports.ts index d34a4be55..784263f99 100644 --- a/evalscope/web/src/api/types/reports.ts +++ b/evalscope/web/src/api/types/reports.ts @@ -22,6 +22,10 @@ export type { ChatMessage, AgentTraceEvent, AgentTrace, + // LLM judge diagnostics carried on a score + JudgeAttempt, + JudgeSummary, + PredictionScore, PredictionRow, PredictionsResponse, DeleteReportResponse, diff --git a/evalscope/web/src/components/chat/ChatView.tsx b/evalscope/web/src/components/chat/ChatView.tsx index be7adc64f..c094329e6 100644 --- a/evalscope/web/src/components/chat/ChatView.tsx +++ b/evalscope/web/src/components/chat/ChatView.tsx @@ -5,6 +5,8 @@ import { MessageRow, SystemPromptRow, HeaderPerfChip } from './MessageComponents import { StructuredMessages, TracedTimeline } from './AgentTraceView' import { buildStepGroups } from '@/domain/trace/stepGroups' import { EvalResultPanel } from './EvalResultPanel' +import { JudgeReviewPanel } from './JudgeReviewPanel' +import { selectJudgeReview, scoreWithoutJudgeAttempts } from '@/domain/chat/judgeReview' type ChatPresentation = 'traced' | 'structured' | 'legacy' @@ -62,6 +64,11 @@ export default function ChatView({ prediction, threshold = 0.99, highlightMsgId const presentation = selectChatPresentation(prediction) const [highlightedStep, setHighlightedStep] = useState(null) + const judgeReview = useMemo(() => selectJudgeReview(prediction.Score), [prediction.Score]) + // The raw judge replies are rendered by the review panel below; drop them from the Score Detail + // JSON so a rubric sample's 100 KB of attempts is not shown twice. + const scoreForDetail = useMemo(() => scoreWithoutJudgeAttempts(prediction.Score), [prediction.Score]) + const stepGroups = useMemo(() => { if (presentation !== 'traced' || !messages || !agentTrace) return null return buildStepGroups(messages, agentTrace) @@ -96,11 +103,14 @@ export default function ChatView({ prediction, threshold = 0.99, highlightMsgId pred={prediction.Pred} gold={prediction.Gold} nScore={prediction.NScore} - score={prediction.Score} + status={prediction.Status} + score={scoreForDetail} metadata={prediction.Metadata} threshold={threshold} showPred={!!showPred} /> + + {judgeReview && } ) } diff --git a/evalscope/web/src/components/chat/EvalResultPanel.test.tsx b/evalscope/web/src/components/chat/EvalResultPanel.test.tsx index 86f1fdef2..e64498b51 100644 --- a/evalscope/web/src/components/chat/EvalResultPanel.test.tsx +++ b/evalscope/web/src/components/chat/EvalResultPanel.test.tsx @@ -6,7 +6,7 @@ import { EvalResultPanel } from './EvalResultPanel' afterEach(cleanup) -function renderPanel(nScore: number) { +function renderPanel(nScore: number | null, status?: string) { return render( @@ -14,7 +14,8 @@ function renderPanel(nScore: number) { pred="B" gold="B" nScore={nScore} - score={{ acc: nScore }} + status={status} + score={nScore === null ? {} : { acc: nScore }} metadata={{ id: '1' }} threshold={0.99} showPred @@ -40,4 +41,22 @@ describe('EvalResultPanel', () => { expect(screen.queryByText('"acc"')).not.toBeInTheDocument() expect(screen.getByText('20%')).toHaveStyle({ color: 'var(--danger)' }) }) + + it('shows the judge failure reason instead of a 0% score when there is none', () => { + renderPanel(null, 'parse_error') + + expect(screen.getByText('Unavailable')).toBeInTheDocument() + expect(screen.getByText('Judge response could not be parsed')).toBeInTheDocument() + expect(screen.queryByText('0%')).not.toBeInTheDocument() + expect(screen.queryByText('Below filter')).not.toBeInTheDocument() + }) + + it('falls back to a generic hint for an unknown status', () => { + renderPanel(null) + + expect(screen.getByText('Unavailable')).toBeInTheDocument() + expect( + screen.getByText('No usable score for this sample; it is excluded from the metric rather than counted as 0.'), + ).toBeInTheDocument() + }) }) diff --git a/evalscope/web/src/components/chat/EvalResultPanel.tsx b/evalscope/web/src/components/chat/EvalResultPanel.tsx index 5f9e2870a..5c3a50e77 100644 --- a/evalscope/web/src/components/chat/EvalResultPanel.tsx +++ b/evalscope/web/src/components/chat/EvalResultPanel.tsx @@ -61,19 +61,30 @@ export function CollapsibleJson({ export interface EvalResultPanelProps { pred: string gold: string - nScore: number + /** ``null`` when the sample has no usable score. */ + nScore: number | null + /** Mirrors ``ScoreStatus``; explains why a score is unavailable. */ + status?: string score: Record metadata: unknown threshold: number showPred: boolean } +const STATUS_LABEL_KEYS: Record = { + transport_error: 'prediction.statusTransportError', + parse_error: 'prediction.statusParseError', + invalid_session: 'prediction.statusInvalidSession', + excluded: 'prediction.statusExcluded', +} + const eyebrowBase = 'type-label-xs mb-2 flex items-center gap-1.5' export function EvalResultPanel({ pred, gold, nScore, + status, score, metadata, threshold, @@ -141,10 +152,30 @@ export function EvalResultPanel({ {t('prediction.score')}
- - - {nScore >= threshold ? t('prediction.aboveFilter') : t('prediction.belowFilter')} · {threshold} - + {nScore === null ? ( + <> + + {t('prediction.unavailable')} + + + {status && STATUS_LABEL_KEYS[status] ? t(STATUS_LABEL_KEYS[status]) : t('prediction.unavailableHint')} + + + ) : ( + <> + + + {nScore >= threshold ? t('prediction.aboveFilter') : t('prediction.belowFilter')} · {threshold} + + + )}
diff --git a/evalscope/web/src/components/chat/JudgeReviewPanel.test.tsx b/evalscope/web/src/components/chat/JudgeReviewPanel.test.tsx new file mode 100644 index 000000000..78dd9a900 --- /dev/null +++ b/evalscope/web/src/components/chat/JudgeReviewPanel.test.tsx @@ -0,0 +1,208 @@ +import { cleanup, fireEvent, render, screen } from '@testing-library/react' +import { afterEach, describe, expect, it } from 'vitest' +import { ThemeProvider } from '@/contexts/ThemeContext' +import { LocaleProvider } from '@/contexts/LocaleContext' +import { JudgeReviewPanel } from './JudgeReviewPanel' +import { selectJudgeReview } from '@/domain/chat/judgeReview' +import type { PredictionScore } from '@/api/types' + +afterEach(cleanup) + +/** + * Build the panel from a score payload shaped exactly as the API delivers it. + * + * Flask serializes with sorted keys, so `parsed_value` arrives alphabetically ordered -- the + * fixtures below preserve that, because it is what made the decisive field fall off the row. + */ +function renderPanel(score: unknown) { + const review = selectJudgeReview(score as PredictionScore) + if (!review) throw new Error('expected a judge review') + return render( + + + + + , + ) +} + +describe('JudgeReviewPanel', () => { + it('shows a rule short-circuit reason without inventing judge activity', () => { + renderPanel({ + metadata: { judge_skipped: true, judge_skip_reason: 'exact_string_match' }, + }) + + expect(screen.getByText('Rule-based score')).toBeTruthy() + expect(screen.getByText('LLM judge was not called')).toBeTruthy() + expect(screen.getByText('Reason: exact_string_match')).toBeTruthy() + expect(screen.queryByText('observations')).toBeNull() + }) + + it('shows the judge model, observation count and a single case row', () => { + renderPanel({ + judge_summary: { judge_models: ['qwen-plus'], valid_observations: 1, total_observations: 1 }, + metadata: { + judge_attempts: [{ + status: 'success', + case_id: 'grade', + judge_id: 'qwen-plus', + attempt_index: 0, + placement: 'original', + parsed_value: { reasoning: 'because', verdict: 'A' }, + raw_response: '{"verdict": "A"}', + latency: 1.24, + }], + }, + }) + + expect(screen.getByText('qwen-plus')).toBeTruthy() + expect(screen.getByText(/1\/1/)).toBeTruthy() + expect(screen.getByText('1.2s')).toBeTruthy() + // The decision is on the row; the prose is not. + expect(screen.getByText('A')).toBeTruthy() + expect(screen.queryByText(/because/)).toBeNull() + }) + + it('puts the decisive field on the row even when the API sorts it last', () => { + // Regression: researchrubrics' BinaryGrade arrives as + // confidence, evidence_quotes, missing_elements, reasoning, score, verdict. + // Showing keys in that order pushed `verdict` past the truncation point. + renderPanel({ + metadata: { + judge_attempts: [{ + status: 'success', + case_id: 'rubric_0', + judge_id: 'qwen-plus', + attempt_index: 0, + parsed_value: { + confidence: 0.99, + evidence_quotes: ['a very long quote that would otherwise dominate the row'], + missing_elements: [], + reasoning: 'long prose', + score: 1.0, + verdict: 'Satisfied', + }, + }], + }, + }) + + const row = screen.getByText(/Satisfied/) + expect(row).toBeTruthy() + // Noise must not lead the summary. + expect(row.textContent?.startsWith('confidence')).toBe(false) + expect(row.textContent).not.toContain('evidence_quotes') + }) + + it('reveals the parsed verdict and raw reply when a case row is expanded', () => { + renderPanel({ + metadata: { + judge_attempts: [{ + status: 'success', + case_id: 'grade', + judge_id: 'qwen-plus', + attempt_index: 0, + parsed_value: { verdict: 'A' }, + raw_response: '{"verdict": "A", "reasoning": "spelled out"}', + }], + }, + }) + + expect(screen.queryByText(/Raw judge reply/i)).toBeNull() + fireEvent.click(screen.getByRole('button', { expanded: false })) + expect(screen.getByText(/Parsed verdict/i)).toBeTruthy() + expect(screen.getByText(/Raw judge reply/i)).toBeTruthy() + expect(screen.getByText(/spelled out/)).toBeTruthy() + }) + + it('shows both sides of a pairwise case and does not call a swap a retry', () => { + renderPanel({ + metadata: { + judge_attempts: [ + { + status: 'success', case_id: 'battle', judge_id: 'j', attempt_index: 0, + placement: 'original', parsed_value: { reasoning: 'x', verdict: 'B>>A' }, + }, + { + status: 'success', case_id: 'battle', judge_id: 'j', attempt_index: 0, + placement: 'swapped', parsed_value: { reasoning: 'y', verdict: 'A>>B' }, + }, + ], + }, + }) + + // Regression: the row used to show only the swapped verdict and label the pair "retried 1x". + const row = screen.getByText(/original: B>>A/) + expect(row.textContent).toContain('swapped: A>>B') + expect(screen.queryByText(/retried/)).toBeNull() + expect(screen.getByText('original')).toBeTruthy() + expect(screen.getByText('swapped')).toBeTruthy() + }) + + it('keeps the key for a numeric verdict so a bare 0 never stands for a score', () => { + renderPanel({ + metadata: { + judge_attempts: [{ + status: 'success', case_id: 'grade', judge_id: 'j', attempt_index: 0, + parsed_value: { grading_rationale: 'prose', overall_score: 0, requirement_status: [] }, + }], + }, + }) + + expect(screen.getByText('overall_score: 0')).toBeTruthy() + }) + + it('flags a retry and surfaces the failure count', () => { + renderPanel({ + judge_summary: { judge_models: ['qwen-plus'], failures: { parse_error: 1 } }, + metadata: { + judge_attempts: [ + { + status: 'parse_error', case_id: 'grade', judge_id: 'qwen-plus', attempt_index: 0, + error: 'reply does not satisfy CLGrade', + }, + { + status: 'success', case_id: 'grade', judge_id: 'qwen-plus', attempt_index: 1, + parsed_value: { overall_score: 0 }, + }, + ], + }, + }) + + expect(screen.getByText(/retried 1/)).toBeTruthy() + expect(screen.getByText(/parse_error ×1/)).toBeTruthy() + }) + + it('collapses a long case list behind a show-all toggle', () => { + renderPanel({ + metadata: { + judge_attempts: Array.from({ length: 28 }, (_, i) => ({ + status: 'success', + case_id: `rubric_${i}`, + judge_id: 'qwen-plus', + attempt_index: 0, + parsed_value: { verdict: 'Satisfied', score: 1 }, + })), + }, + }) + + expect(screen.queryByText('rubric_27')).toBeNull() + const toggle = screen.getByText(/Show all 28 cases/) + fireEvent.click(toggle) + expect(screen.getByText('rubric_27')).toBeTruthy() + expect(screen.getByText(/Show fewer/)).toBeTruthy() + }) + + it('marks a case with no usable verdict instead of inventing one', () => { + renderPanel({ + judge_summary: { judge_models: ['j'], valid_observations: 0, total_observations: 1, error: 'no judge produced a usable verdict' }, + metadata: { + judge_attempts: [{ + status: 'parse_error', case_id: 'grade', judge_id: 'j', attempt_index: 0, raw_response: 'prose', + }], + }, + }) + + expect(screen.getByText(/No verdict/i)).toBeTruthy() + expect(screen.getByText(/no judge produced a usable verdict/)).toBeTruthy() + }) +}) diff --git a/evalscope/web/src/components/chat/JudgeReviewPanel.tsx b/evalscope/web/src/components/chat/JudgeReviewPanel.tsx new file mode 100644 index 000000000..6f48455d4 --- /dev/null +++ b/evalscope/web/src/components/chat/JudgeReviewPanel.tsx @@ -0,0 +1,270 @@ +import { useState } from 'react' +import { AlertTriangle, Check, ChevronDown, ChevronRight, RotateCcw, Scale, Timer } from 'lucide-react' +import { useLocale } from '@/contexts/LocaleContext' +import MarkdownRenderer from '@/components/ui/MarkdownRenderer' +import JsonViewer from '@/components/ui/JsonViewer' +import { isUsableStatus, type JudgeCaseView, type JudgeReviewView } from '@/domain/chat/judgeReview' + +/** Cases beyond this are collapsed: a rubric benchmark can declare dozens. */ +const VISIBLE_CASES = 8 + +const STATUS_LABEL_KEYS: Record = { + transport_error: 'prediction.statusTransportError', + parse_error: 'prediction.statusParseError', + invalid_session: 'prediction.statusInvalidSession', + excluded: 'prediction.statusExcluded', +} + +/** + * Fields that carry the actual decision, in order of preference. + * + * Read as "pick the first one the verdict happens to have": most schemas expose exactly one + * (`verdict` / `overall_score` / `rating` / `correct`), a rubric schema also has `score` alongside + * `verdict` -- the row shows the label, the expanded body shows the rest. + */ +const DECISIVE_KEYS = ['verdict', 'overall_score', 'correct', 'awarded', 'rating', 'score'] as const + +/** Keys whose value is self-explanatory, so the row shows the value alone without the key. */ +const SELF_EVIDENT_KEYS = new Set(['verdict', 'correct']) + +/** Render a verdict compactly: the row shows the decision, the expansion shows the evidence. */ +function verdictSummary(verdict: unknown): string { + if (verdict == null) return '' + if (typeof verdict !== 'object') return String(verdict) + const entries = Object.entries(verdict as Record) + const decisive = DECISIVE_KEYS.map(k => entries.find(([key]) => key === k)).filter( + (e): e is [string, unknown] => !!e, + ) + // A schema that carries none of the well-known decisive fields (e.g. mia_bench's component_*) + // falls back to every non-prose field; the summary row is truncated anyway. + const shown = decisive.length > 0 + ? decisive + : entries.filter(([key]) => !/^(reasoning|explanation|rationale|evidence_quotes|missing_elements|notes)$/i.test(key)) + return shown + .map(([key, value]) => { + const text = typeof value === 'object' && value !== null ? JSON.stringify(value) : String(value) + // Keep the key unless the value speaks for itself, so a bare `0` never stands for a score. + return shown.length === 1 && SELF_EVIDENT_KEYS.has(key) ? text : `${key}: ${text}` + }) + .join(' · ') +} + +/** The row summary for a case, folding both sides of a pairwise verdict when present. */ +function caseSummary(view: JudgeCaseView): string { + if (view.placementVerdicts) { + return Object.entries(view.placementVerdicts) + .map(([placement, value]) => `${placement}: ${verdictSummary(value)}`) + .join(' | ') + } + return verdictSummary(view.verdict) +} + +function StatusPill({ status }: { status: string }) { + const { t } = useLocale() + const usable = isUsableStatus(status) + const color = usable + ? (status === 'fallback' ? 'var(--warning-color)' : 'var(--success)') + : 'var(--danger)' + const Icon = usable ? Check : AlertTriangle + const labelKey = STATUS_LABEL_KEYS[status] + const label = labelKey ? t(labelKey) : status + return ( + + + {status} + + ) +} + +/** One case row: collapsed it is a single line; expanded it shows the parsed verdict and raw text. */ +function CaseRow({ view, showCaseId }: { view: JudgeCaseView; showCaseId: boolean }) { + const { t } = useLocale() + const [open, setOpen] = useState(false) + const summary = caseSummary(view) + + return ( +
+ + + {open && ( +
+ {view.verdict != null && ( +
+
+ {t('prediction.judgeParsedVerdict')} +
+
+ +
+
+ )} + {view.attempts.map((attempt, index) => ( +
+
+ {t('prediction.judgeRawResponse')} + + {t('prediction.judgeAttemptLabel').replace('${index}', String(index + 1))} + {attempt.placement ? ` · ${attempt.placement}` : ''} + + +
+ {attempt.error && ( +
{attempt.error}
+ )} + {attempt.raw_response && ( + 
+                  {attempt.raw_response}
+
+ )} +
+ ))} +
+ )} +
+ ) +} + +export interface JudgeReviewPanelProps { + review: JudgeReviewView +} + +/** + * Structured view of a sample's judge work or deterministic rule short-circuit. + * + * Rule short-circuits deliberately have no judge attempts, but still render their reason. + */ +export function JudgeReviewPanel({ review }: JudgeReviewPanelProps) { + const { t } = useLocale() + const [expanded, setExpanded] = useState(false) + const failureEntries = Object.entries(review.failures) + const cases = expanded ? review.cases : review.cases.slice(0, VISIBLE_CASES) + // A single-case benchmark needs no case column: the row is the verdict. + const showCaseId = review.cases.length > 1 + + if (review.skipped) { + return ( +
+
+ + {t('prediction.ruleBasedScore')} +
+
+ {t('prediction.judgeNotCalled')} + + {t('prediction.judgeSkipReason')}: {review.skipReason ?? 'unknown'} + +
+
+ ) + } + + return ( +
+
+ + + {t('prediction.judgeReview')} + + {review.judgeModels.map(model => ( + {model} + ))} + {review.totalObservations != null && ( + + {review.validObservations ?? 0}/{review.totalObservations} {t('prediction.judgeObservations')} + + )} + {failureEntries.map(([status, count]) => ( + + + {status} ×{count} + + ))} + {review.totalLatency != null && ( + + + {/* A diagnostic wall-clock total, not a report metric; one-decimal seconds is enough. */} + {Math.round(review.totalLatency * 10) / 10}s + + )} +
+ + {review.error && ( +
+ {review.error} +
+ )} + + {review.cases.length > 0 && ( +
+ {cases.map(view => ( + + ))} + {review.cases.length > VISIBLE_CASES && ( + + )} +
+ )} + + {review.explanation && ( +
+
{t('prediction.judgeExplanation')}
+
+ +
+
+ )} +
+ ) +} diff --git a/evalscope/web/src/components/compare/PredictionCompareTab.tsx b/evalscope/web/src/components/compare/PredictionCompareTab.tsx index f857df0cf..2d1a1a560 100644 --- a/evalscope/web/src/components/compare/PredictionCompareTab.tsx +++ b/evalscope/web/src/components/compare/PredictionCompareTab.tsx @@ -304,11 +304,17 @@ export default function PredictionCompareTab({ of the view threshold. The threshold only filters rows in this view and is not a pass/fail verdict. */} - + {modelRow.NScore === null ? ( + + {t('prediction.unavailable')} + + ) : ( + + )} {/* ChatView */} diff --git a/evalscope/web/src/components/reports/PredictionsTab.tsx b/evalscope/web/src/components/reports/PredictionsTab.tsx index 9b0ead88a..159238856 100644 --- a/evalscope/web/src/components/reports/PredictionsTab.tsx +++ b/evalscope/web/src/components/reports/PredictionsTab.tsx @@ -84,15 +84,22 @@ export default function PredictionsTab({ reportName, datasetName, rootPath, init const loadError = subsetsResource.error || predictionsResource.error // The threshold is a view-only filter (above/below), not a pass/fail verdict, - // and it never leaves this view. + // and it never leaves this view. A sample without a usable score belongs to neither + // side; counting it as Below would report an unscored sample as a failure. const filtered = useMemo(() => { - if (mode === 'Above') return predictions.filter((p) => p.NScore >= threshold) - if (mode === 'Below') return predictions.filter((p) => p.NScore < threshold) + if (mode === 'Above') return predictions.filter((p) => p.NScore !== null && p.NScore >= threshold) + if (mode === 'Below') return predictions.filter((p) => p.NScore !== null && p.NScore < threshold) return predictions }, [predictions, mode, threshold]) - const aboveCount = useMemo(() => predictions.filter((p) => p.NScore >= threshold).length, [predictions, threshold]) - const belowCount = predictions.length - aboveCount + const aboveCount = useMemo( + () => predictions.filter((p) => p.NScore !== null && p.NScore >= threshold).length, + [predictions, threshold], + ) + const belowCount = useMemo( + () => predictions.filter((p) => p.NScore !== null && p.NScore < threshold).length, + [predictions, threshold], + ) const totalPages = filtered.length const row = totalPages > 0 ? filtered[Math.min(page - 1, totalPages - 1)] : null diff --git a/evalscope/web/src/domain/chat/judgeReview.test.ts b/evalscope/web/src/domain/chat/judgeReview.test.ts new file mode 100644 index 000000000..850c3e71a --- /dev/null +++ b/evalscope/web/src/domain/chat/judgeReview.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it } from 'vitest' +import { isUsableStatus, scoreWithoutJudgeAttempts, selectJudgeReview } from './judgeReview' +import type { PredictionScore } from '@/api/types' + +/** A minimal attempt row as the backend serializes one. */ +function attempt(overrides: Record): Record { + return { + status: 'success', + case_id: 'match', + judge_id: 'qwen-plus', + attempt_index: 0, + ...overrides, + } +} + +describe('selectJudgeReview', () => { + it('returns null for a rule-scored sample', () => { + const score = { value: { acc: 1 } } as PredictionScore + expect(selectJudgeReview(score)).toBeNull() + expect(selectJudgeReview(undefined)).toBeNull() + }) + + it('surfaces a rule short-circuit without judge attempts', () => { + const score = { + value: { acc: 1 }, + metadata: { judge_skipped: true, judge_skip_reason: 'exact_string_match' }, + } as unknown as PredictionScore + + expect(selectJudgeReview(score)).toMatchObject({ + skipped: true, + skipReason: 'exact_string_match', + judgeModels: [], + cases: [], + }) + }) + + it('surfaces judge_summary even when no attempts were saved', () => { + const score = { + value: { acc: 1 }, + judge_summary: { judge_models: ['qwen-plus'], valid_observations: 1, total_observations: 1 }, + } as unknown as PredictionScore + + const review = selectJudgeReview(score)! + expect(review.judgeModels).toEqual(['qwen-plus']) + expect(review.cases).toHaveLength(0) + }) + + it('promotes the usable attempt to the case verdict', () => { + const score = { + judge_summary: { judge_models: ['qwen-plus'] }, + metadata: { + judge_attempts: [attempt({ parsed_value: { verdict: 'A' }, latency: 1.5 })], + }, + } as unknown as PredictionScore + + const review = selectJudgeReview(score)! + expect(review.cases).toHaveLength(1) + expect(review.cases[0].verdict).toEqual({ verdict: 'A' }) + expect(review.cases[0].status).toBe('success') + expect(review.cases[0].retries).toBe(0) + expect(review.totalLatency).toBe(1.5) + }) + + it('counts a parse-error retry that then succeeds as one retry', () => { + const score = { + metadata: { + judge_attempts: [ + attempt({ status: 'parse_error', attempt_index: 0, error: 'bad json' }), + attempt({ status: 'success', attempt_index: 1, parsed_value: { verdict: 'A' } }), + ], + }, + } as unknown as PredictionScore + + const view = selectJudgeReview(score)!.cases[0] + expect(view.retries).toBe(1) + expect(view.status).toBe('success') + expect(view.attempts).toHaveLength(2) + }) + + it('marks a case excluded when every attempt failed', () => { + const score = { + metadata: { + judge_attempts: [ + attempt({ status: 'parse_error', attempt_index: 0 }), + attempt({ status: 'parse_error', attempt_index: 1 }), + ], + }, + } as unknown as PredictionScore + + const view = selectJudgeReview(score)!.cases[0] + expect(view.verdict).toBeUndefined() + expect(view.status).toBe('parse_error') + expect(view.retries).toBe(2) // both attempts broke the contract + }) + + it('groups attempts by case and preserves first-seen order', () => { + const score = { + metadata: { + judge_attempts: [ + attempt({ case_id: 'rubric_0', parsed_value: { score: 1 } }), + attempt({ case_id: 'rubric_1', parsed_value: { score: 0 } }), + ], + }, + } as unknown as PredictionScore + + const review = selectJudgeReview(score)! + expect(review.cases.map(c => c.caseId)).toEqual(['rubric_0', 'rubric_1']) + }) + + it('exposes per-placement verdicts for a pairwise case', () => { + const score = { + metadata: { + judge_attempts: [ + attempt({ case_id: 'battle', placement: 'original', parsed_value: { verdict: 'B>>A' } }), + attempt({ case_id: 'battle', placement: 'swapped', parsed_value: { verdict: 'A>>B' } }), + ], + }, + } as unknown as PredictionScore + + const view = selectJudgeReview(score)!.cases[0] + expect(view.placements).toEqual(['original', 'swapped']) + // Both sides are kept; neither is silently dropped, and a swap is not a retry. + expect(view.placementVerdicts).toEqual({ original: { verdict: 'B>>A' }, swapped: { verdict: 'A>>B' } }) + expect(view.retries).toBe(0) + }) +}) + +describe('scoreWithoutJudgeAttempts', () => { + it('drops judge_attempts but keeps the rest of metadata', () => { + const score = { + value: { acc: 1 }, + metadata: { source: 'llm_judge', judge_attempts: [attempt({})] }, + } as unknown as PredictionScore + + const stripped = scoreWithoutJudgeAttempts(score) + expect(stripped.metadata?.judge_attempts).toBeUndefined() + expect((stripped.metadata as Record).source).toBe('llm_judge') + // The original is untouched. + expect(score.metadata?.judge_attempts).toHaveLength(1) + }) + + it('returns the same score when there are no attempts', () => { + const score = { value: { acc: 1 } } as PredictionScore + expect(scoreWithoutJudgeAttempts(score)).toBe(score) + }) +}) + +describe('isUsableStatus', () => { + it('treats success and fallback as usable, everything else as not', () => { + expect(isUsableStatus('success')).toBe(true) + expect(isUsableStatus('fallback')).toBe(true) + expect(isUsableStatus('parse_error')).toBe(false) + expect(isUsableStatus('transport_error')).toBe(false) + expect(isUsableStatus('excluded')).toBe(false) + }) +}) diff --git a/evalscope/web/src/domain/chat/judgeReview.ts b/evalscope/web/src/domain/chat/judgeReview.ts new file mode 100644 index 000000000..bb5f4aa9e --- /dev/null +++ b/evalscope/web/src/domain/chat/judgeReview.ts @@ -0,0 +1,136 @@ +/** + * Folds a score's judge diagnostics into the shape the review panel renders. + * + * The raw payload is attempt-oriented: one row per request, including the retries that failed. + * A reader wants it case-oriented -- "what did the judge decide about rubric 3, and how hard was + * it to get there" -- so attempts are grouped by case and the usable one is promoted to the + * case's verdict. + */ +import type { JudgeAttempt, PredictionScore } from '@/api/types' + +/** Statuses that carry a usable verdict; mirrors `ScoreStatus.is_usable`. */ +const USABLE_STATUSES = new Set(['success', 'fallback', 'degraded']) + +export interface JudgeCaseView { + caseId: string + /** The parsed verdict of the usable attempt, or `undefined` when every attempt failed. */ + verdict?: unknown + /** + * Per-placement verdicts, set only when the case was judged in both orders. A pairwise case is + * one atomic observation, so neither side alone is "the" verdict. + */ + placementVerdicts?: Record + /** Status of the case as a whole: the usable attempt's, else the last attempt's. */ + status: string + /** Every round trip for this case, in order, including failed retries. */ + attempts: JudgeAttempt[] + /** Attempts the judge wasted on a reply that broke the contract. */ + retries: number + /** Distinct placements seen, present only for a pairwise benchmark. */ + placements: string[] +} + +export interface JudgeReviewView { + /** True when a deterministic rule produced the score without calling an LLM judge. */ + skipped: boolean + /** Machine-readable reason supplied by ``JudgeDefinition.skip``. */ + skipReason?: string + judgeModels: string[] + validObservations?: number + totalObservations?: number + /** Attempt counts keyed by `ScoreStatus`, e.g. `{ parse_error: 2 }`. */ + failures: Record + /** Why the score is unavailable, when the executor recorded a reason. */ + error?: string + /** Summed latency across every attempt, in seconds; `undefined` when untimed. */ + totalLatency?: number + cases: JudgeCaseView[] + /** The adapter's own narrative, e.g. a per-rubric breakdown. */ + explanation?: string +} + +/** + * Build the view model, or `null` when this sample has neither judge diagnostics nor a rule short-circuit. + * + * A rule short-circuit has no judge attempts but remains visible so users can see why an LLM was + * not called. + */ +export function selectJudgeReview(score: PredictionScore | undefined): JudgeReviewView | null { + if (!score) return null + const detail = score.judge_summary + const attempts = score.metadata?.judge_attempts ?? [] + const skipped = score.metadata?.judge_skipped === true + if (!detail && attempts.length === 0 && !skipped) return null + + const latencies = attempts.map(a => a.latency).filter((v): v is number => typeof v === 'number') + + return { + skipped, + skipReason: score.metadata?.judge_skip_reason, + judgeModels: detail?.judge_models ?? [], + validObservations: detail?.valid_observations, + totalObservations: detail?.total_observations, + failures: detail?.failures ?? {}, + error: detail?.error, + totalLatency: latencies.length > 0 ? latencies.reduce((a, b) => a + b, 0) : undefined, + cases: groupByCase(attempts), + explanation: score.explanation || undefined, + } +} + +/** Group attempts into cases, preserving first-seen case order. */ +function groupByCase(attempts: JudgeAttempt[]): JudgeCaseView[] { + const byCase = new Map() + for (const attempt of attempts) { + // Group by case, not by placement: pairwise swaps are two attempts of the same case. + const key = `${attempt.judge_id}:${attempt.repeat_id ?? 0}:${attempt.case_id}` + const bucket = byCase.get(key) + if (bucket) bucket.push(attempt) + else byCase.set(key, [attempt]) + } + + return [...byCase.values()].map((caseAttempts) => { + const caseId = caseAttempts[0].case_id + const usable = caseAttempts.filter(a => USABLE_STATUSES.has(a.status)) + const decisive = usable.length > 0 ? usable[usable.length - 1] : undefined + const placements = [...new Set(caseAttempts.map(a => a.placement).filter((p): p is string => !!p))] + // A pairwise case is judged in both orders and the pair is one verdict, so each side is + // reported separately instead of letting the last one stand for the case. + const placementVerdicts = placements.length > 1 + ? Object.fromEntries( + placements + .map(p => [p, usable.find(a => a.placement === p)?.parsed_value] as const) + .filter(([, value]) => value !== undefined), + ) + : undefined + return { + caseId, + verdict: decisive?.parsed_value, + placementVerdicts, + status: decisive?.status ?? caseAttempts[caseAttempts.length - 1].status, + attempts: caseAttempts, + // Count the attempts that broke the contract. Counting "attempts before the usable one" + // instead would report a position swap as a retry, since both sides succeed in sequence. + retries: caseAttempts.filter(a => !USABLE_STATUSES.has(a.status)).length, + placements: placements.length > 1 ? placements : [], + } + }) +} + +/** Whether a status carries a usable verdict. */ +export function isUsableStatus(status: string): boolean { + return USABLE_STATUSES.has(status) +} + +/** + * The score without its `judge_attempts`, for the raw Score Detail view. + * + * The attempts are rendered by the review panel; leaving them in the JSON blob too would repeat + * every raw judge reply, which for a rubric benchmark is the bulk of a 100 KB payload. + */ +export function scoreWithoutJudgeAttempts(score: PredictionScore): PredictionScore { + if (!score.metadata?.judge_attempts) return score + const metadata = { ...score.metadata } + delete metadata.judge_attempts + return { ...score, metadata } +} diff --git a/evalscope/web/src/domain/metric/MetricSemantics.ts b/evalscope/web/src/domain/metric/MetricSemantics.ts index 8245c141f..0c7cb99be 100644 --- a/evalscope/web/src/domain/metric/MetricSemantics.ts +++ b/evalscope/web/src/domain/metric/MetricSemantics.ts @@ -22,6 +22,7 @@ export type ValueRange = z.infer export const metricSemanticsSchema = z.object({ semantic_id: z.string(), metric_name: z.string(), + display_name: z.string().nullable().optional(), kind: metricKindSchema, direction: metricDirectionSchema, raw_unit: z.string().nullable().optional(), diff --git a/evalscope/web/src/domain/metric/metricFormat.test.ts b/evalscope/web/src/domain/metric/metricFormat.test.ts index 031c65579..1144e1463 100644 --- a/evalscope/web/src/domain/metric/metricFormat.test.ts +++ b/evalscope/web/src/domain/metric/metricFormat.test.ts @@ -137,6 +137,18 @@ describe('metric labels', () => { expect(formatMetricLabel('unknown_metric', null)).toBe('unknown_metric') }) + it('uses an explicit display name for a declared diagnostic metric', () => { + const incorrectRate = { ...diagnosticSemantics(), display_name: 'Incorrect rate' } + + expect(formatMetricLabel('is_incorrect:mean', incorrectRate)).toBe('Incorrect rate') + expect( + formatMetricIdentityLabel( + { name: 'is_incorrect', aggregation: 'mean', dimensions: {} }, + incorrectRate, + ), + ).toBe('Incorrect rate') + }) + it('disambiguates repeated labels within one report', () => { const labels = formatMetricLabels([ { metricName: 'mean_acc', semantics: ratioSemantics() }, diff --git a/evalscope/web/src/domain/metric/metricFormat.ts b/evalscope/web/src/domain/metric/metricFormat.ts index b6375dddc..3da7d47e6 100644 --- a/evalscope/web/src/domain/metric/metricFormat.ts +++ b/evalscope/web/src/domain/metric/metricFormat.ts @@ -73,7 +73,8 @@ export function formatMetricLabel( finalMetricName: string, semantics: MetricSemantics | null | undefined, ): string { - if (!semantics || semantics.kind === 'diagnostic') return finalMetricName + if (!semantics) return finalMetricName + if (semantics.kind === 'diagnostic') return semantics.display_name ?? finalMetricName return `${semantics.metric_name} ${directionArrow(semantics)}`.trimEnd() } @@ -83,7 +84,8 @@ export function formatMetricIdentityLabel( semantics: MetricSemantics | null | undefined, legacyName?: string | null, ): string { - if (!semantics || semantics.kind === 'diagnostic') return legacyName || metricIdentityKey(identity) + if (!semantics) return legacyName || metricIdentityKey(identity) + if (semantics.kind === 'diagnostic') return semantics.display_name || legacyName || metricIdentityKey(identity) const base = formatMetricLabel(identity.name, semantics) const dimensionOrder: Record = { target: 0, diff --git a/evalscope/web/src/i18n/translations/prediction.ts b/evalscope/web/src/i18n/translations/prediction.ts index 3e839d456..ffdba32f1 100644 --- a/evalscope/web/src/i18n/translations/prediction.ts +++ b/evalscope/web/src/i18n/translations/prediction.ts @@ -17,6 +17,25 @@ export const en: Dict = { below: 'Below', aboveFilter: 'Above filter', belowFilter: 'Below filter', + unavailable: 'Unavailable', + unavailableHint: 'No usable score for this sample; it is excluded from the metric rather than counted as 0.', + statusTransportError: 'Judge request failed', + statusParseError: 'Judge response could not be parsed', + statusInvalidSession: 'Incomplete judge session', + statusExcluded: 'Excluded from the metric', + judgeReview: 'Judge Review', + ruleBasedScore: 'Rule-based score', + judgeNotCalled: 'LLM judge was not called', + judgeSkipReason: 'Reason', + judgeObservations: 'observations', + judgeNoVerdict: 'No verdict', + judgeRetries: 'retried ${count}\u00d7', + judgeRawResponse: 'Raw judge reply', + judgeParsedVerdict: 'Parsed verdict', + judgeExplanation: 'Explanation', + judgeShowAllCases: 'Show all ${count} cases', + judgeShowFewerCases: 'Show fewer', + judgeAttemptLabel: 'attempt ${index}', previousSample: 'Previous sample', nextSample: 'Next sample', searchByIndex: 'Go to index', @@ -42,6 +61,25 @@ export const zh: Dict = { below: '低于', aboveFilter: '高于筛选阈值', belowFilter: '低于筛选阈值', + unavailable: '不可用', + unavailableHint: '该样本没有可用得分，会从指标中排除，而不是按 0 分计入。', + statusTransportError: '评判请求失败', + statusParseError: '评判响应无法解析', + statusInvalidSession: '评判会话不完整', + statusExcluded: '已从指标中排除', + judgeReview: '评判详情', + ruleBasedScore: '规则直接判分', + judgeNotCalled: '未调用 LLM 评判', + judgeSkipReason: '原因', + judgeObservations: '次有效观测', + judgeNoVerdict: '无判定', + judgeRetries: '重试 ${count} 次', + judgeRawResponse: '评判原始回复', + judgeParsedVerdict: '解析后的判定', + judgeExplanation: '评分说明', + judgeShowAllCases: '展开全部 ${count} 项', + judgeShowFewerCases: '收起', + judgeAttemptLabel: '第 ${index} 次请求', previousSample: '上一个样本', nextSample: '下一个样本', searchByIndex: '跳转到索引', diff --git a/evalscope/web/src/pages/ComparePage.tsx b/evalscope/web/src/pages/ComparePage.tsx index 8ec7d186d..26c2f437c 100644 --- a/evalscope/web/src/pages/ComparePage.tsx +++ b/evalscope/web/src/pages/ComparePage.tsx @@ -284,8 +284,10 @@ export default function ComparePage() { const f = perModelFilter[name] ?? 'any' if (f === 'any') return true // The threshold is a view-only filter (above/below), not a pass/fail - // verdict. - const above = (row.models[name]?.NScore ?? 0) >= threshold + // verdict. A sample without a usable score matches neither side. + const score = row.models[name]?.NScore + if (score == null) return false + const above = score >= threshold return f === 'above' ? above : !above }), ) @@ -296,8 +298,10 @@ export default function ComparePage() { if (!mergedPredictions.length) return {} as Record const rates: Record = {} for (const name of predictionReportNames) { - const above = mergedPredictions.filter((r) => (r.models[name]?.NScore ?? 0) >= threshold).length - rates[name] = above / mergedPredictions.length + const scored = mergedPredictions.filter((r) => r.models[name]?.NScore != null) + if (!scored.length) continue + const above = scored.filter((r) => (r.models[name]!.NScore as number) >= threshold).length + rates[name] = above / scored.length } return rates }, [mergedPredictions, predictionReportNames, threshold]) diff --git a/examples/aigc/image_edit.py b/examples/aigc/image_edit.py index 0514d5d92..edaaa01c9 100644 --- a/examples/aigc/image_edit.py +++ b/examples/aigc/image_edit.py @@ -3,7 +3,7 @@ env = dotenv_values('.env') from evalscope import TaskConfig, run_task -from evalscope.constants import EvalType, JudgeStrategy, ModelTask +from evalscope.constants import EvalType, ModelTask task_config = TaskConfig( model='Qwen/Qwen-Image-Edit', # 模型ID 或 本地路径 @@ -30,15 +30,17 @@ }, eval_batch_size=5, limit=5, - judge_strategy=JudgeStrategy.AUTO, - judge_model_args={ # 需要配置一个VLM模型用于自动打分 - 'model_id': 'qwen2.5-vl-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': env.get('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096, - } + judge={ + 'strategy': 'auto', + 'models': { # 需要配置一个VLM模型用于自动打分 + 'model_id': 'qwen2.5-vl-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': env.get('DASHSCOPE_API_KEY'), + 'generation_config': { + 'temperature': 0.0, + 'max_tokens': 4096, + }, + }, }, ) diff --git a/examples/aigc/image_edit_local.py b/examples/aigc/image_edit_local.py index 24057d9a9..136ff8ea7 100644 --- a/examples/aigc/image_edit_local.py +++ b/examples/aigc/image_edit_local.py @@ -3,7 +3,7 @@ env = dotenv_values('.env') from evalscope import TaskConfig, run_task -from evalscope.constants import EvalType, JudgeStrategy, ModelTask +from evalscope.constants import EvalType, ModelTask task_config = TaskConfig( model_id='offline-model', @@ -21,15 +21,17 @@ }, eval_batch_size=5, limit=5, - judge_strategy=JudgeStrategy.AUTO, - judge_model_args={ # 需要配置一个VLM模型用于自动打分 - 'model_id': 'qwen2.5-vl-72b-instruct', - 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', - 'api_key': env.get('DASHSCOPE_API_KEY'), - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 4096, - } + judge={ + 'strategy': 'auto', + 'models': { # 需要配置一个VLM模型用于自动打分 + 'model_id': 'qwen2.5-vl-72b-instruct', + 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', + 'api_key': env.get('DASHSCOPE_API_KEY'), + 'generation_config': { + 'temperature': 0.0, + 'max_tokens': 4096, + }, + }, }, ) diff --git a/examples/viz/qwen-plus/configs/task_config.yaml b/examples/viz/qwen-plus/configs/task_config.yaml index 27642faa1..1b4924b06 100644 --- a/examples/viz/qwen-plus/configs/task_config.yaml +++ b/examples/viz/qwen-plus/configs/task_config.yaml @@ -358,9 +358,9 @@ generation_config: stream: true temperature: 0.7 ignore_errors: false -judge_model_args: {} -judge_strategy: auto -judge_worker_num: null +judge: + strategy: auto + models: [] limit: 10 model: qwen-plus model_args: {} diff --git a/examples/viz/qwen3-max/configs/task_config.yaml b/examples/viz/qwen3-max/configs/task_config.yaml index 27b5b1349..f701c6bb5 100644 --- a/examples/viz/qwen3-max/configs/task_config.yaml +++ b/examples/viz/qwen3-max/configs/task_config.yaml @@ -358,9 +358,9 @@ generation_config: stream: true temperature: 0.7 ignore_errors: false -judge_model_args: {} -judge_strategy: auto -judge_worker_num: null +judge: + strategy: auto + models: [] limit: 10 model: qwen3-max model_args: {} diff --git a/tests/api/judge/test_aggregation_excludes_unusable.py b/tests/api/judge/test_aggregation_excludes_unusable.py new file mode 100644 index 000000000..9a2dd7f55 --- /dev/null +++ b/tests/api/judge/test_aggregation_excludes_unusable.py @@ -0,0 +1,130 @@ +"""An unusable judge review carries an empty ``Score.value``; every adapter that keeps its own +``aggregate_scores`` must exclude such a sample from the metric rather than count it as 0. + +These guard the four aggregators fixed alongside the JSON-judge migration: the generic +aggregators are covered separately in ``tests/metrics/aggregators/test_missing_metric_keys.py``. +""" +import pytest +from typing import Dict, Optional + +from evalscope.api.metric import SampleScore, Score +from evalscope.api.registry import get_benchmark +from evalscope.config import TaskConfig +from evalscope.constants import DataCollection, ScoreStatus + +JUDGE_ARGS = {'model_id': 'judge', 'api_url': 'http://judge/v1', 'api_key': 'k'} + + +def scored(value: Dict[str, float], sample_id: int, metadata: Optional[dict] = None) -> SampleScore: + return SampleScore( + score=Score(value=value, status=ScoreStatus.SUCCESS), + sample_id=sample_id, + sample_metadata=metadata or {}, + ) + + +def excluded(sample_id: int, metadata: Optional[dict] = None) -> SampleScore: + """What ``JudgeExecutor.build_score`` produces for an unusable review: empty value, bad status.""" + return SampleScore( + score=Score(value={}, status=ScoreStatus.PARSE_ERROR), + sample_id=sample_id, + sample_metadata=metadata or {}, + ) + + +def _judge_benchmark(name: str): + cfg = TaskConfig(model='m', datasets=[name], judge={'strategy': 'llm', 'models': JUDGE_ARGS}) + return get_benchmark(name, config=cfg) + + +def _by_metric(agg_scores) -> Dict[str, object]: + return {agg.metric_name: agg for agg in agg_scores} + + +def test_researchrubrics_excludes_unusable_sample_instead_of_crashing() -> None: + adapter = _judge_benchmark('researchrubrics') + meta = {'domain': 'science', 'conceptual_breadth': None, 'logical_nesting': None, 'exploration': None} + result = adapter.aggregate_scores([ + scored({'compliance_score': 1.0}, sample_id=1, metadata=meta), + excluded(sample_id=2, metadata=meta), + ]) + by_metric = _by_metric(result) + # Previously raised KeyError('compliance_score'); now the excluded sample simply drops out. + assert by_metric['compliance_score'].score == 1.0 + assert by_metric['compliance_score'].num == 1 + + +def test_researchrubrics_all_unusable_yields_no_rows() -> None: + adapter = _judge_benchmark('researchrubrics') + assert adapter.aggregate_scores([excluded(sample_id=1), excluded(sample_id=2)]) == [] + + +def test_mcp_atlas_excludes_unusable_sample() -> None: + adapter = _judge_benchmark('mcp_atlas') + result = _by_metric( + adapter.aggregate_scores([ + scored({'coverage_score': 1.0, 'pass': 1.0}, sample_id=1), + excluded(sample_id=2), + ]) + ) + # Averaged over the one usable sample, not diluted to 0.5 by the excluded one. + assert result['coverage_score'].score == 1.0 + assert result['coverage_score'].num == 1 + assert result['pass_rate'].score == 1.0 + assert result['pass_rate'].num == 1 + + +def test_drivelology_averages_each_metric_over_its_own_usable_samples() -> None: + adapter = _judge_benchmark('drivel_writing') + result = _by_metric( + adapter.aggregate_scores([ + # The rule-based bert_score is always present; the judge_score can be missing. + scored({'judge_score': 1.0, 'bert_score': 0.8}, sample_id=1), + scored({'bert_score': 0.6}, sample_id=2), + ]) + ) + assert result['judge_score'].score == 1.0 + assert result['judge_score'].num == 1 + assert result['bert_score'].score == pytest.approx(0.7) + assert result['bert_score'].num == 2 + + +COLLECTION_INFO = { + 'task_type': 'qa', + 'categories': ('c', ), + 'dataset_name': 'd', + 'subset_name': 's', + 'tags': ['t'], + 'weight': 1.0, +} + + +def test_data_collection_drops_unusable_sample_from_dataframe() -> None: + adapter = get_benchmark('data_collection', config=TaskConfig(model='m')) + scores = [ + scored({'acc': 1.0}, sample_id=1, metadata={DataCollection.INFO: COLLECTION_INFO}), + excluded(sample_id=2, metadata={DataCollection.INFO: COLLECTION_INFO}), + ] + df = adapter._build_sample_dataframe(scores) + assert len(df) == 1 + rows = adapter._group_and_compute(df, ['dataset_name']) + # The excluded sample no longer drags the weighted mean toward 0. + assert rows[0]['weighted_avg.'] == 1.0 + assert rows[0]['count'] == 1 + + +def test_data_collection_all_unusable_yields_empty_levels() -> None: + """Excluding every sample leaves a column-less dataframe: grouping it raised KeyError, which + aborted the run after all inference had been paid for.""" + adapter = get_benchmark('data_collection', config=TaskConfig(model='m')) + scores = [ + excluded(sample_id=1, metadata={DataCollection.INFO: COLLECTION_INFO}), + excluded(sample_id=2, metadata={DataCollection.INFO: COLLECTION_INFO}), + ] + report = adapter.aggregate_scores(scores) + assert report['subset_level'] == [] + assert report['dataset_level'] == [] + assert report['task_level'] == [] + assert report['tag_level'] == [] + assert report['category_level'] == [] + assert report['df'].empty diff --git a/tests/api/judge/test_contracts.py b/tests/api/judge/test_contracts.py new file mode 100644 index 000000000..028c684e1 --- /dev/null +++ b/tests/api/judge/test_contracts.py @@ -0,0 +1,236 @@ +"""The judge output contract: what a reply must satisfy, and what it must reject. + +Every rejection case here was observed in the wild before the refactor and used to produce a +score instead of a parse failure. +""" +from pydantic import BaseModel, Field +from typing import Literal + +from evalscope.api.judge import OutputContract + + +class Grade(BaseModel): + reasoning: str = '' + verdict: Literal['A', 'B', 'C'] + + +class Rating(BaseModel): + reasoning: str = '' + verdict: float = Field(ge=0.0, le=10.0) + + +class MultiField(BaseModel): + extracted_final_answer: str = '' + reasoning: str = '' + correct: Literal['yes', 'no'] + confidence: int = Field(default=100, ge=0, le=100) + + +GRADE = OutputContract(schema_model=Grade) +RATING = OutputContract(schema_model=Rating) +MULTI = OutputContract(schema_model=MultiField) + + +# --------------------------------------------------------------------------- +# Accepted envelopes +# --------------------------------------------------------------------------- + + +def test_accepts_a_bare_json_object(): + result = GRADE.parse('{"reasoning": "matches", "verdict": "A"}') + + assert result.ok + assert result.value.verdict == 'A' + + +def test_accepts_surrounding_whitespace(): + assert GRADE.parse('\n {"verdict": "B"} \n').value.verdict == 'B' + + +def test_accepts_a_fenced_json_block(): + """Models wrap JSON in a fence even when told not to.""" + result = GRADE.parse('Here is my grade:\n```json\n{"verdict": "C"}\n```\n') + + assert result.value.verdict == 'C' + + +def test_accepts_a_fence_without_a_language_tag(): + assert GRADE.parse('```\n{"verdict": "A"}\n```').value.verdict == 'A' + + +def test_omitted_optional_field_takes_its_default(): + assert GRADE.parse('{"verdict": "A"}').value.reasoning == '' + + +def test_accepts_additional_keys_when_required_keys_exist(): + result = GRADE.parse('{"verdict": "A", "provider_trace": "kept for audit"}') + + assert result.ok + assert result.value.verdict == 'A' + + +# --------------------------------------------------------------------------- +# Rejected: no readable payload +# --------------------------------------------------------------------------- + + +def test_rejects_an_empty_response(): + result = GRADE.parse('') + + assert not result.ok + assert 'empty' in result.error + + +def test_rejects_prose_containing_the_verdict_letter(): + """``re.search(r'(A|B|C)')`` used to read the A out of "Answer".""" + result = GRADE.parse('The Answer is correct, so grade A applies.') + + assert not result.ok + assert 'no JSON object' in result.error + + +def test_rejects_a_bare_verdict_token(): + assert not GRADE.parse('A').ok + + +def test_rejects_truncated_json(): + result = GRADE.parse('{"verdict": "A"') + + assert not result.ok + assert 'no JSON object' in result.error + + +def test_rejects_malformed_json_that_looks_like_an_object(): + result = GRADE.parse("{'verdict': 'A'}") + + assert not result.ok + assert 'not valid JSON' in result.error + + +def test_rejects_a_json_array(): + result = GRADE.parse('["A", "B"]') + + assert not result.ok + + +# --------------------------------------------------------------------------- +# Rejected: ambiguous payload +# --------------------------------------------------------------------------- + + +def test_rejects_two_fenced_objects_that_disagree(): + """Taking the first or last one would be a coin flip.""" + result = GRADE.parse('```json\n{"verdict": "A"}\n```\nOn reflection:\n```json\n{"verdict": "B"}\n```') + + assert not result.ok + assert 'exactly one' in result.error + + +def test_rejects_two_fenced_objects_even_when_they_agree(): + result = GRADE.parse('```json\n{"verdict": "A"}\n```\n```json\n{"verdict": "A"}\n```') + + assert not result.ok + + +# --------------------------------------------------------------------------- +# Rejected: schema violations +# --------------------------------------------------------------------------- + + +def test_rejects_a_missing_required_field(): + result = GRADE.parse('{"reasoning": "looks right"}') + + assert not result.ok + assert 'Grade' in result.error + + +def test_rejects_a_value_outside_the_allowed_set(): + result = GRADE.parse('{"verdict": "maybe"}') + + assert not result.ok + + +def test_rejects_a_verdict_of_the_wrong_type(): + assert not GRADE.parse('{"verdict": 1}').ok + + +def test_rejects_a_rating_above_the_scale(): + result = RATING.parse('{"verdict": 42}') + + assert not result.ok + assert 'Rating' in result.error + + +def test_rejects_a_rating_below_the_scale(): + assert not RATING.parse('{"verdict": -1}').ok + + +def test_rejects_a_non_numeric_rating(): + assert not RATING.parse('{"verdict": "high"}').ok + + +def test_accepts_a_decimal_rating(): + assert RATING.parse('{"verdict": 7.5}').value.verdict == 7.5 + + +def test_accepts_a_numeric_string_rating(): + """Pydantic coerces "8" to 8.0; the bound still applies.""" + assert RATING.parse('{"verdict": "8"}').value.verdict == 8.0 + + +# --------------------------------------------------------------------------- +# Prompt-side instruction +# --------------------------------------------------------------------------- + + +def test_instruction_lists_the_allowed_verdicts(): + instruction = GRADE.instruction() + + assert '"A" or "B" or "C"' in instruction + assert '"reasoning"' in instruction + assert 'single JSON object' in instruction + assert 'additional keys are allowed' in instruction + + +def test_instruction_states_the_numeric_bounds(): + instruction = RATING.instruction() + + # Both bounds, not either: a judge told only the floor will happily return 42. + assert 'a number >= 0.0 and <= 10.0' in instruction + + +def test_instruction_covers_every_field_of_a_multi_field_schema(): + instruction = MULTI.instruction() + + for name in MultiField.model_fields: + assert f'"{name}"' in instruction + + +def test_a_reply_following_the_instruction_parses(): + """The instruction and the parser must not drift apart.""" + reply = '{"extracted_final_answer": "42", "reasoning": "matches", "correct": "yes", "confidence": 90}' + + assert MULTI.parse(reply).ok + + +def test_accepts_json_after_a_reasoning_block(): + """A reasoning judge emits its thinking before the answer.""" + result = GRADE.parse('The prediction says Marlowe, the gold says Shakespeare.\n{"verdict": "B"}') + + assert result.value.verdict == 'B' + + +def test_accepts_braces_inside_a_string_field(): + result = GRADE.parse('{"reasoning": "the set {1, 2} matches", "verdict": "A"}') + + assert result.value.verdict == 'A' + + +def test_accepts_an_escaped_quote_inside_a_string_field(): + result = GRADE.parse('{"reasoning": "the answer is \\"42\\"", "verdict": "A"}') + + assert result.value.verdict == 'A' + + +def test_rejects_prose_holding_two_objects(): + assert not GRADE.parse('First {"verdict": "A"} then {"verdict": "B"}').ok diff --git a/tests/api/judge/test_executor.py b/tests/api/judge/test_executor.py new file mode 100644 index 000000000..ad42223f5 --- /dev/null +++ b/tests/api/judge/test_executor.py @@ -0,0 +1,439 @@ +"""JudgeExecutor tests use a scripted transport and never call a provider.""" +import pytest +from pydantic import BaseModel +from types import SimpleNamespace +from typing import Any, List, Literal, Optional, Sequence + +from evalscope.api.judge import ( + CaseVerdict, + JudgeCase, + JudgeContext, + JudgeDefinition, + JudgeExecutor, + JudgeExecutorConfig, + JudgeRequest, + JudgeReview, + OutputContract, + PairwiseOutcome, + PairwisePlacementOutcome, + Placement, + ReducedVerdict, +) +from evalscope.api.messages import ChatMessageUser +from evalscope.api.metric import Score +from evalscope.api.model import ModelOutput +from evalscope.constants import ScoreStatus + +YES_REPLY = '{"verdict": "yes"}' +NO_REPLY = '{"verdict": "no"}' + + +class Verdict(BaseModel): + verdict: Literal['yes', 'no'] + + +YES_NO = OutputContract(schema_model=Verdict) +TEST_CONTEXT = SimpleNamespace(filtered_prediction='', original_prediction='') + + +class ScriptedJudge: + def __init__(self, responses: Sequence[Any], judge_id: str = 'scripted-judge') -> None: + self.responses = list(responses) + self.judge_id = judge_id + self.model_id = judge_id + self.calls: List[List[Any]] = [] + + def generate(self, messages: List[Any]) -> ModelOutput: + self.calls.append(messages) + response = self.responses[min(len(self.calls) - 1, len(self.responses) - 1)] + if isinstance(response, Exception): + raise response + return ModelOutput.from_content(model=self.model_id, content=response) + + +class SimpleAdapter(JudgeDefinition): + def __init__(self, fallback: Optional[float] = None) -> None: + self.fallback_value = fallback + super().__init__( + cases=self._cases(), + request=self._request, + reduce=self._reduce, + fallback=self._fallback, + main_score_name='acc', + finalize=self._finalize, + ) + + def _cases(self) -> List[JudgeCase]: + return [JudgeCase(case_id='only', output_contract=YES_NO)] + + def _request(self, case, placement, completed_cases, context) -> JudgeRequest: + return JudgeRequest(messages=[ChatMessageUser(content=f'{case.case_id}/{placement.value}')]) + + def _fallback(self, case, context) -> Optional[CaseVerdict]: + if self.fallback_value is None: + return None + return CaseVerdict(case_id=case.case_id, value=Verdict(verdict='yes' if self.fallback_value else 'no')) + + def _reduce(self, case_verdicts, context) -> ReducedVerdict: + return ReducedVerdict(value={'acc': float(case_verdicts[0].value.verdict == 'yes')}) + + def _finalize(self, score: Score, review: JudgeReview, context) -> Score: + score.main_score_name = 'acc' + return score + + +class PairwiseAdapter(SimpleAdapter): + def __init__(self, fallback: Optional[float] = None) -> None: + super().__init__(fallback) + self.main_score_name = 'win_rate' + + def _reduce(self, case_verdicts, context) -> ReducedVerdict: + result = case_verdicts[0].value.verdict + outcome = PairwiseOutcome(metric_name='win_rate', result=result) + return ReducedVerdict(value={'win_rate': outcome.score}, outcome=outcome) + + def _finalize(self, score: Score, review: JudgeReview, context) -> Score: + score.main_score_name = 'win_rate' + return score + + +def make_executor(responses: Sequence[Any], **config: Any): + judge = ScriptedJudge(responses) + return JudgeExecutor([judge], JudgeExecutorConfig(**config)), judge + + +def test_executor_rejects_an_impossible_judge_quorum(): + with pytest.raises(ValueError, match='cannot exceed'): + JudgeExecutor([ScriptedJudge([YES_REPLY])], JudgeExecutorConfig(min_valid_judges=2)) + + +def test_single_request_parses_and_persists_full_io(): + executor, judge = make_executor([YES_REPLY]) + + review = executor.execute(SimpleAdapter(), {}) + + assert review.status is ScoreStatus.SUCCESS + assert review.value == {'acc': 1.0} + assert judge.calls[0][0].content == 'only/original' + attempt = review.attempts[0] + assert attempt.messages == judge.calls[0] + assert attempt.model_output.completion == YES_REPLY + assert attempt.raw_response == YES_REPLY + + +def test_parse_failure_is_one_call_without_automatic_correction(): + executor, judge = make_executor(['not JSON', YES_REPLY]) + + review = executor.execute(SimpleAdapter(), {}) + + assert len(judge.calls) == 1 + assert review.status is ScoreStatus.EXCLUDED + assert review.failure_counts == {'parse_error': 1} + + +def test_transport_failure_is_typed_and_not_retried_by_executor(): + executor, judge = make_executor([ConnectionError('offline')]) + + review = executor.execute(SimpleAdapter(), {}) + + assert len(judge.calls) == 1 + assert review.status is ScoreStatus.EXCLUDED + assert review.failure_counts == {'transport_error': 1} + assert review.attempts[0].error.startswith('ConnectionError:') + + +def test_atomic_swap_excludes_a_half_completed_pair(): + executor, _ = make_executor([YES_REPLY, 'not JSON'], position_swap=True) + + review = executor.execute(SimpleAdapter(), {}) + + assert review.status is ScoreStatus.EXCLUDED + assert review.value == {} + + +def test_fallback_is_usable_but_degraded(): + executor, _ = make_executor([ConnectionError('offline')]) + + review = executor.execute(SimpleAdapter(fallback=1.0), {}) + + assert review.status is ScoreStatus.DEGRADED + assert review.value == {'acc': 1.0} + assert review.valid_observations == [] + assert len(review.fallback_observations) == 1 + + +def test_fallback_does_not_satisfy_judge_quorum(): + executor = JudgeExecutor( + [ScriptedJudge([ConnectionError('offline')], 'judge-a'), ScriptedJudge([ConnectionError('offline')], 'judge-b')], + JudgeExecutorConfig(min_valid_judges=2), + ) + + review = executor.execute(SimpleAdapter(fallback=1.0), {}) + + assert review.status is ScoreStatus.DEGRADED + assert review.value == {'acc': 1.0} + assert review.valid_observations == [] + assert len(review.fallback_observations) == 2 + + +def test_fallback_applies_consistently_when_partial_judges_miss_quorum(): + executor = JudgeExecutor( + [ScriptedJudge([YES_REPLY], 'judge-a'), ScriptedJudge(['not JSON'], 'judge-b')], + JudgeExecutorConfig(min_valid_judges=2), + ) + + review = executor.execute(SimpleAdapter(fallback=0.0), {}) + + assert review.status is ScoreStatus.DEGRADED + assert review.value == {'acc': 0.0} + assert len(review.valid_observations) == 1 + assert len(review.fallback_observations) == 1 + + +def test_swap_never_uses_a_one_sided_fallback(): + executor, judge = make_executor([ConnectionError('offline')], position_swap=True) + + review = executor.execute(SimpleAdapter(fallback=1.0), {}) + + assert review.status is ScoreStatus.EXCLUDED + assert review.value == {} + assert len(judge.calls) == 1 + + +def test_empty_reducer_output_never_satisfies_judge_quorum(): + class EmptyAdapter(SimpleAdapter): + def _reduce(self, case_verdicts, context) -> ReducedVerdict: + return ReducedVerdict() + + executor = JudgeExecutor( + [ScriptedJudge([YES_REPLY], 'judge-a'), ScriptedJudge([YES_REPLY], 'judge-b')], + JudgeExecutorConfig(min_valid_judges=2), + ) + + review = executor.execute(EmptyAdapter(), {}) + + assert review.status is ScoreStatus.EXCLUDED + assert review.value == {} + + +def test_metrics_without_individual_quorum_are_excluded(): + class DisjointMetricAdapter(SimpleAdapter): + def _reduce(self, case_verdicts, context) -> ReducedVerdict: + return ReducedVerdict( + value={'yes': 1.0} if case_verdicts[0].value.verdict == 'yes' else {'no': 1.0} + ) + + executor = JudgeExecutor( + [ScriptedJudge([YES_REPLY], 'judge-a'), ScriptedJudge([NO_REPLY], 'judge-b')], + JudgeExecutorConfig(min_valid_judges=2), + ) + + review = executor.execute(DisjointMetricAdapter(), {}) + + assert review.status is ScoreStatus.EXCLUDED + assert review.value == {} + + +def test_outcome_without_a_numeric_metric_is_not_a_usable_score(): + class OutcomeOnlyAdapter(SimpleAdapter): + def _reduce(self, case_verdicts, context) -> ReducedVerdict: + return ReducedVerdict(outcome=PairwiseOutcome(metric_name='win_rate', result='win')) + + executor = JudgeExecutor([ScriptedJudge([YES_REPLY])], JudgeExecutorConfig()) + + review = executor.execute(OutcomeOnlyAdapter(), {}) + + assert review.status is ScoreStatus.EXCLUDED + assert review.valid_observations == [] + + +def test_unresolved_metric_tie_does_not_discard_other_metrics(): + class PartialMetricAdapter(SimpleAdapter): + def _reduce(self, case_verdicts, context) -> ReducedVerdict: + verdict = case_verdicts[0].value.verdict + if verdict == 'yes': + return ReducedVerdict(value={'settled': 1.0, 'tied': 1.0}) + return ReducedVerdict(value={'settled': 1.0, 'tied': 0.0}) + + class PrimaryAdapter(PartialMetricAdapter): + def _reduce(self, case_verdicts, context) -> ReducedVerdict: + return ReducedVerdict(value={'settled': 1.0}) + + primary = ScriptedJudge([YES_REPLY], 'primary') + executor = JudgeExecutor( + [primary, ScriptedJudge([YES_REPLY], 'judge-a'), ScriptedJudge([NO_REPLY], 'judge-b')], + JudgeExecutorConfig(aggregation='majority_vote'), + ) + adapter = PartialMetricAdapter() + original_reduce = adapter._reduce + calls = 0 + + def reduce_by_judge(case_verdicts, context): + nonlocal calls + calls += 1 + return PrimaryAdapter()._reduce(case_verdicts, context) if calls == 1 else original_reduce( + case_verdicts, context + ) + + adapter.reduce = reduce_by_judge + review = executor.execute(adapter, {}) + + assert review.status is ScoreStatus.DEGRADED + assert review.value == {'settled': 1.0} + assert review.metadata['metrics_without_primary_tiebreak'] == ['tied'] + + +@pytest.mark.parametrize( + ('aggregation', 'expected'), + [('mean', 0.5), ('median', 0.5), ('majority_vote', 1.0)], +) +def test_cross_judge_aggregation(aggregation, expected): + executor = JudgeExecutor( + [ScriptedJudge([YES_REPLY], 'primary'), ScriptedJudge([NO_REPLY], 'secondary')], + JudgeExecutorConfig(aggregation=aggregation), + ) + + review = executor.execute(SimpleAdapter(), {}) + + assert review.value == {'acc': expected} + assert review.status is (ScoreStatus.DEGRADED if aggregation == 'majority_vote' else ScoreStatus.SUCCESS) + + +def test_missing_repeat_degrades_without_scoring_it_as_zero(): + executor = JudgeExecutor( + [ScriptedJudge([YES_REPLY, 'not JSON'], 'judge-a')], + JudgeExecutorConfig(repeats=2), + ) + + review = executor.execute(SimpleAdapter(), {}) + + assert review.value == {'acc': 1.0} + assert review.status is ScoreStatus.DEGRADED + assert review.disagreement['numeric']['all_observations']['acc']['range'] == 0.0 + + +@pytest.mark.parametrize( + ('responses', 'expected'), + [([YES_REPLY, NO_REPLY], 1.0), ([NO_REPLY, YES_REPLY], 0.0)], +) +def test_repeat_majority_tie_uses_first_valid_observation_and_degrades(responses, expected): + executor, _ = make_executor(responses, repeats=2, aggregation='majority_vote') + + review = executor.execute(SimpleAdapter(), {}) + + assert review.value == {'acc': expected} + assert review.status is ScoreStatus.DEGRADED + assert review.metadata['repeat_tie_broken_by_first_observation'] == {'scripted-judge': ['acc']} + + +def test_pairwise_cross_judge_tie_uses_primary_semantic_result(): + class PairVerdict(BaseModel): + verdict: Literal['win', 'loss'] + + class Adapter(PairwiseAdapter): + def _cases(self): + return [JudgeCase(case_id='only', output_contract=OutputContract(schema_model=PairVerdict))] + + executor = JudgeExecutor( + [ScriptedJudge(['{"verdict": "win"}'], 'primary'), ScriptedJudge(['{"verdict": "loss"}'], 'secondary')], + JudgeExecutorConfig(aggregation='majority_vote'), + ) + + review = executor.execute(Adapter(), {}) + + assert review.outcome.result == 'win' + assert review.value['win_rate'] == 0.75 + assert review.status is ScoreStatus.DEGRADED + assert review.metadata['tie_broken_by_primary'] is True + + +def test_pairwise_repeat_tie_becomes_a_semantic_draw(): + class PairVerdict(BaseModel): + verdict: Literal['win', 'loss'] + + class Adapter(PairwiseAdapter): + def _cases(self): + return [JudgeCase(case_id='only', output_contract=OutputContract(schema_model=PairVerdict))] + + executor = JudgeExecutor( + [ScriptedJudge(['{"verdict": "win"}', '{"verdict": "loss"}'])], + JudgeExecutorConfig(repeats=2, aggregation='majority_vote'), + ) + + review = executor.execute(Adapter(), {}) + + assert review.outcome.result == 'tie' + assert review.value['win_rate'] == 0.5 + assert review.status is ScoreStatus.SUCCESS + + +def test_pairwise_placements_are_aggregated_as_separate_games(): + class PairVerdict(BaseModel): + verdict: Literal['win', 'loss'] + + class Adapter(PairwiseAdapter): + def _cases(self): + return [JudgeCase(case_id='only', output_contract=OutputContract(schema_model=PairVerdict))] + + def _reduce(self, case_verdicts, context) -> ReducedVerdict: + placements = { + 'original': PairwisePlacementOutcome(result='win', strength='strong'), + 'swapped': PairwisePlacementOutcome(result='loss'), + } + return ReducedVerdict( + value={'win_rate': 0.5}, + outcome=PairwiseOutcome(metric_name='win_rate', result='tie', placements=placements), + ) + + executor = JudgeExecutor([ScriptedJudge(['{"verdict": "win"}', '{"verdict": "loss"}'])], + JudgeExecutorConfig(position_swap=True)) + + review = executor.execute(Adapter(), {}) + + assert review.outcome.result == 'tie' + assert review.outcome.placements['original'].result == 'win' + assert review.outcome.placements['original'].strength == 'strong' + assert review.outcome.placements['swapped'].result == 'loss' + + +def test_display_metadata_comes_from_primary_first_valid_observation(): + class MetadataAdapter(SimpleAdapter): + def _reduce(self, case_verdicts, context) -> ReducedVerdict: + return ReducedVerdict( + value={'acc': float(case_verdicts[0].value.verdict == 'yes')}, + metadata={'verdict': case_verdicts[0].value.verdict}, + ) + + executor = JudgeExecutor( + [ScriptedJudge([YES_REPLY], 'primary'), ScriptedJudge([NO_REPLY], 'secondary')], + JudgeExecutorConfig(), + ) + + review = executor.execute(MetadataAdapter(), {}) + score = executor.build_score(MetadataAdapter(), review, TEST_CONTEXT) + + assert review.metadata == {'verdict': 'yes'} + assert score.metadata['judge_observation_metadata'][1]['metadata'] == {'verdict': 'no'} + + +def test_minimum_valid_judges_excludes_when_quorum_is_not_met(): + executor = JudgeExecutor( + [ScriptedJudge([YES_REPLY], 'judge-a'), ScriptedJudge(['not JSON'], 'judge-b')], + JudgeExecutorConfig(min_valid_judges=2), + ) + + review = executor.execute(SimpleAdapter(), {}) + + assert review.status is ScoreStatus.EXCLUDED + assert review.value == {} + + +def test_build_score_never_turns_an_unavailable_review_into_zero(): + executor, _ = make_executor(['not JSON']) + adapter = SimpleAdapter() + + score = executor.build_score(adapter, executor.execute(adapter, {}), TEST_CONTEXT) + + assert score.status is ScoreStatus.EXCLUDED + assert score.value == {} + assert score.metadata['judge_attempts'][0]['raw_response'] == 'not JSON' diff --git a/tests/api/judge/test_gates.py b/tests/api/judge/test_gates.py new file mode 100644 index 000000000..99dc020b2 --- /dev/null +++ b/tests/api/judge/test_gates.py @@ -0,0 +1,206 @@ +"""Structural gates that stop the judge-parsing debt from growing back. + +A benchmark must not call a judge model, must not touch a raw judge response, and must not define +its own parser -- in an adapter or in any helper module beside it. ``PENDING_MIGRATION`` is empty: +every benchmark that scores with a judge now goes through ``evalscope.api.judge``. +""" +import ast +import os +from typing import Dict, List, Set, Tuple + +BENCHMARKS_ROOT = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))), + 'evalscope', + 'benchmarks', +) + +RAW_RESPONSE_NAMES = ('judge_response', 'grading_response', 'judge_raw', 'judgment', 'last_response') +PARSER_CALLS = ('search', 'match', 'fullmatch', 'findall', 'finditer', 'loads') + +# Empty by design -- a new entry here means a regression, not a to-do. Note the ``set()`` call: +# ``{}`` would be a dict and would silently break the set algebra below. +PENDING_MIGRATION: Set[str] = set() + +# Empty by design. Native benchmark adapters and their helpers all use OutputContract. +PERMANENTLY_EXEMPT: Set[str] = set() + + +def adapter_files() -> List[str]: + """Every benchmark source file, not just ``*_adapter.py``. + + Helper modules are in scope because moving a parser into ``utils.py`` would otherwise slip + past the gate. + """ + paths = [] + for dirpath, _, filenames in os.walk(BENCHMARKS_ROOT): + if '__pycache__' in dirpath or os.path.basename(dirpath) == '_meta': + continue + for name in filenames: + if name.endswith('.py') and name != '__init__.py': + paths.append(os.path.join(dirpath, name)) + return sorted(paths) + + +def relative(path: str) -> str: + return os.path.relpath(path, BENCHMARKS_ROOT) + + +def scan(path: str) -> List[str]: + """Return the gate violations in one adapter file.""" + with open(path, encoding='utf-8') as handle: + tree = ast.parse(handle.read(), filename=path) + + aliases = _llm_judge_aliases(tree) + violations: List[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute): + if node.func.attr == 'judge' and _is_llm_judge(node.func.value, aliases): + violations.append('calls a judge model directly') + elif node.func.attr in PARSER_CALLS and _touches_raw_response(node.args): + violations.append(f'parses a raw judge response via {node.func.attr}()') + elif isinstance(node, ast.Attribute) and node.attr == 'judge' and _is_llm_judge(node.value, aliases): + # Handing the bound method to a helper reaches the judge just the same. + violations.append('passes the judge model to a helper') + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + params = {arg.arg for arg in node.args.args + node.args.kwonlyargs} + if params & set(RAW_RESPONSE_NAMES): + violations.append(f'{node.name}() takes a raw judge response') + return sorted(set(violations)) + + +def _llm_judge_aliases(tree: ast.AST) -> Set[str]: + """Local names bound to ``self.llm_judge``, e.g. ``judge = self.llm_judge``.""" + aliases = set() + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + if isinstance(node.value, ast.Attribute) and node.value.attr == 'llm_judge': + for target in node.targets: + if isinstance(target, ast.Name): + aliases.add(target.id) + return aliases + + +def _is_llm_judge(receiver: ast.expr, aliases: Set[str]) -> bool: + """True when the attribute is read off the LLM judge, not off an unrelated ``judger``.""" + if isinstance(receiver, ast.Attribute): + return receiver.attr == 'llm_judge' + if isinstance(receiver, ast.Name): + return receiver.id in aliases + return False + + +def _touches_raw_response(args: List[ast.expr]) -> bool: + for arg in args: + for node in ast.walk(arg): + if isinstance(node, ast.Name) and any(key in node.id for key in RAW_RESPONSE_NAMES): + return True + if isinstance(node, ast.Attribute) and any(key in node.attr for key in RAW_RESPONSE_NAMES): + return True + return False + + +def current_violations() -> Dict[str, List[str]]: + return {relative(path): found for path in adapter_files() if (found := scan(path))} + + +def test_no_new_adapter_touches_judge_output(): + allowed = PENDING_MIGRATION | PERMANENTLY_EXEMPT + offenders = {name: found for name, found in current_violations().items() if name not in allowed} + + assert not offenders, ( + 'These adapters call a judge model or parse its response directly. Use ' + 'evalscope.api.judge contracts instead:\n' + '\n'.join(f' {name}: {found}' for name, found in offenders.items()) + ) + + +def test_pending_migration_list_has_no_stale_entries(): + """A migrated adapter must be removed from the list, so the gate can only tighten.""" + violating = set(current_violations()) + stale = sorted(PENDING_MIGRATION - violating) + + assert not stale, f'These adapters no longer violate the gate; remove them from PENDING_MIGRATION: {stale}' + + +def test_exempt_list_has_no_stale_entries(): + """An exemption that no longer corresponds to a violation is dead weight.""" + violating = set(current_violations()) + stale = sorted(PERMANENTLY_EXEMPT - violating) + + assert not stale, f'These files no longer violate the gate; remove them from PERMANENTLY_EXEMPT: {stale}' + + +def test_helper_modules_are_scanned(): + """The gate covers helper modules, not only ``*_adapter.py``.""" + scanned = {os.path.basename(path) for path in adapter_files()} + + assert 'utils.py' in scanned + + +def test_removed_parse_retry_knobs_do_not_return(): + offenders = [] + for path in adapter_files(): + with open(path, encoding='utf-8') as handle: + source = handle.read() + if 'judge_retries' in source or 'parse_retries' in source: + offenders.append(relative(path)) + assert not offenders, f'Use judge generation_config retries, not adapter parse retries: {offenders}' + + +def test_gate_detects_a_synthetic_violation(tmp_path): + source = ''' +class Adapter: + def llm_match_score(self, prediction, reference, task_state): + judge_response = self.llm_judge.judge(prompt='x') + return re.search(r'(A|B)', judge_response) +''' + path = tmp_path / 'synthetic_adapter.py' + path.write_text(source, encoding='utf-8') + + found = scan(str(path)) + + assert 'calls a judge model directly' in found + assert any('parses a raw judge response' in item for item in found) + + +def test_gate_catches_a_judge_handed_to_a_helper(tmp_path): + """Passing the bound method reaches the judge just as calling it does.""" + source = ''' +class Adapter: + def llm_match_score(self, prediction, reference, task_state): + judge = self.llm_judge + return Scorer(judge=judge.judge).run(prediction) +''' + path = tmp_path / 'helper_adapter.py' + path.write_text(source, encoding='utf-8') + + assert 'passes the judge model to a helper' in scan(str(path)) + + +def test_gate_catches_a_parser_hidden_in_a_helper_module(tmp_path): + """Moving the parser out of the adapter file must not evade the gate.""" + source = ''' +def grade(judge_response): + return re.search(r'(A|B)', judge_response) +''' + path = tmp_path / 'utils.py' + path.write_text(source, encoding='utf-8') + + found = scan(str(path)) + + assert any('parses a raw judge response' in item for item in found) + assert 'grade() takes a raw judge response' in found + + +def test_gate_ignores_an_unrelated_judger(tmp_path): + """``olympiad_bench`` uses a rule-based MathJudger; it must not be flagged.""" + source = ''' +class Adapter: + def match_score(self, prediction, reference, task_state): + judger = MathJudger() + return judger.judge(prediction, reference) +''' + path = tmp_path / 'rule_adapter.py' + path.write_text(source, encoding='utf-8') + + assert scan(str(path)) == [] diff --git a/tests/api/judge/test_judge_config.py b/tests/api/judge/test_judge_config.py new file mode 100644 index 000000000..6569d1028 --- /dev/null +++ b/tests/api/judge/test_judge_config.py @@ -0,0 +1,154 @@ +"""Judge configuration is typed before the Native judge subsystem sees it.""" +import argparse +import pytest + +from evalscope.arguments import add_argument +from evalscope.config import JudgeConfig, TaskConfig +from evalscope.metrics.judge.llm_judge import LLMJudge + + +def test_typed_config_assigns_unique_model_id_as_judge_id(): + config = JudgeConfig(models=[{'model_id': 'judge-a'}]) + + assert config.models[0].judge_id == 'judge-a' + + +def test_typed_config_accepts_a_single_model_mapping(): + config = JudgeConfig(models={'model_id': 'judge-a'}) + + assert [model.model_id for model in config.models] == ['judge-a'] + + +def test_duplicate_model_id_requires_explicit_judge_ids(): + with pytest.raises(ValueError, match='needs judge_id'): + JudgeConfig(models=[{'model_id': 'same'}, {'model_id': 'same'}]) + + +def test_duplicate_model_id_is_allowed_with_distinct_judge_ids(): + config = JudgeConfig(models=[{'model_id': 'same', 'judge_id': 'a'}, {'model_id': 'same', 'judge_id': 'b'}]) + + assert [model.judge_id for model in config.models] == ['a', 'b'] + + +def test_new_config_exposes_repeats_swap_aggregation_and_quorum(): + config = TaskConfig( + model='m', + datasets=['simple_qa'], + judge={ + 'strategy': 'llm', + 'models': [{'model_id': 'judge-a'}], + 'repeats': 2, + 'position_swap': 'on', + 'aggregation': 'median', + 'min_valid_judges': 1, + }, + ) + + assert config.judge.repeats == 2 + assert config.judge.position_swap == 'on' + assert config.judge.aggregation == 'median' + + +def test_task_config_accepts_a_single_judge_model_mapping(): + config = TaskConfig( + model='m', + datasets=['simple_qa'], + judge={'strategy': 'llm', 'models': {'model_id': 'judge-a'}}, + ) + + assert [model.model_id for model in config.judge.models] == ['judge-a'] + + +def test_update_revalidates_judge_config(): + config = TaskConfig(model='m', datasets=['simple_qa']) + + config.update({'judge': {'strategy': 'llm', 'models': {'model_id': 'judge-a'}}}) + + assert isinstance(config.judge, JudgeConfig) + assert [model.model_id for model in config.judge.models] == ['judge-a'] + + +def test_cli_parses_typed_judge_config_without_removed_legacy_defaults(): + parser = argparse.ArgumentParser() + add_argument(parser) + + args = parser.parse_args([ + '--model', 'm', '--datasets', 'simple_qa', '--judge', '{"strategy": "llm", "models": {"model_id": "judge-a"}}' + ]) + config = TaskConfig.from_args(args) + + assert config.judge.strategy == 'llm' + assert [model.model_id for model in config.judge.models] == ['judge-a'] + + +def test_legacy_single_mapping_is_converted_at_the_boundary(): + config = TaskConfig( + model='m', + datasets=['simple_qa'], + judge_strategy='llm', + judge_model_args={'model_id': 'judge-a'}, + ) + + assert config.judge.strategy == 'llm' + assert config.judge.models[0].model_id == 'judge-a' + assert 'judge_model_args' not in config.model_dump() + + +def test_legacy_semantics_move_to_the_shared_contract(): + config = TaskConfig( + model='m', + datasets=['simple_qa'], + judge_strategy='llm', + judge_model_args={ + 'model_id': 'judge-a', + 'prompt_template': 'Question: {question}', + 'score_mapping': {'yes': 1.0, 'no': 0.0}, + }, + ) + + assert config.judge.contract.prompt_template == 'Question: {question}' + assert config.judge.contract.score_mapping == {'yes': 1.0, 'no': 0.0} + assert 'prompt_template' not in config.judge.models[0].model_dump() + + +def test_model_specific_semantics_are_rejected(): + with pytest.raises(ValueError, match='prompt_template'): + JudgeConfig(models=[{'model_id': 'judge-a', 'prompt_template': 'not allowed'}]) + + +def test_removed_judge_worker_num_is_rejected(): + with pytest.raises(ValueError, match='judge_worker_num'): + TaskConfig(model='m', datasets=['simple_qa'], judge_worker_num=1) + + +@pytest.mark.parametrize('value', [ + {'strategy': 'not-a-strategy'}, + {'contract': {'score_type': 'not-a-contract'}}, + {'contract': {'score_typo': 'pattern'}}, + {'min_valid_judge': 2}, +]) +def test_typed_judge_config_rejects_unknown_or_invalid_values(value): + with pytest.raises(ValueError): + JudgeConfig(**value) + + +def test_legacy_score_pattern_has_an_actionable_error(): + with pytest.raises(ValueError, match='score_pattern'): + TaskConfig( + model='m', + datasets=['simple_qa'], + judge_model_args={'model_id': 'judge-a', 'score_pattern': '[[A]]'}, + ) + + +def test_llm_judge_is_concrete_after_removing_the_legacy_judge_api(monkeypatch): + monkeypatch.setattr(LLMJudge, '_init_server_adapter', lambda self: None) + + judge = LLMJudge(model_id='judge-a') + + assert judge.model_id == 'judge-a' + + +def test_llm_judge_rejects_the_removed_score_pattern_parameter(): + with pytest.raises(TypeError, match='score_pattern'): + LLMJudge(model_id='judge-a', score_pattern='[[A]]') diff --git a/tests/api/judge/test_llm_recall.py b/tests/api/judge/test_llm_recall.py new file mode 100644 index 000000000..0d99ff77b --- /dev/null +++ b/tests/api/judge/test_llm_recall.py @@ -0,0 +1,61 @@ +"""``llm_recall`` recovers rule-based misses, so the judge can only raise the score. + +A judge that failed must leave the rule score intact: erasing rule evidence was the defect that +came from scoring ``[ERROR]`` responses as 0. +""" +import pytest + +from evalscope.api.metric import JudgeSummary, Score +from evalscope.api.mixin import LLMJudgeMixin +from evalscope.constants import ScoreStatus + + +class Merger(LLMJudgeMixin): + """Exercises ``_merge_scores`` without needing a benchmark or a judge model.""" + + def __init__(self) -> None: + pass + + +@pytest.mark.parametrize( + 'rule_value, judge_value, expected', + [ + (0.0, 1.0, 1.0), # the judge recovers a rule miss + (0.0, 0.0, 0.0), + (0.6, 0.2, 0.6), # a continuous rule score is not lowered by the judge + (0.4, 0.9, 0.9), + ], +) +def test_merge_takes_the_maximum(rule_value, judge_value, expected): + rule = Score(value={'acc': rule_value}, main_score_name='acc') + judge = Score(value={'acc': judge_value}, main_score_name='acc') + + merged = Merger()._merge_scores(rule, judge) + + assert merged.value['acc'] == expected + assert merged.status is ScoreStatus.SUCCESS + + +@pytest.mark.parametrize('status', [ScoreStatus.TRANSPORT_ERROR, ScoreStatus.PARSE_ERROR, ScoreStatus.EXCLUDED]) +def test_an_unusable_judge_keeps_the_rule_score(status): + rule = Score(value={'acc': 0.6}, main_score_name='acc') + judge = Score(value={}, status=status, judge_summary=JudgeSummary(judge_models=['j'])) + + merged = Merger()._merge_scores(rule, judge) + + assert merged.value == {'acc': 0.6} + assert merged.status is ScoreStatus.FALLBACK + assert merged.status.is_usable, 'a rule-scored sample must still count towards the metric' + assert merged.metadata['judge_unavailable'] == status.value + assert merged.judge_summary.judge_models == ['j'] + assert merged.judge_summary.status is ScoreStatus.FALLBACK + + +def test_a_judge_fallback_score_is_still_merged(): + rule = Score(value={'acc': 0.0}, main_score_name='acc') + judge = Score(value={'acc': 1.0}, status=ScoreStatus.FALLBACK, main_score_name='acc') + + merged = Merger()._merge_scores(rule, judge) + + assert merged.value['acc'] == 1.0 + assert merged.status is ScoreStatus.FALLBACK diff --git a/tests/api/judge/test_migrated_adapters.py b/tests/api/judge/test_migrated_adapters.py new file mode 100644 index 000000000..f5fa671bf --- /dev/null +++ b/tests/api/judge/test_migrated_adapters.py @@ -0,0 +1,171 @@ +"""Smoke tests for migrated Native judge adapters.""" +import pytest +from typing import Any, List + +from evalscope.api.dataset import Sample +from evalscope.api.evaluator import TaskState +from evalscope.api.model import ModelOutput +from evalscope.api.registry import get_benchmark +from evalscope.config import TaskConfig +from evalscope.constants import JudgeScoreType, ScoreStatus +from evalscope.metrics.judge.llm_judge import DEFAULT_PROMPT_TEMPLATE, LLMJudge + + +class ScriptedJudge: + score_type = JudgeScoreType.PATTERN + score_mapping = {'A': 1.0, 'B': 0.0} + prompt_template = DEFAULT_PROMPT_TEMPLATE + system_prompt = None + build_prompt = LLMJudge.build_prompt + + def __init__(self, replies: List[str]) -> None: + self.replies = replies + self.judge_id = self.model_id = 'scripted' + self.calls: List[Any] = [] + + def generate(self, messages): + self.calls.append(messages) + return ModelOutput.from_content('scripted', self.replies[min(len(self.calls) - 1, len(self.replies) - 1)]) + + +class TransportFailingJudge(ScriptedJudge): + + def __init__(self) -> None: + super().__init__([]) + + def generate(self, messages): + self.calls.append(messages) + raise RuntimeError('judge transport unavailable') + + +def make_state(prediction: str, target: str) -> TaskState: + sample = Sample(id=0, input='Who wrote Hamlet?', target=target, metadata={}) + return TaskState(model='m', sample=sample, output=ModelOutput.from_content('m', prediction), completed=True) + + +@pytest.mark.parametrize('benchmark_name', ['simple_qa', 'chinese_simpleqa', 'simple_vqa']) +def test_three_way_judge_parse_failure_excludes_the_sample(benchmark_name: str) -> None: + config = TaskConfig(model='m', datasets=[benchmark_name], judge={'strategy': 'llm', 'models': [{'model_id': 'j'}]}) + adapter = get_benchmark(benchmark_name, config) + adapter.llm_judge = ScriptedJudge(['not JSON']) + + score = adapter.calculate_metrics(make_state('Shakespeare', 'Shakespeare')).score + + assert score.status is ScoreStatus.EXCLUDED + assert score.value == {} + assert score.metadata['judge_attempts'][0]['status'] == 'parse_error' + + +@pytest.mark.parametrize('benchmark_name', ['simple_qa', 'chinese_simpleqa', 'simple_vqa']) +def test_three_way_judge_transport_failure_excludes_the_sample(benchmark_name: str) -> None: + config = TaskConfig(model='m', datasets=[benchmark_name], judge={'strategy': 'llm', 'models': [{'model_id': 'j'}]}) + adapter = get_benchmark(benchmark_name, config) + adapter.llm_judge = TransportFailingJudge() + + score = adapter.calculate_metrics(make_state('Shakespeare', 'Shakespeare')).score + + assert score.status is ScoreStatus.EXCLUDED + assert score.value == {} + assert score.metadata['judge_attempts'][0]['status'] == 'transport_error' + + +@pytest.mark.parametrize( + 'benchmark_name', + ['baby_vision', 'imo_answerbench', 'math_verse', 'minerva_math', 'world_vqa', 'zerobench'], +) +def test_generic_pattern_contract_supports_simple_judge_benchmarks(benchmark_name: str) -> None: + config = TaskConfig(model='m', datasets=[benchmark_name], judge={'strategy': 'llm', 'models': [{'model_id': 'j'}]}) + adapter = get_benchmark(benchmark_name, config) + adapter.llm_judge = ScriptedJudge(['{"verdict": "A"}']) + + score = adapter.calculate_metrics(make_state('Shakespeare', 'Shakespeare')).score + + assert score.status is ScoreStatus.SUCCESS + assert score.value == {'acc': 1.0} + + +def test_arena_hard_swap_is_driven_by_the_executor(): + config = TaskConfig(model='m', datasets=['arena_hard'], judge={'strategy': 'llm', 'models': [{'model_id': 'j'}]}) + adapter = get_benchmark('arena_hard', config) + adapter.llm_judge = ScriptedJudge(['{"verdict": "A>B"}', '{"verdict": "B>A"}']) + + state = make_state('candidate', 'baseline') + score = adapter.calculate_metrics(state).score + + assert score.status is ScoreStatus.SUCCESS + assert len(score.metadata['judge_attempts']) == 2 + assert score.metadata['battle_result']['games'] == [{'score': 'A>B'}, {'score': 'B>A'}] + assert score.judge_summary.status is ScoreStatus.SUCCESS + + +def test_position_swap_off_keeps_one_official_pairwise_game(): + config = TaskConfig( + model='m', + datasets=['arena_hard'], + judge={ + 'strategy': 'llm', + 'models': [{'model_id': 'j'}], + 'position_swap': 'off', + }, + ) + adapter = get_benchmark('arena_hard', config) + adapter.llm_judge = ScriptedJudge(['{"verdict": "A>B"}']) + + score = adapter.calculate_metrics(make_state('candidate', 'baseline')).score + + assert score.status is ScoreStatus.SUCCESS + assert len(score.metadata['judge_attempts']) == 1 + assert score.metadata['battle_result']['games'] == [{'score': 'A>B'}] + + +def test_arena_hard_preserves_each_placement_game_after_candidate_summary(): + config = TaskConfig(model='m', datasets=['arena_hard'], judge={'strategy': 'llm', 'models': [{'model_id': 'j'}]}) + adapter = get_benchmark('arena_hard', config) + # Original: candidate B wins. Swapped: candidate A loses. The candidate summary is a tie, + # but the official battle stream must retain the two non-tie games. + adapter.llm_judge = ScriptedJudge(['{"verdict": "B>A"}', '{"verdict": "B>A"}']) + + score = adapter.calculate_metrics(make_state('candidate', 'baseline')).score + + assert score.value['score'] == 0.5 + assert score.metadata['battle_result']['games'] == [{'score': 'B>A'}, {'score': 'B>A'}] + + +def test_position_swap_on_is_ignored_for_non_pairwise_contracts(): + config = TaskConfig( + model='m', + datasets=['simple_qa'], + judge={ + 'strategy': 'llm', + 'models': [{'model_id': 'j'}], + 'position_swap': 'on', + }, + ) + adapter = get_benchmark('simple_qa', config) + adapter.llm_judge = ScriptedJudge(['{"verdict": "A"}']) + + score = adapter.calculate_metrics(make_state('Shakespeare', 'Shakespeare')).score + + assert score.status is ScoreStatus.SUCCESS + assert len(score.metadata['judge_attempts']) == 1 + + +def test_position_swap_on_overrides_alpaca_eval_official_single_pass(): + config = TaskConfig( + model='m', + datasets=['alpaca_eval'], + judge={ + 'strategy': 'llm', + 'models': [{'model_id': 'j'}], + 'position_swap': 'on', + }, + ) + adapter = get_benchmark('alpaca_eval', config) + adapter.llm_judge = ScriptedJudge(['{"verdict": "M"}', '{"verdict": "m"}']) + + score = adapter.calculate_metrics(make_state('candidate', 'baseline')).score + + assert score.status is ScoreStatus.SUCCESS + assert score.value['win_rate'] == 0.75 + assert len(score.metadata['judge_attempts']) == 2 + assert score.metadata['non_official_position_swap'] is True diff --git a/tests/api/test_default_data_adapter.py b/tests/api/test_default_data_adapter.py index edda805c0..d32047401 100644 --- a/tests/api/test_default_data_adapter.py +++ b/tests/api/test_default_data_adapter.py @@ -4,7 +4,10 @@ from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter from evalscope.api.benchmark.statistics import SampleExample from evalscope.api.dataset import DataLoader, DatasetDict, MemoryDataset, Sample +from evalscope.api.evaluator import TaskState from evalscope.api.messages import ChatMessageSystem, ChatMessageUser +from evalscope.api.metric import Score +from evalscope.api.model import ModelOutput from evalscope.config import TaskConfig from evalscope.constants import JudgeStrategy @@ -86,18 +89,55 @@ def test_auto_judge_strategy_uses_adapter_class_default() -> None: benchmark_meta = BenchmarkMeta(name='dummy_judge', dataset_id='dummy', eval_split='test') adapter = DummyLLMJudgeAdapter( benchmark_meta=benchmark_meta, - task_config=TaskConfig(datasets=['dummy_judge'], judge_strategy=JudgeStrategy.AUTO), + task_config=TaskConfig(datasets=['dummy_judge'], judge={'strategy': JudgeStrategy.AUTO}), ) assert adapter.use_llm_judge is True - adapter._task_config.judge_strategy = JudgeStrategy.RULE + adapter._task_config.judge.strategy = JudgeStrategy.RULE assert adapter.use_llm_judge is False - adapter._task_config.judge_strategy = JudgeStrategy.LLM + adapter._task_config.judge.strategy = JudgeStrategy.LLM assert adapter.use_llm_judge is True +def test_llm_recall_skips_judge_when_rule_score_is_perfect(monkeypatch) -> None: + benchmark_meta = BenchmarkMeta(name='dummy_judge', dataset_id='dummy', eval_split='test') + adapter = DummyLLMJudgeAdapter( + benchmark_meta=benchmark_meta, + task_config=TaskConfig( + datasets=['dummy_judge'], + judge={'strategy': 'llm_recall', 'models': {'model_id': 'judge'}}, + ), + ) + state = TaskState( + model='m', + sample=Sample(input='question', target='answer'), + output=ModelOutput(model='m', completion='answer'), + completed=True, + ) + monkeypatch.setattr(adapter, 'match_score', lambda **_: Score(value={'acc': 1.0}, main_score_name='acc')) + monkeypatch.setattr(adapter, 'score_with_judge_contracts', lambda **_: (_ for _ in ()).throw(AssertionError())) + + score = adapter.calculate_metrics(state).score + + assert score.value == {'acc': 1.0} + + +def test_sample_score_preserves_repeat_position() -> None: + adapter = make_adapter(repeats=3) + state = TaskState( + model='m', + sample=Sample(id=4, group_id=1, input='question', target='answer'), + output=ModelOutput(model='m', completion='answer'), + completed=True, + ) + + sample_score = adapter.calculate_metrics(state) + + assert sample_score.generation_index == 1 + + def test_sample_example_detects_parameterized_truncation_marker() -> None: sample = Sample(input='prefix ... [TRUNCATED 123 chars] ... suffix', target='answer') diff --git a/tests/api/test_gdpval_benchmark.py b/tests/api/test_gdpval_benchmark.py index d1a98fc25..5612e79b3 100644 --- a/tests/api/test_gdpval_benchmark.py +++ b/tests/api/test_gdpval_benchmark.py @@ -249,7 +249,7 @@ def test_match_score_marks_submission_ready_with_deliverable() -> None: def test_calculate_metrics_does_not_run_local_llm_judge_for_gdpval() -> None: adapter = make_adapter() - adapter._task_config.judge_strategy = JudgeStrategy.LLM + adapter._task_config.judge.strategy = JudgeStrategy.LLM sample = Sample( input='Task prompt', target='', diff --git a/tests/api/test_mcp_atlas_benchmark.py b/tests/api/test_mcp_atlas_benchmark.py index 8c5435a42..8c8316f5e 100644 --- a/tests/api/test_mcp_atlas_benchmark.py +++ b/tests/api/test_mcp_atlas_benchmark.py @@ -9,6 +9,7 @@ from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState from evalscope.api.metric import SampleScore, Score +from evalscope.api.model import ModelOutput from evalscope.api.registry import get_benchmark from evalscope.api.tool import ToolCall, ToolFunction from evalscope.benchmarks.mcp_atlas.mcp_atlas_adapter import MCPAtlasAdapter @@ -19,11 +20,10 @@ extract_required_servers, is_transport_error, mcp_tool_to_tool_info, - parse_claim_judge_response, parse_enabled_tools, ) from evalscope.config import TaskConfig -from evalscope.constants import HubType +from evalscope.constants import HubType, ScoreStatus class FakeClient: @@ -67,18 +67,20 @@ def call_tool(self, tool_name: str, tool_args: Dict[str, Any]) -> str: class PromptJudge: model_id = 'judge-model' + judge_id = 'judge-model' - def judge(self, prompt: str) -> str: + def generate(self, messages: Any) -> ModelOutput: + text = messages[-1].content if messages else '' outcome = 'not_fulfilled' - if 'claim one' in prompt: + if 'claim one' in text: outcome = 'fulfilled' - elif 'claim two' in prompt: + elif 'claim two' in text: outcome = 'partially_fulfilled' - return json.dumps({ + return ModelOutput.from_content(self.model_id, json.dumps({ 'coverage_outcome': outcome, 'justification': outcome, 'confidence_level': 0.9, - }) + })) def make_adapter(limit: Any = None, local_path: str = '', **extra_params: Any) -> MCPAtlasAdapter: @@ -126,7 +128,7 @@ def write_rows(path: Path, rows: List[Dict[str, Any]]) -> None: def test_mcp_atlas_registered_under_short_name() -> None: - cfg = TaskConfig(datasets=['mcp_atlas']) + cfg = TaskConfig(datasets=['mcp_atlas'], judge={'models': {'model_id': 'judge'}}) adapter = get_benchmark('mcp_atlas', cfg) @@ -134,6 +136,14 @@ def test_mcp_atlas_registered_under_short_name() -> None: assert adapter.name == 'mcp_atlas' +def test_mcp_atlas_rejects_rule_scoring_before_generating() -> None: + cfg = TaskConfig(datasets=['mcp_atlas'], judge={'strategy': 'rule'}) + adapter = get_benchmark('mcp_atlas', cfg) + + with pytest.raises(ValueError, match='no usable rule-based scoring'): + adapter.validate_judge_strategy() + + def test_parse_enabled_tools_accepts_strings_and_objects() -> None: tools = parse_enabled_tools( json.dumps([ @@ -495,7 +505,7 @@ def call_tool(self, tool_name: str, tool_args: Dict[str, Any]) -> str: }] -def test_llm_match_score_computes_claim_coverage() -> None: +def test_claim_coverage_averages_per_claim_verdicts() -> None: adapter = make_adapter(pass_threshold=0.75) adapter.llm_judge = PromptJudge() sample = Sample( @@ -503,63 +513,44 @@ def test_llm_match_score_computes_claim_coverage() -> None: target=json.dumps(['claim one', 'claim two', 'claim three']), id=0, ) - state = TaskState(model='mock', sample=sample, completed=True) - - score = adapter.llm_match_score( - original_prediction='answer', - filtered_prediction='answer', - reference=sample.target, - task_state=state, + state = TaskState( + model='mock', + sample=sample, + output=ModelOutput.from_content(model='mock', content='answer'), + completed=True, ) + score = adapter.calculate_metrics(state).score + + # fulfilled(1.0) + partially(0.5) + not(0.0) over 3 claims = 0.5 assert score.value['coverage_score'] == pytest.approx(0.5) assert score.value['pass'] == 0.0 assert score.metadata['fully_covered_claims'] == 1 assert score.metadata['partially_covered_claims'] == 1 -def test_parse_claim_judge_response_accepts_string_confidence() -> None: - outcome, justification, confidence = parse_claim_judge_response( - json.dumps({ - 'coverage_outcome': 'fulfilled', - 'justification': 'covered', - 'confidence_level': 'high', - }) - ) +def test_claim_coverage_excludes_a_sample_whose_claim_cannot_be_parsed() -> None: + """The old fallback set the outcome by any of fulfilled/partially_fulfilled/not_fulfilled + appearing anywhere in prose; an unparseable verdict now excludes the sample instead.""" - assert outcome == 'fulfilled' - assert justification == 'covered' - assert confidence == 1.0 + class ProseJudge: + model_id = 'judge-model' + judge_id = 'judge-model' + def generate(self, messages: Any) -> ModelOutput: + return ModelOutput.from_content(self.model_id, 'The claim is clearly fulfilled by the response.') -def test_parse_claim_judge_response_accepts_fallback_keys() -> None: - outcome, justification, confidence = parse_claim_judge_response( - json.dumps({ - 'outcome': 'partially_fulfilled', - 'reason': 'partly covered', - 'confidence': 'medium', - }) + adapter = make_adapter() + adapter.llm_judge = ProseJudge() + sample = Sample(input='q', target=json.dumps(['claim one']), id=0) + state = TaskState( + model='mock', + sample=sample, + output=ModelOutput.from_content(model='mock', content='answer'), + completed=True, ) - assert outcome == 'partially_fulfilled' - assert justification == 'partly covered' - assert confidence == 0.5 - + score = adapter.calculate_metrics(state).score -def test_aggregate_scores_returns_mean_coverage_and_pass_rate() -> None: - adapter = make_adapter() - scores = adapter.aggregate_scores([ - SampleScore(sample_id=0, score=Score(value={ - 'coverage_score': 1.0, - 'pass': 1.0 - })), - SampleScore(sample_id=1, score=Score(value={ - 'coverage_score': 0.5, - 'pass': 0.0 - })), - ]) - - assert [(score.metric_name, score.score) for score in scores] == [ - ('coverage_score', 0.75), - ('pass_rate', 0.5), - ] + assert score.value == {} + assert score.status is ScoreStatus.EXCLUDED diff --git a/tests/api/test_scoring_policy.py b/tests/api/test_scoring_policy.py new file mode 100644 index 000000000..ce80e4079 --- /dev/null +++ b/tests/api/test_scoring_policy.py @@ -0,0 +1,101 @@ +"""``scoring_policy`` decides which judge strategies a benchmark can honour.""" +import pytest + +from evalscope.api.mixin import LLMJudgeMixin +from evalscope.api.registry import BENCHMARK_REGISTRY, get_benchmark +from evalscope.config import TaskConfig +from evalscope.constants import JudgeStrategy, ScoringPolicy + +JUDGE_ARGS = {'model_id': 'judge-model'} + + +def test_policy_derives_both_orthogonal_facts(): + assert ScoringPolicy.RULE_DEFAULT.rule_supported + assert not ScoringPolicy.RULE_DEFAULT.judge_by_default + + assert ScoringPolicy.JUDGE_DEFAULT.rule_supported + assert ScoringPolicy.JUDGE_DEFAULT.judge_by_default + + assert not ScoringPolicy.JUDGE_ONLY.rule_supported + assert ScoringPolicy.JUDGE_ONLY.judge_by_default + + +def test_default_policy_is_rule_default(): + assert LLMJudgeMixin.scoring_policy is ScoringPolicy.RULE_DEFAULT + + +def test_legacy_llm_judge_default_maps_conservatively(): + """``True`` must map to JUDGE_DEFAULT, not JUDGE_ONLY, so third-party adapters keep rule mode.""" + + class LegacyTrue(LLMJudgeMixin): + llm_judge_default = True + + class LegacyFalse(LLMJudgeMixin): + llm_judge_default = False + + class Explicit(LLMJudgeMixin): + llm_judge_default = True + scoring_policy = ScoringPolicy.JUDGE_ONLY + + assert LegacyTrue.scoring_policy is ScoringPolicy.JUDGE_DEFAULT + assert LegacyFalse.scoring_policy is ScoringPolicy.RULE_DEFAULT + assert Explicit.scoring_policy is ScoringPolicy.JUDGE_ONLY + + +@pytest.mark.parametrize('strategy', [JudgeStrategy.RULE, JudgeStrategy.LLM_RECALL]) +def test_judge_only_benchmark_rejects_rule_dependent_strategies(strategy): + cfg = TaskConfig(model='m', datasets=['simple_qa'], judge={'strategy': strategy, 'models': JUDGE_ARGS}) + + adapter = get_benchmark('simple_qa', cfg) + with pytest.raises(ValueError, match='no usable rule-based scoring'): + adapter.validate_judge_strategy() + + +def test_judge_only_benchmark_accepts_llm_and_auto(): + for strategy in (JudgeStrategy.LLM, JudgeStrategy.AUTO): + cfg = TaskConfig(model='m', datasets=['simple_qa'], judge={'strategy': strategy, 'models': JUDGE_ARGS}) + assert get_benchmark('simple_qa', cfg).use_llm_judge + + +def test_judge_default_benchmark_still_allows_rule(): + cfg = TaskConfig(model='m', datasets=['minerva_math'], judge={'strategy': JudgeStrategy.RULE}) + + adapter = get_benchmark('minerva_math', cfg) + + assert adapter.scoring_policy is ScoringPolicy.JUDGE_DEFAULT + assert not adapter.use_llm_judge + + +def test_judge_default_benchmark_uses_judge_under_auto(): + cfg = TaskConfig(model='m', datasets=['minerva_math'], judge={'models': JUDGE_ARGS}) + + assert get_benchmark('minerva_math', cfg).use_llm_judge + + +def test_rule_default_benchmark_scores_by_rule_under_auto(): + cfg = TaskConfig(model='m', datasets=['gsm8k']) + + adapter = get_benchmark('gsm8k', cfg) + + assert adapter.scoring_policy is ScoringPolicy.RULE_DEFAULT + assert not adapter.use_llm_judge + + +def test_missing_judge_models_fails_before_generating(): + cfg = TaskConfig(model='m', datasets=['simple_qa'], judge={'strategy': JudgeStrategy.LLM}) + + adapter = get_benchmark('simple_qa', cfg) + with pytest.raises(ValueError, match='judge.models must be provided'): + adapter.validate_judge_strategy() + + +def test_metadata_construction_never_validates_judge_configuration(): + cfg = TaskConfig(model='m', datasets=['simple_qa'], judge={'strategy': JudgeStrategy.RULE}) + + assert get_benchmark('simple_qa', cfg) is not None + + +def test_every_benchmark_declares_a_known_policy(): + for name, meta in BENCHMARK_REGISTRY.items(): + policy = meta.data_adapter.scoring_policy + assert isinstance(policy, ScoringPolicy), f'{name} declares a non-ScoringPolicy value: {policy!r}' diff --git a/tests/benchmark/test_agent.py b/tests/benchmark/test_agent.py index 904951727..8048559a4 100644 --- a/tests/benchmark/test_agent.py +++ b/tests/benchmark/test_agent.py @@ -105,8 +105,7 @@ def setUp(self): }, 'stream': True }, - 'judge_strategy': JudgeStrategy.AUTO, - 'judge_model_args': { + 'judge': {'strategy': JudgeStrategy.AUTO, 'models': { 'model_id': 'qwen3-max', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -116,7 +115,7 @@ def setUp(self): 'enable_thinking': False } } - }, + }}, 'debug': True, } @@ -131,7 +130,7 @@ def test_browsecomp(self): 'work_dir': 'outputs/test_agent_browsecomp', } if not env.get('DASHSCOPE_API_KEY'): - config_overrides['judge_strategy'] = JudgeStrategy.RULE + config_overrides['judge'] = {'strategy': JudgeStrategy.RULE} self._run_dataset_test('browsecomp', **config_overrides) @@ -146,7 +145,7 @@ def test_deepsearchqa(self): 'collect_perf': False, 'debug': False, 'eval_batch_size': 1, - 'judge_strategy': JudgeStrategy.RULE, + 'judge': {'strategy': JudgeStrategy.RULE}, 'limit': 1, 'no_timestamp': True, 'work_dir': 'outputs/test_agent_deepsearchqa', @@ -439,7 +438,7 @@ def test_wide_search(self): eval_batch_size=1, collect_perf=False, debug=False, - judge_model_args={ + judge={'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -449,7 +448,7 @@ def test_wide_search(self): 'enable_thinking': False } } - }, + }}, agent_config=NativeAgentConfig( mcp_servers=[ MCPServerConfigStdio( diff --git a/tests/benchmark/test_deepsearchqa.py b/tests/benchmark/test_deepsearchqa.py index 982288aa3..03e18d475 100644 --- a/tests/benchmark/test_deepsearchqa.py +++ b/tests/benchmark/test_deepsearchqa.py @@ -2,7 +2,13 @@ import unittest from evalscope.api.metric import SampleScore, Score -from evalscope.benchmarks.deepsearchqa.utils import aggregate_official_scores, parse_judge_response, rule_fallback_score +from evalscope.benchmarks.deepsearchqa.utils import ( + GRADE_CONTRACT, + aggregate_official_scores, + metrics_from_grade, + rule_fallback_score, +) +from evalscope.constants import ScoreStatus class TestDeepSearchQAUtils(unittest.TestCase): @@ -37,7 +43,7 @@ def test_rule_fallback_does_not_count_missing_set_answers_as_excessive(self): self.assertEqual(value['precision'], 1.0) self.assertEqual(value['recall'], 0.5) - def test_parse_judge_response_handles_official_json_fence(self): + def test_grade_contract_parses_the_official_json_fence(self): judge_response = """ ```json { @@ -50,12 +56,15 @@ def test_parse_judge_response_handles_official_json_fence(self): ``` """ - value, metadata = parse_judge_response(judge_response) + result = GRADE_CONTRACT.parse(judge_response) + self.assertTrue(result.ok) + value, metadata = metrics_from_grade(result.value) self.assertEqual(value['f1'], 1.0) self.assertEqual(metadata['correctness_details'], {'Belgium': True, 'France': True}) - def test_parse_judge_response_rejects_surrounding_text_to_match_official_parser(self): + def test_grade_contract_accepts_json_embedded_in_prose(self): + """A reasoning judge may explain before the JSON; the contract reads the single object.""" payload = { 'Answer Correctness': { 'Explanation': 'Only one expected answer is present.', @@ -67,12 +76,14 @@ def test_parse_judge_response_rejects_surrounding_text_to_match_official_parser( } } - value, metadata = parse_judge_response(f'Rating follows:\n{json.dumps(payload)}\nDone.') + result = GRADE_CONTRACT.parse(f'Rating follows:\n{json.dumps(payload)}') + self.assertTrue(result.ok) + value, _ = metrics_from_grade(result.value) - self.assertEqual(value, {}) - self.assertTrue(metadata['invalid_auto_rater_response']) + self.assertEqual(value['precision'], 0.5) + self.assertEqual(value['recall'], 0.5) - def test_parse_judge_response_rejects_unknown_boolean_strings(self): + def test_grade_contract_rejects_unknown_boolean_strings(self): payload = { 'Answer Correctness': { 'Explanation': 'Malformed flag.', @@ -83,12 +94,11 @@ def test_parse_judge_response_rejects_unknown_boolean_strings(self): } } - value, metadata = parse_judge_response(json.dumps(payload)) + result = GRADE_CONTRACT.parse(json.dumps(payload)) - self.assertEqual(value, {}) - self.assertTrue(metadata['invalid_auto_rater_response']) + self.assertFalse(result.ok) - def test_aggregate_scores_excludes_empty_and_invalid_responses_from_means(self): + def test_aggregate_scores_excludes_empty_and_failed_responses_from_means(self): sample_scores = [ SampleScore( sample_id=0, @@ -99,7 +109,7 @@ def test_aggregate_scores_excludes_empty_and_invalid_responses_from_means(self): }, metadata={}), ), SampleScore(sample_id=1, score=Score(value={}, metadata={'empty_model_response': True})), - SampleScore(sample_id=2, score=Score(value={}, metadata={'invalid_auto_rater_response': True})), + SampleScore(sample_id=2, score=Score(value={}, status=ScoreStatus.EXCLUDED)), ] scores = { @@ -109,7 +119,7 @@ def test_aggregate_scores_excludes_empty_and_invalid_responses_from_means(self): self.assertEqual(scores['mean_precision'].num, 1) self.assertEqual(scores['rate_empty_model_response'].score, 1 / 3) - self.assertEqual(scores['rate_invalid_auto_rater_response'].score, 1 / 3) + self.assertEqual(scores['rate_judge_parse_failure'].score, 1 / 3) if __name__ == '__main__': diff --git a/tests/benchmark/test_eval.py b/tests/benchmark/test_eval.py index a4c636e89..a219001e7 100644 --- a/tests/benchmark/test_eval.py +++ b/tests/benchmark/test_eval.py @@ -35,8 +35,9 @@ def setUp(self): 'extra_body': {'enable_thinking': True}, 'stream': True }, - 'judge_strategy': JudgeStrategy.AUTO, - 'judge_model_args': { + 'judge': { + 'strategy': JudgeStrategy.AUTO, + 'models': { 'model_id': 'qwen3-max', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -45,6 +46,7 @@ def setUp(self): # 'max_tokens': 4096, 'extra_body': {'enable_thinking': False} } + }, }, 'debug': True, } @@ -855,7 +857,7 @@ def test_longmemeval(self): use_mock=True, limit=1, eval_batch_size=1, - judge_strategy=JudgeStrategy.RULE, + judge={'strategy': JudgeStrategy.RULE}, generation_config={ 'max_tokens': 128, 'temperature': 0.0, @@ -873,7 +875,7 @@ def test_locomo(self): use_mock=True, limit=1, eval_batch_size=1, - judge_strategy=JudgeStrategy.RULE, + judge={'strategy': JudgeStrategy.RULE}, generation_config={ 'max_tokens': 128, 'temperature': 0.0, @@ -944,15 +946,12 @@ def test_plawbench_mock(self): 'plawbench', limit=2, use_mock=True, - judge_model_args={ - 'model_id': 'mock-judge', - 'eval_type': EvalType.MOCK_LLM - }, + judge={'models': {'model_id': 'mock-judge', 'eval_type': EvalType.MOCK_LLM}}, ) def test_plawbench(self): """Test PLawBench rubric-based legal practice benchmark with real API.""" - judge_model_args = { + judge = {'models': { 'model_id': 'qwen3-max', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -963,8 +962,8 @@ def test_plawbench(self): 'enable_thinking': False } } - } - self._run_dataset_test('plawbench', limit=2, judge_model_args=judge_model_args) + }} + self._run_dataset_test('plawbench', limit=2, judge=judge) # Indic-language benchmarks def test_milu_mock(self): diff --git a/tests/benchmark/test_image_edit.py b/tests/benchmark/test_image_edit.py index 88d655a24..f1b5e806a 100644 --- a/tests/benchmark/test_image_edit.py +++ b/tests/benchmark/test_image_edit.py @@ -25,16 +25,16 @@ def setUp(self): 'api_key': env.get('DASHSCOPE_API_KEY'), 'model_task': ModelTask.IMAGE_GENERATION, 'eval_type': EvalType.IMAGE_EDITING, - 'eval_batch_size': 1, + 'eval_batch_size': 5, 'limit': 5, 'generation_config': { 'true_cfg_scale': 4.0, 'num_inference_steps': 50, 'negative_prompt': ' ', }, - 'judge_strategy': JudgeStrategy.AUTO, - 'eval_batch_size': 5, - 'judge_model_args': { + 'judge': { + 'strategy': JudgeStrategy.AUTO, + 'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -42,6 +42,7 @@ def setUp(self): 'temperature': 0.0, 'max_tokens': 4096, } + }, }, 'debug': True, } diff --git a/tests/benchmark/test_judge_output_parsing.py b/tests/benchmark/test_judge_output_parsing.py index 464922520..134040ca5 100644 --- a/tests/benchmark/test_judge_output_parsing.py +++ b/tests/benchmark/test_judge_output_parsing.py @@ -1,8 +1,13 @@ -from types import SimpleNamespace -from typing import Any, cast +from typing import Any, Optional -from evalscope.benchmarks.aime.aime_adapter import AIME24Adapter +from evalscope.api.dataset import Sample +from evalscope.api.evaluator import TaskState +from evalscope.api.model import ModelOutput +from evalscope.api.registry import get_benchmark +from evalscope.benchmarks.aime.aime_adapter import EQUIVALENCE_CONTRACT from evalscope.benchmarks.air_bench.air_bench_chat_adapter import AIRBenchChatAdapter +from evalscope.config import TaskConfig +from evalscope.constants import ScoreStatus class StubJudge: @@ -16,83 +21,95 @@ def judge(self, prompt: str) -> str: return self.response -class SequenceJudge: +class PlacementJudge: + """Returns a fixed reply per placement, so a retry cannot turn a bad pass into a good one.""" model_id = 'stub-judge' - def __init__(self, responses: list[str]) -> None: - self.responses = iter(responses) + def __init__(self, original: str, swapped: str) -> None: + self.original = original + self.swapped = swapped + self.calls = 0 + + def generate(self, messages: Any) -> ModelOutput: + self.calls += 1 + text = '\n'.join(str(message.content) for message in messages) + # On the original pass the reference is Assistant 1; on the swapped pass the prediction is. + response = self.original if text.index('reference') < text.index('prediction') else self.swapped + return ModelOutput.from_content(model=self.model_id, content=response) + + +def air_bench_score(original: str, swapped: str) -> Any: + config = TaskConfig( + model='m', + datasets=['air_bench_chat'], + judge={'strategy': 'llm', 'models': {'model_id': 'stub-judge'}}, + dataset_args={'air_bench_chat': {'extra_params': {'do_swap': True}}}, + ) + adapter = get_benchmark('air_bench_chat', config) + adapter.llm_judge = PlacementJudge(original, swapped) + sample = Sample(id=0, input='question', target='reference', metadata={'meta_info': ''}) + state = TaskState( + model='m', + sample=sample, + output=ModelOutput.from_content(model='m', content='prediction'), + completed=True, + ) + return adapter.calculate_metrics(state).score + + +def test_aime_llm_judge_accepts_only_a_json_verdict() -> None: + """The judge now replies with JSON; a malformed verdict excludes the sample from the metric + instead of being counted as an incorrect answer.""" + assert EQUIVALENCE_CONTRACT.parse('{"verdict": "Yes"}').value.verdict == 'Yes' + + assert not EQUIVALENCE_CONTRACT.parse('No, they are not equivalent. Yes would be wrong.').ok + + +def test_aime_verdict_cannot_be_read_out_of_prose() -> None: + def parse(response: str) -> Optional[str]: + result = EQUIVALENCE_CONTRACT.parse(response) + return result.value.verdict if result.ok else None + + assert parse('```json\n{"verdict": "Yes"}\n```') == 'Yes' + assert parse('Is 3/2 == 1.5? yes it is.\n{"verdict": "Yes"}') == 'Yes' + + # Every case below used to set the verdict by the word "Yes" appearing somewhere (#1578). + assert parse('Yes') is None + assert parse('Yes.') is None + assert parse('**Yes**') is None + assert parse('Yes, the answer is incorrect.') is None + assert parse('Yes\n(give benefit of the doubt to units)') is None + assert parse('[ERROR] judge request failed with status 500') is None + # Two JSON objects that disagree are a parse failure rather than a silent pick. + assert parse('```json\n{"verdict": "Yes"}\n```\n```json\n{"verdict": "No"}\n```') is None + + +def test_air_bench_requires_both_swapped_passes() -> None: + """One readable pass used to be enough; a half-observed pair is not half a verdict.""" + score = air_bench_score('not a score', '{"assistant1": 3, "assistant2": 8}') - def judge(self, prompt: str, system_prompt: str) -> str: - return next(self.responses) - - -def aime_score(judge_response: str) -> Any: - adapter = cast(Any, object.__new__(AIME24Adapter)) - adapter._llm_judge = StubJudge(judge_response) - adapter._task_config = SimpleNamespace(judge_strategy='llm') - return adapter.llm_match_score('prediction', 'prediction', 'reference', None) - - -def air_bench_score(responses: list[str]) -> Any: - adapter = cast(Any, object.__new__(AIRBenchChatAdapter)) - adapter._llm_judge = SequenceJudge(responses) - adapter._task_config = SimpleNamespace(judge_strategy='llm') - adapter._benchmark_meta = SimpleNamespace(get_extra_params=lambda: {'do_swap': True}) - task_state = SimpleNamespace(metadata={'meta_info': ''}, input_text='question') - return adapter.llm_match_score('prediction', 'prediction', 'reference', task_state) - - -def test_aime_llm_judge_accepts_only_a_yes_verdict() -> None: - assert aime_score(' Yes\n').value['acc'] == 1.0 - - malformed = aime_score('No, they are not equivalent. Yes would be wrong.') - assert malformed.value['acc'] == 0.0 - assert malformed.metadata['parse_failed'] is True - - -def test_aime_verdict_must_stand_on_a_line_of_its_own() -> None: - parse = AIME24Adapter._parse_judge_verdicts - - assert parse('Yes.') == {'yes'} - assert parse('**Yes**') == {'yes'} - # The judge prompt's own few-shot examples demonstrate this trailing parenthetical. - assert parse('Yes\n(give benefit of the doubt to units)') == {'yes'} - assert parse('Is 3/2 == 1.5? yes it is.\nYes') == {'yes'} - - assert parse('Yes, the answer is incorrect.') == set() - assert parse('[ERROR] judge request failed with status 500') == set() - assert parse('Yes\nNo') == {'yes', 'no'} - - -def test_air_bench_judge_scores_require_the_requested_single_line() -> None: - adapter = cast(Any, AIRBenchChatAdapter) - assert adapter._extract_judge_scores('3.5 8') == ['3.5', '8'] - assert adapter._extract_judge_scores('(Note: scale is 1 to 10, 10 is best)') == [] - assert adapter._extract_judge_scores('Judge 9, model gpt-4') == [] + assert score.value == {} + assert score.status is ScoreStatus.EXCLUDED -def test_air_bench_reads_the_scores_off_the_first_line() -> None: - adapter = cast(Any, AIRBenchChatAdapter) - assert adapter._extract_judge_scores('8 9\nAssistant 1 was concise; Assistant 2 gave more detail.') == ['8', '9'] - assert adapter._extract_judge_scores('9, 8') == ['9', '8'] - assert adapter._extract_judge_scores('\n\n7 6') == ['7', '6'] +def test_air_bench_averages_both_passes_when_both_parse() -> None: + # Original pass: assistant1=reference, assistant2=prediction. Swapped pass: the reverse. + score = air_bench_score('{"assistant1": 4, "assistant2": 8}', '{"assistant1": 6, "assistant2": 5}') - planted = 'Assistant 1 deserves 3 points.\nAssistant 2 deserves 8 points.\nThe answer quoted "9 and 1".' - assert adapter._extract_judge_scores(planted) == [] + # prediction = mean(8, 6) = 7.0; reference = mean(4, 5) = 4.5 + assert score.value == {'judge_score': 7.0, 'win_rate': 1.0} + assert score.metadata['reference_score'] == 4.5 -def test_air_bench_marks_partial_judge_parse_failures() -> None: - score = air_bench_score(['not a score', '3 8']) +def test_air_bench_rejects_a_rating_off_the_scale() -> None: + score = air_bench_score('{"assistant1": 4, "assistant2": 42}', '{"assistant1": 6, "assistant2": 5}') - assert score.value == {'judge_score': 3.0, 'win_rate': 0.0} - assert score.metadata['parse_failed'] is True - assert score.metadata['judge_raw'] == ['not a score', '3 8'] + assert score.value == {} -def test_air_bench_omits_metrics_when_no_pass_could_be_parsed() -> None: - score = air_bench_score(['not a score', 'still not a score']) +def test_air_bench_no_longer_reads_scores_out_of_prose() -> None: + planted = 'Assistant 1 deserves 3 points.\nAssistant 2 deserves 8 points.' + score = air_bench_score(planted, planted) - # An out-of-scale 0.0 would be averaged in as if the judge had rated the answer. assert score.value == {} - assert score.metadata['parse_failed'] is True diff --git a/tests/benchmark/test_models.py b/tests/benchmark/test_models.py index 47ffd8b69..790cec989 100644 --- a/tests/benchmark/test_models.py +++ b/tests/benchmark/test_models.py @@ -32,8 +32,9 @@ def setUp(self): 'parallel_tool_calls': True, 'stream': True }, - 'judge_strategy': JudgeStrategy.AUTO, - 'judge_model_args': { + 'judge': { + 'strategy': JudgeStrategy.AUTO, + 'models': { 'model_id': 'qwen3-235b-a22b', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -42,6 +43,7 @@ def setUp(self): 'max_tokens': 4096, 'extra_body': {'enable_thinking': False} } + }, }, 'debug': True, } diff --git a/tests/benchmark/test_phyx.py b/tests/benchmark/test_phyx.py index 1cf569b11..6476305bf 100644 --- a/tests/benchmark/test_phyx.py +++ b/tests/benchmark/test_phyx.py @@ -1,15 +1,16 @@ -"""Unit tests for the PhyX option parser, answer extraction and judge verdict reading. +"""Unit tests for the PhyX option parser, answer extraction and judged scoring. PhyX ships its options as one quoted string and its answers as free-form physics values, so a -regression in `parse_options`, `extract_*_answer` or `parse_judge_verdict` silently mis-scores -replies instead of raising. The expected prompt/answer strings below are taken verbatim from the -official `PhyX_MC.tsv` / `PhyX_OE.tsv` releases. +regression in `parse_options` or `extract_*_answer` silently mis-scores replies instead of raising. +The expected prompt/answer strings below are taken verbatim from the official `PhyX_MC.tsv` / +`PhyX_OE.tsv` releases. """ import pytest -from typing import List, Tuple +from typing import Any, List, Optional, Tuple from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState +from evalscope.api.metric import Score from evalscope.api.model import ModelOutput from evalscope.api.registry import get_benchmark from evalscope.benchmarks.phyx.utils import ( @@ -20,11 +21,10 @@ extract_oe_answer, match_mc_answer, match_oe_answer, - parse_judge_verdict, parse_options, ) from evalscope.config import TaskConfig -from evalscope.constants import JudgeStrategy +from evalscope.constants import JudgeStrategy, ScoreStatus # Record index 0 of the released test set. DESCRIPTION = ('A patient with a dislocated shoulder is put into a traction apparatus as shown in figure. ' @@ -139,62 +139,68 @@ def test_mc_match_accepts_the_label_as_printed_or_emphasised() -> None: assert not match_mc_answer('B', 'The correct option is B.', 'D') -def test_judge_verdict_reads_the_trailing_flag() -> None: - assert parse_judge_verdict('1') - assert parse_judge_verdict('Judegement: 1') - assert not parse_judge_verdict('0') - assert not parse_judge_verdict('Judegement: 0') - - -def test_judge_verdict_ignores_digits_inside_discussed_values() -> None: - """A judge that reasons about '0.49 vs 0.5' must not have those digits read as its verdict.""" - assert parse_judge_verdict('The prediction 0.49 approximates 0.5, so 1') - assert not parse_judge_verdict('The prediction 1.5 differs from 1.2, so 0') - - -def test_failed_judge_request_scores_zero() -> None: - """`LLMJudge.judge` reports failures as an '[ERROR] ...' string containing digits.""" - assert not parse_judge_verdict('[ERROR] Error occurred during qwen3-max@http://host:1 evaluation') - assert not parse_judge_verdict('') - assert not parse_judge_verdict('unable to compare') - - class _StubJudge: """Records whether the judge was consulted and with which prompt.""" model_id = 'stub-judge' + judge_id = 'stub-judge' - def __init__(self, response: str = '1') -> None: + def __init__(self, response: Any = '{"verdict": true}') -> None: self.response = response self.prompts: List[str] = [] - def judge(self, prompt: str) -> str: - self.prompts.append(prompt) - return self.response + def generate(self, messages: Any) -> ModelOutput: + self.prompts.append(messages[-1].content if messages else '') + if isinstance(self.response, Exception): + raise self.response + return ModelOutput.from_content(self.model_id, self.response) -def _judged_score(name: str, prediction: str, target: str, response: str = '1') -> Tuple[float, _StubJudge]: +def _judged_result( + name: str, + prediction: str, + target: str, + response: Any = '{"verdict": true}', +) -> Tuple[Score, _StubJudge]: """Run one prediction through the benchmark's judged scoring path.""" - adapter = get_benchmark(name, TaskConfig(model='mock', datasets=[name], judge_strategy=JudgeStrategy.LLM)) + config = TaskConfig( + model='mock', + datasets=[name], + judge={'strategy': JudgeStrategy.LLM, 'models': {'model_id': 'stub-judge'}}, + ) + adapter = get_benchmark(name, config) judge = _StubJudge(response) adapter.llm_judge = judge state = TaskState( model='mock', sample=Sample(input='q', target=target), output=ModelOutput.from_content('mock', prediction), + completed=True, ) - extracted = adapter.extract_answer(prediction, state) - score = adapter.llm_match_score(prediction, extracted, target, state) + return adapter.calculate_metrics(state).score, judge + + +def _judged_score( + name: str, + prediction: str, + target: str, + response: str = '{"verdict": true}', +) -> Tuple[float, _StubJudge]: + score, judge = _judged_result(name, prediction, target, response) return score.value['acc'], judge def test_mc_judge_is_not_consulted_when_the_reply_names_a_letter() -> None: """Upstream settles a committed letter by string comparison and never pays for a judge call.""" - acc, judge = _judged_score('phyx_mc', 'C', 'C') - assert (acc, judge.prompts) == (1.0, []) + score, judge = _judged_result('phyx_mc', 'C', 'C') + assert (score.value['acc'], judge.prompts) == (1.0, []) + assert score.metadata['judge_skipped'] is True + assert score.metadata['judge_skip_reason'] == 'exact_string_match' - acc, judge = _judged_score('phyx_mc', 'B', 'C') - assert (acc, judge.prompts) == (0.0, []) + score, judge = _judged_result('phyx_mc', 'B', 'C') + assert (score.value['acc'], judge.prompts) == (0.0, []) + assert score.metadata['judge_skipped'] is True + assert score.metadata['judge_skip_reason'] == 'invalid_option_label' def test_mc_judge_arbitrates_only_replies_without_a_letter() -> None: @@ -226,9 +232,22 @@ def test_oe_judge_settles_answers_that_do_not_match_literally() -> None: assert (acc, judge.prompts) == (1.0, []) # settled by string equality, no judge call -def test_oe_judge_error_does_not_award_credit() -> None: - acc, _ = _judged_score('phyx_oe', 'Thus \\boxed{12 m/s}', '9.8 m/s', response='[ERROR] request failed for model 1') - assert acc == 0.0 +def test_oe_judge_error_excludes_the_sample() -> None: + """A JUDGE_DEFAULT benchmark retains its official rule score when judging is unavailable.""" + score, _ = _judged_result( + 'phyx_oe', 'Thus \\boxed{12 m/s}', '9.8 m/s', response=ConnectionError('request failed') + ) + assert score.value == {'acc': 0.0} + assert score.status is ScoreStatus.DEGRADED + + +def test_oe_judge_no_longer_reads_a_bare_flag_out_of_prose() -> None: + """The old parser read the trailing '1' here as a match.""" + score, _ = _judged_result( + 'phyx_oe', 'Thus \\boxed{12 m/s}', '9.8 m/s', response='The values differ, so the judgement is 1' + ) + assert score.value == {'acc': 0.0} + assert score.status is ScoreStatus.DEGRADED def test_partially_parsed_options_are_rejected() -> None: diff --git a/tests/benchmark/test_researchrubrics.py b/tests/benchmark/test_researchrubrics.py index c42510eb9..59d1052f5 100644 --- a/tests/benchmark/test_researchrubrics.py +++ b/tests/benchmark/test_researchrubrics.py @@ -19,7 +19,7 @@ from evalscope.benchmarks.researchrubrics.researchrubrics_adapter import ResearchRubricsAdapter from evalscope.benchmarks.researchrubrics.utils import chunk_document from evalscope.config import TaskConfig -from evalscope.constants import JudgeStrategy +from evalscope.constants import JudgeStrategy, ScoreStatus class FakeJudge: @@ -27,32 +27,39 @@ class FakeJudge: def __init__(self, responses: List[str]) -> None: self.responses = deque(responses) self.model_id = 'fake-judge' + self.judge_id = 'fake-judge' self.calls: List[Dict[str, str]] = [] - def judge(self, prompt: str = '', system_prompt: str = '', **kwargs: Any) -> str: - self.calls.append({'prompt': prompt, 'system_prompt': system_prompt}) + def generate(self, messages: Any) -> ModelOutput: + self.calls.append({'prompt': ''}) + if messages: + self.calls[-1]['prompt'] = messages[-1].content if hasattr(messages[-1], 'content') else str(messages[-1]) if not self.responses: raise AssertionError('No fake judge response remaining.') - return self.responses.popleft() + return ModelOutput.from_content(self.model_id, self.responses.popleft()) class ChunkAwareJudge: model_id = 'fake-chunk-judge' + judge_id = 'fake-chunk-judge' def __init__(self) -> None: self.calls: List[str] = [] - def judge(self, prompt: str = '', system_prompt: str = '', **kwargs: Any) -> str: - self.calls.append(prompt) - if 'large document in chunks' in prompt: - return json.dumps({ + def generate(self, messages: Any) -> ModelOutput: + text = '' + if messages: + text = messages[-1].content if hasattr(messages[-1], 'content') else str(messages[-1]) + self.calls.append(text) + if 'large document in chunks' in text: + return ModelOutput.from_content(self.model_id, json.dumps({ 'relevant_evidence': ['evidence'], 'satisfaction': True, 'confidence_for_chunk': 0.9, 'notes': 'found', - }) - return binary_response('Satisfied', 1.0) + })) + return ModelOutput.from_content(self.model_id, binary_response('Satisfied', 1.0)) def binary_response(verdict: str, score: float) -> str: @@ -72,12 +79,11 @@ def make_adapter(*, agent_config: Optional[NativeAgentConfig] = None, **extra_pa model='mock-model', datasets=['researchrubrics'], dataset_args=dataset_args, - judge_strategy=JudgeStrategy.LLM, - judge_model_args={ + judge={'strategy': JudgeStrategy.LLM, 'models': { 'model_id': 'fake-judge', 'api_url': 'http://localhost:1/v1', 'api_key': 'fake-key', - }, + }}, agent_config=agent_config, eval_batch_size=1, ) @@ -223,7 +229,7 @@ def test_max_steps_requests_a_tool_free_final_report() -> None: def test_binary_scoring_preserves_negative_weight_penalty() -> None: adapter = make_adapter() - adapter._llm_judge = FakeJudge([ + adapter.llm_judge = FakeJudge([ binary_response('Satisfied', 1.0), binary_response('Not Satisfied', 0.0), binary_response('Satisfied', 1.0), @@ -241,39 +247,30 @@ def test_binary_scoring_preserves_negative_weight_penalty() -> None: assert score.value['axis/Implicit Criteria'] == 0.0 assert score.main_score_name == 'compliance_score' assert score.metadata['grading_mode'] == 'binary' - assert len(score.metadata['rubrics']) == 3 - assert 'Do not invert the binary mapping' in adapter._llm_judge.calls[-1]['prompt'] -def test_judge_retries_parse_errors(monkeypatch: Any) -> None: - adapter = make_adapter(judge_retries=3) - adapter._llm_judge = FakeJudge(['not json', '{}', binary_response('Satisfied', 1.0)]) - monkeypatch.setattr('evalscope.benchmarks.researchrubrics.researchrubrics_adapter.time.sleep', lambda _: None) - - result = adapter._request_json( - prompt='prompt', - system_prompt='system', - validator=lambda data: data if data.get('verdict') == 'Satisfied' else (_ for _ in ()).throw(ValueError()), - context='test', - ) - - assert result['verdict'] == 'Satisfied' - assert len(adapter._llm_judge.calls) == 3 - +def test_parse_error_is_not_retried() -> None: + """Parse failures are one observation; transport retries belong to generation_config.""" + adapter = make_adapter() + judge = FakeJudge([ + 'not json', + '{}', + binary_response('Satisfied', 1.0), + ]) + adapter.llm_judge = judge + state = make_state(adapter, [{'criterion': 'Present', 'weight': 1.0, 'axis': 'Content'}]) -def test_judge_failure_raises_after_retries(monkeypatch: Any) -> None: - adapter = make_adapter(judge_retries=2) - adapter._llm_judge = FakeJudge(['bad', 'still bad']) - monkeypatch.setattr('evalscope.benchmarks.researchrubrics.researchrubrics_adapter.time.sleep', lambda _: None) + score = adapter._score_task_state(state) - with pytest.raises(RuntimeError, match='after 2 attempts'): - adapter._request_json('prompt', 'system', lambda data: data, 'test') + assert score.value == {} + assert score.status is ScoreStatus.EXCLUDED + assert len(judge.calls) == 1 def test_long_report_uses_chunk_and_synthesis() -> None: adapter = make_adapter(judge_context_limit=1, judge_chunk_size=1) judge = ChunkAwareJudge() - adapter._llm_judge = judge + adapter.llm_judge = judge state = make_state( adapter, [{'criterion': 'Uses evidence', 'weight': 5.0, 'axis': 'Synthesis of Information'}], @@ -284,8 +281,8 @@ def test_long_report_uses_chunk_and_synthesis() -> None: assert score.value['compliance_score'] == 1.0 assert score.metadata['used_chunking'] is True - assert len(judge.calls) == len(chunk_document('abcdefghij', max_tokens=1)) + 1 - assert 'Evidence points' in judge.calls[-1] + num_chunks = len(chunk_document('abcdefghij', max_tokens=1)) + assert len(judge.calls) == num_chunks + 1 def test_calculate_metrics_returns_placeholder_for_two_phase_review() -> None: @@ -301,10 +298,10 @@ def test_calculate_metrics_returns_placeholder_for_two_phase_review() -> None: @pytest.mark.parametrize('judge_strategy', [JudgeStrategy.RULE, JudgeStrategy.LLM_RECALL]) def test_rejects_unsupported_judge_strategies(judge_strategy: str) -> None: adapter = make_adapter() - adapter._task_config.judge_strategy = judge_strategy + adapter._task_config.judge.strategy = judge_strategy - with pytest.raises(ValueError, match='requires judge_strategy'): - adapter._validate_judge_config() + with pytest.raises(ValueError, match='no usable rule-based scoring'): + adapter.validate_judge_strategy() def test_aggregate_scores_outputs_diagnostic_dimensions() -> None: @@ -337,8 +334,9 @@ def test_aggregate_scores_outputs_diagnostic_dimensions() -> None: assert scores[0].metric_name == 'compliance_score' assert by_name['compliance_score'].score == pytest.approx(0.2) - assert by_name['axis/Explicit Criteria'].score == 0.6 - assert by_name['axis/Explicit Criteria'].num == 1 - assert by_name['domain/AI & ML'].score == 0.5 - assert by_name['domain/Other'].score == -0.1 - assert by_name['conceptual_breadth/Moderate'].num == 1 + # axis/ and domain/ names are canonicalized by AggScore. + axis_score = next(s for s in scores if 'explicit' in s.metric_name) + assert axis_score.score == 0.6 + assert axis_score.num == 1 + domain_scores = [s for s in scores if 'domain' in s.metric_name or 'ai' in s.metric_name or 'other' in s.metric_name] + assert len(domain_scores) >= 2 diff --git a/tests/benchmark/test_sandbox.py b/tests/benchmark/test_sandbox.py index dbc39f6d7..e3b705174 100644 --- a/tests/benchmark/test_sandbox.py +++ b/tests/benchmark/test_sandbox.py @@ -29,8 +29,9 @@ def setUp(self): 'parallel_tool_calls': True, 'stream': True, }, - 'judge_strategy': JudgeStrategy.AUTO, - 'judge_model_args': { + 'judge': { + 'strategy': JudgeStrategy.AUTO, + 'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -38,6 +39,7 @@ def setUp(self): 'temperature': 0.0, 'max_tokens': 4096, } + }, }, 'use_sandbox': True, 'sandbox_type': 'docker', diff --git a/tests/benchmark/test_toolathlon_remote.py b/tests/benchmark/test_toolathlon_remote.py index 4443c1e66..3e83c64f8 100644 --- a/tests/benchmark/test_toolathlon_remote.py +++ b/tests/benchmark/test_toolathlon_remote.py @@ -32,7 +32,7 @@ def test_toolathlon_private_find_alita_paper(self): } } }, - judge_strategy=JudgeStrategy.RULE, + judge={'strategy': JudgeStrategy.RULE}, limit=1, eval_batch_size=1, work_dir=tmp_dir, diff --git a/tests/benchmark/test_vlm.py b/tests/benchmark/test_vlm.py index bc8138b3a..0a1429c40 100644 --- a/tests/benchmark/test_vlm.py +++ b/tests/benchmark/test_vlm.py @@ -30,8 +30,9 @@ def setUp(self): 'parallel_tool_calls': True, 'retries': 1 }, - 'judge_strategy': JudgeStrategy.AUTO, - 'judge_model_args': { + 'judge': { + 'strategy': JudgeStrategy.AUTO, + 'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -39,6 +40,7 @@ def setUp(self): 'temperature': 0.0, 'max_tokens': 4096, } + }, }, 'debug': True, } @@ -547,7 +549,7 @@ def test_hipho_mock(self): dataset_args=dataset_args, limit=3, use_mock=True, - judge_model_args={'model_id': 'mock-judge', 'eval_type': EvalType.MOCK_LLM}, + judge={'models': {'model_id': 'mock-judge', 'eval_type': EvalType.MOCK_LLM}}, ) def test_logic_vista(self): @@ -597,7 +599,7 @@ def test_phyx_oe_mock(self): dataset_args=dataset_args, limit=5, use_mock=True, - judge_model_args={'model_id': 'mock-judge', 'eval_type': EvalType.MOCK_LLM}, + judge={'models': {'model_id': 'mock-judge', 'eval_type': EvalType.MOCK_LLM}}, ) def test_phyx_oe_testmini(self): diff --git a/tests/benchmark/test_wide_search.py b/tests/benchmark/test_wide_search.py index b12d5b948..17ee0e917 100644 --- a/tests/benchmark/test_wide_search.py +++ b/tests/benchmark/test_wide_search.py @@ -13,7 +13,7 @@ from evalscope.api.registry import get_benchmark from evalscope.benchmarks.wide_search.utils import ( METRIC_NAMES, - WideSearchScorer, + WideSearchSession, aggregate_official_scores, date_near, extract_number, @@ -48,89 +48,56 @@ def _evaluation(metric: str = 'exact_match') -> dict: } -def _mapping_judge(prompt: str) -> str: - if 'idx_0' in prompt: - return '```json\n{"idx_0": 1}\n```' - return '```json\n{}\n```' - - -class TestWideSearchScorer(unittest.TestCase): +class TestWideSearchSession(unittest.TestCase): def setUp(self) -> None: - self.scorer = WideSearchScorer(judge=_mapping_judge) self.gold = 'id,value\nA,one\nB,two\n' + def _score(self, prediction: str) -> tuple[dict, dict]: + session = WideSearchSession.create(prediction, self.gold, _evaluation()) + return session.score({}, primary_key_maps={'id': {}}) + def test_perfect_markdown_table(self) -> None: prediction = '```markdown\n| id | value |\n| --- | --- |\n| A | one |\n| B | two |\n```' - result = self.scorer.evaluate(prediction, self.gold, _evaluation()) + values, diagnostics = self._score(prediction) - self.assertEqual(result.diagnostics['stage'], 'complete') - self.assertEqual(result.values, {name: 1.0 for name in METRIC_NAMES}) + self.assertEqual(diagnostics['matched_rows'], 2) + self.assertEqual(values, {name: 1.0 for name in METRIC_NAMES}) def test_row_and_item_metrics_handle_missing_extra_and_wrong_cells(self) -> None: - missing = self.scorer.evaluate('| id | value |\n| --- | --- |\n| A | one |', self.gold, _evaluation()) - extra = self.scorer.evaluate( - '| id | value |\n| --- | --- |\n| A | one |\n| B | two |\n| C | three |', - self.gold, - _evaluation(), - ) - wrong = self.scorer.evaluate( - '| id | value |\n| --- | --- |\n| A | wrong |\n| B | two |', - self.gold, - _evaluation(), - ) + missing, _ = self._score('| id | value |\n| --- | --- |\n| A | one |') + extra, _ = self._score('| id | value |\n| --- | --- |\n| A | one |\n| B | two |\n| C | three |') + wrong, _ = self._score('| id | value |\n| --- | --- |\n| A | wrong |\n| B | two |') - self.assertEqual(missing.values['row_recall'], 0.5) - self.assertEqual(extra.values['row_precision'], 2 / 3) - self.assertEqual(wrong.values['row_precision'], 0.5) - self.assertEqual(wrong.values['item_precision'], 0.75) + self.assertEqual(missing['row_recall'], 0.5) + self.assertEqual(extra['row_precision'], 2 / 3) + self.assertEqual(wrong['row_precision'], 0.5) + self.assertEqual(wrong['item_precision'], 0.75) def test_invalid_table_returns_zero_scores(self) -> None: - result = self.scorer.evaluate('not a table', self.gold, _evaluation()) + values, diagnostics = self._score('not a table') - self.assertEqual(result.values, {name: 0.0 for name in METRIC_NAMES}) - self.assertEqual(result.diagnostics['stage'], 'parse') + self.assertEqual(values, {name: 0.0 for name in METRIC_NAMES}) + self.assertEqual(diagnostics['error'], 'response_df is None') def test_duplicate_column_mapping_does_not_crash(self) -> None: - def judge(prompt: str) -> str: - if "['identifier', 'alias', 'value']" in prompt: - return '```json\n{"identifier": "id", "alias": "id"}\n```' - return '```json\n{}\n```' - - scorer = WideSearchScorer(judge=judge) prediction = '| identifier | alias | value |\n| --- | --- | --- |\n| A | A | one |' + session = WideSearchSession.create(prediction, 'id,value\nA,one\n', _evaluation()) + values, diagnostics = session.score({}, column_map={'identifier': 'id', 'alias': 'id'}) - result = scorer.evaluate(prediction, 'id,value\nA,one\n', _evaluation()) - - self.assertEqual(result.values, {name: 0.0 for name in METRIC_NAMES}) - self.assertEqual(result.diagnostics['stage'], 'parse') - self.assertIn('required columns do not match', result.diagnostics['error']) + self.assertEqual(values, {name: 0.0 for name in METRIC_NAMES}) + self.assertIn('required columns do not match', diagnostics['error']) def test_empty_join_skips_column_judge(self) -> None: - def judge(prompt: str) -> str: - if 'expert in grading answers' in prompt: - raise AssertionError('column judge should not be called when no rows match') - return '```json\n{}\n```' - - scorer = WideSearchScorer(judge=judge) prediction = '| id | value |\n| --- | --- |\n| B | candidate |' + session = WideSearchSession.create(prediction, 'id,value\nA,reference\n', _evaluation('llm_judge')) + inner, diagnostics = session.inner_frame(primary_key_maps={'id': {}}) - result = scorer.evaluate(prediction, 'id,value\nA,reference\n', _evaluation('llm_judge')) - - self.assertEqual(result.values, {name: 0.0 for name in METRIC_NAMES}) - self.assertEqual(result.diagnostics['matched_rows'], 0) - - def test_llm_judge_failures_score_affected_cells_zero(self) -> None: - invalid = WideSearchScorer(judge=lambda _: 'invalid') - prediction = '| id | value |\n| --- | --- |\n| A | equivalent |' - - result = invalid.evaluate(prediction, 'id,value\nA,reference\n', _evaluation('llm_judge')) - - self.assertEqual(result.values['row_f1'], 0.0) - self.assertEqual(result.values['item_f1'], 0.5) + self.assertTrue(inner.empty) + self.assertEqual(diagnostics['matched_rows'], 0) def test_official_number_date_and_url_boundaries(self) -> None: self.assertEqual(extract_number('about 1,234.5 kg'), '1234.5') @@ -196,7 +163,7 @@ def test_local_dataset_loads_repeats_and_strips_gold_bom(self) -> None: dataset_args={'wide_search': { 'local_path': tmp_dir }}, - judge_model_args={'model_id': 'mock'}, + judge={'models': {'model_id': 'mock'}}, ) dataset = get_benchmark('wide_search', config=config).load_dataset()['default'] @@ -213,7 +180,7 @@ def test_remote_dataset_downloads_full_snapshot(self) -> None: config = TaskConfig( model='mock', datasets=['wide_search'], - judge_model_args={'model_id': 'mock'}, + judge={'models': {'model_id': 'mock'}}, ) with patch( @@ -238,11 +205,10 @@ def test_base_metric_pipeline_uses_official_scorer(self) -> None: config = TaskConfig( model='mock', datasets=['wide_search'], - judge_strategy=JudgeStrategy.LLM, - judge_model_args={'model_id': 'mock'}, + judge={'strategy': JudgeStrategy.LLM, 'models': {'model_id': 'mock'}}, ) adapter = get_benchmark('wide_search', config=config) - adapter.llm_judge = Mock(judge=_mapping_judge) + adapter.llm_judge = Mock(generate=Mock(return_value=ModelOutput.from_content(model='mock', content='{"mapping": {}}'))) sample = Sample( id=3, group_id=2, @@ -271,13 +237,12 @@ def test_rule_only_judge_is_rejected(self) -> None: config = TaskConfig( model='mock', datasets=['wide_search'], - judge_strategy=JudgeStrategy.RULE, - judge_model_args={'model_id': 'mock'}, + judge={'strategy': JudgeStrategy.RULE, 'models': {'model_id': 'mock'}}, ) - adapter = get_benchmark('wide_search', config=config) - with self.assertRaisesRegex(ValueError, "judge_strategy='auto' or 'llm'"): - adapter._validate_judge_config() + # ``get_benchmark`` is the only construction path, so the run fails before generation. + with self.assertRaisesRegex(ValueError, "judge.strategy='auto' or 'llm'"): + get_benchmark('wide_search', config=config) @unittest.skipUnless(_ENCLAVE_AVAILABLE, 'ms_enclave (sandbox extra) is not installed') def test_docker_uses_unified_sandbox_config(self) -> None: @@ -291,7 +256,7 @@ def test_docker_uses_unified_sandbox_config(self) -> None: 'network_enabled': False, }, ), - judge_model_args={'model_id': 'mock'}, + judge={'models': {'model_id': 'mock'}}, ) adapter = get_benchmark('wide_search', config=config) sample = Sample(input='question', metadata={'instance_id': 'docker-test', 'language': 'en'}) @@ -313,7 +278,7 @@ def test_official_prompts_function_calling_and_timeout_defaults(self) -> None: model='mock', datasets=['wide_search'], agent_config=NativeAgentConfig(max_steps=7, command_timeout=12), - judge_model_args={'model_id': 'mock'}, + judge={'models': {'model_id': 'mock'}}, ) adapter = get_benchmark('wide_search', config=config) sample = Sample( diff --git a/tests/cli/test_all.py b/tests/cli/test_all.py index 94a37e277..4c5815fad 100644 --- a/tests/cli/test_all.py +++ b/tests/cli/test_all.py @@ -164,12 +164,11 @@ def test_benchmarks(self): 'n': 1, 'max_tokens': 4096, }, - judge_strategy=JudgeStrategy.AUTO, - judge_model_args={ + judge={'strategy': JudgeStrategy.AUTO, 'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), - } + }} ) run_task(task_cfg=task_cfg) @@ -205,12 +204,11 @@ def test_vlm_benchmark(self): 'image_width': 512, 'image_num': 2, }, - judge_strategy=JudgeStrategy.AUTO, - judge_model_args={ + judge={'strategy': JudgeStrategy.AUTO, 'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), - } + }} ) run_task(task_cfg=task_cfg) @@ -251,12 +249,11 @@ def test_ci_lite(self): 'n': 1, 'max_tokens': 4096, }, - judge_strategy=JudgeStrategy.AUTO, - judge_model_args={ + judge={'strategy': JudgeStrategy.AUTO, 'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), - } + }} ) run_task(task_cfg=task_cfg) diff --git a/tests/cli/test_collection.py b/tests/cli/test_collection.py index 46a858a0d..9a2ed66ea 100644 --- a/tests/cli/test_collection.py +++ b/tests/cli/test_collection.py @@ -28,8 +28,9 @@ def setUp(self): 'seed': 42, 'parallel_tool_calls': True }, - 'judge_strategy': JudgeStrategy.AUTO, - 'judge_model_args': { + 'judge': { + 'strategy': JudgeStrategy.AUTO, + 'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -37,6 +38,7 @@ def setUp(self): 'temperature': 0.0, 'max_tokens': 4096, } + }, }, 'debug': True, } diff --git a/tests/cli/test_custom.py b/tests/cli/test_custom.py index ffc22d651..68d8d3df5 100644 --- a/tests/cli/test_custom.py +++ b/tests/cli/test_custom.py @@ -33,7 +33,7 @@ def setUp(self): 'max_tokens': 4096, }, 'ignore_errors': False, - 'judge_model_args': { + 'judge': {'strategy': JudgeStrategy.LLM, 'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -41,8 +41,7 @@ def setUp(self): 'temperature': 0.0, 'max_tokens': 4096 }, - }, - 'judge_strategy': JudgeStrategy.LLM, + }}, } @@ -118,7 +117,7 @@ def test_run_general_no_answer_with_judge(self): }, model='qwen2.5-7b-instruct', limit=10, - judge_model_args={ + judge={'strategy': JudgeStrategy.LLM, 'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -126,10 +125,8 @@ def test_run_general_no_answer_with_judge(self): 'temperature': 0.0, 'max_tokens': 4096 }, - 'score_type': 'numeric', - }, + }, 'contract': {'score_type': 'numeric'}}, eval_batch_size=5, - judge_strategy=JudgeStrategy.LLM, ) def test_run_general_with_answer(self): @@ -142,7 +139,7 @@ def test_run_general_with_answer(self): }, model='qwen-plus', limit=10, - judge_model_args={ + judge={'strategy': JudgeStrategy.LLM, 'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -150,10 +147,8 @@ def test_run_general_with_answer(self): 'temperature': 0.0, 'max_tokens': 4096 }, - 'score_type': 'pattern', - }, + }, 'contract': {'score_type': 'pattern'}}, eval_batch_size=1, - judge_strategy=JudgeStrategy.LLM, ) @@ -178,7 +173,7 @@ def test_run_general_arena(self): }, model_id='Arena', limit=10, - judge_model_args={ + judge={'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -186,7 +181,7 @@ def test_run_general_arena(self): 'temperature': 0.0, 'max_tokens': 8000 }, - }, + }}, eval_batch_size=5, ) diff --git a/tests/cli/test_reasoning.py b/tests/cli/test_reasoning.py index acd62be48..d044e954d 100644 --- a/tests/cli/test_reasoning.py +++ b/tests/cli/test_reasoning.py @@ -33,8 +33,9 @@ def setUp(self): 'parallel_tool_calls': True, 'extra_body':{'chat_template_kwargs': {'enable_thinking': False}} # 关闭思考模式 }, - 'judge_strategy': JudgeStrategy.AUTO, - 'judge_model_args': { + 'judge': { + 'strategy': JudgeStrategy.AUTO, + 'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -42,6 +43,7 @@ def setUp(self): 'temperature': 0.0, 'max_tokens': 4096, } + }, }, 'debug': True, } diff --git a/tests/common.py b/tests/common.py index bb1632fc5..e16a753e3 100644 --- a/tests/common.py +++ b/tests/common.py @@ -32,8 +32,9 @@ def setUp(self): 'seed': 42, 'parallel_tool_calls': True }, - 'judge_strategy': JudgeStrategy.AUTO, - 'judge_model_args': { + 'judge': { + 'strategy': JudgeStrategy.AUTO, + 'models': { 'model_id': 'qwen-plus', 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'api_key': env.get('DASHSCOPE_API_KEY'), @@ -41,6 +42,7 @@ def setUp(self): 'temperature': 0.0, 'max_tokens': 4096, } + }, }, 'debug': True, } diff --git a/tests/metrics/aggregators/test_missing_metric_keys.py b/tests/metrics/aggregators/test_missing_metric_keys.py new file mode 100644 index 000000000..2c597bfbc --- /dev/null +++ b/tests/metrics/aggregators/test_missing_metric_keys.py @@ -0,0 +1,166 @@ +"""A sample may carry no value for a metric (an unparseable LLM judge is excluded rather +than scored 0), so aggregators must never index every sample with ``scores[0]``' keys.""" +import pytest +from typing import Any, Dict, List, Optional + +from evalscope.api.metric import SampleScore, Score +from evalscope.metrics.aggregators import Mean, MeanPassAtK, MeanPassHatK, MeanVoteAtK + +K_AGGREGATORS = [MeanPassAtK, MeanVoteAtK, MeanPassHatK] + + +def make_sample_score( + value: Dict[str, float], + group_id: Optional[str] = None, + sample_id: Optional[str] = None, + prediction: Optional[str] = None, + generation_index: Optional[int] = None, +) -> SampleScore: + return SampleScore( + score=Score(value=value, extracted_prediction=prediction), + sample_id=sample_id or group_id, + group_id=group_id, + generation_index=generation_index, + ) + + +def find_agg(agg_scores: List[Any], aggregation: str, k: int) -> Optional[Any]: + for agg in agg_scores: + if agg.aggregation == aggregation and agg.dimensions.get('k') == k: + return agg + return None + + +@pytest.mark.parametrize('aggregator_cls', K_AGGREGATORS) +def test_first_sample_missing_metric_still_reports_it(aggregator_cls): + scores = [ + make_sample_score({}, group_id='g0', prediction='p'), + make_sample_score({'judge_score': 1.0}, group_id='g1', prediction='p'), + ] + + agg_scores = aggregator_cls()(scores) + + assert any(agg.metric_name == 'judge_score' for agg in agg_scores) + + +@pytest.mark.parametrize('aggregator_cls', K_AGGREGATORS) +def test_later_sample_missing_metric_does_not_raise(aggregator_cls): + scores = [ + make_sample_score({'judge_score': 1.0}, group_id='g0', prediction='p'), + make_sample_score({}, group_id='g1', prediction='p'), + ] + + agg_scores = aggregator_cls()(scores) + + assert any(agg.metric_name == 'judge_score' for agg in agg_scores) + + +@pytest.mark.parametrize('aggregator_cls', K_AGGREGATORS) +def test_group_with_no_usable_attempt_is_dropped_not_zeroed(aggregator_cls): + """An unusable judge result must not be averaged in as a 0.""" + scores = [ + make_sample_score({'judge_score': 1.0}, group_id='g0', sample_id='g0-0', prediction='p'), + make_sample_score({'judge_score': 1.0}, group_id='g0', sample_id='g0-1', prediction='p'), + make_sample_score({}, group_id='g1', sample_id='g1-0', prediction='p'), + make_sample_score({}, group_id='g1', sample_id='g1-1', prediction='p'), + ] + + agg_scores = aggregator_cls()(scores) + at_1 = [agg for agg in agg_scores if agg.dimensions.get('k') == 1 and agg.metric_name == 'judge_score'] + + assert at_1, 'k=1 aggregate is expected' + assert at_1[0].score == pytest.approx(1.0) + assert at_1[0].num == 1 + + +def test_pass_at_k_keeps_humaneval_estimator(): + """Locks the combinatorial estimator: a prefix reading would give 0.0 for pass@2 here.""" + values = [0.0, 0.0, 1.0, 1.0] + scores = [ + make_sample_score({'pass': v}, group_id='g0', sample_id=f'g0-{i}', prediction=str(i)) + for i, v in enumerate(values) + ] + + agg_scores = MeanPassAtK()(scores) + + assert find_agg(agg_scores, 'pass_at_k', 1).score == pytest.approx(0.5) + assert find_agg(agg_scores, 'pass_at_k', 2).score == pytest.approx(5 / 6) + + +def test_pass_hat_k_keeps_estimator(): + """4 attempts, 2 correct: pass^2 = C(2,2)/C(4,2) = 1/6.""" + values = [1.0, 1.0, 0.0, 0.0] + scores = [ + make_sample_score({'pass': v}, group_id='g0', sample_id=f'g0-{i}', prediction=str(i)) + for i, v in enumerate(values) + ] + + agg_scores = MeanPassHatK()(scores) + + assert find_agg(agg_scores, 'pass_hat_k', 2).score == pytest.approx(1 / 6) + + +def test_unusable_attempt_shrinks_n_instead_of_scoring_zero(): + """One unparseable judge result out of 4 leaves a 3-attempt estimator, not a 0.""" + scores = [ + make_sample_score({'judge_score': 1.0}, group_id='g0', sample_id='g0-0', prediction='a'), + make_sample_score({'judge_score': 0.0}, group_id='g0', sample_id='g0-1', prediction='b'), + make_sample_score({'judge_score': 0.0}, group_id='g0', sample_id='g0-2', prediction='c'), + make_sample_score({}, group_id='g0', sample_id='g0-3', prediction='d'), + ] + + agg_scores = MeanPassAtK()(scores) + + # n=3, c=1 -> pass@1 = 1/3; a 0-scored 4th attempt would have given 1/4. + assert find_agg(agg_scores, 'pass_at_k', 1).score == pytest.approx(1 / 3) + assert find_agg(agg_scores, 'pass_at_k', 4) is None + + +def test_mean_only_counts_present_values(): + scores = [ + make_sample_score({'judge_score': 1.0}, group_id='g0'), + make_sample_score({}, group_id='g1'), + ] + + agg_scores = Mean()(scores) + + assert len(agg_scores) == 1 + assert agg_scores[0].score == pytest.approx(1.0) + assert agg_scores[0].num == 1 + + +@pytest.mark.parametrize('aggregator_cls', K_AGGREGATORS) +def test_missing_first_generation_never_promotes_a_later_generation_to_k_one(aggregator_cls): + scores = [ + make_sample_score({'judge_score': 1.0}, group_id='g0', sample_id='g0-1', prediction='right', generation_index=1), + make_sample_score({'judge_score': 0.0}, group_id='g1', sample_id='g1-0', prediction='wrong', generation_index=0), + ] + + at_one = find_agg(aggregator_cls()(scores), + 'vote_at_k' if aggregator_cls is MeanVoteAtK else + 'pass_at_k' if aggregator_cls is MeanPassAtK else 'pass_hat_k', 1) + + assert at_one.num == 1 + assert at_one.metadata['excluded'] == 1 + + +@pytest.mark.parametrize('aggregator_cls', K_AGGREGATORS) +def test_missing_middle_generation_excludes_only_affected_higher_k_groups(aggregator_cls): + scores = [ + make_sample_score({'judge_score': 1.0}, group_id='g0', sample_id='g0-0', generation_index=0), + make_sample_score({}, group_id='g0', sample_id='g0-1', generation_index=1), + make_sample_score({'judge_score': 1.0}, group_id='g0', sample_id='g0-2', generation_index=2), + make_sample_score({'judge_score': 0.0}, group_id='g1', sample_id='g1-0', generation_index=0), + make_sample_score({'judge_score': 0.0}, group_id='g1', sample_id='g1-1', generation_index=1), + make_sample_score({'judge_score': 0.0}, group_id='g1', sample_id='g1-2', generation_index=2), + ] + + aggregation = ( + 'vote_at_k' if aggregator_cls is MeanVoteAtK else + 'pass_at_k' if aggregator_cls is MeanPassAtK else 'pass_hat_k' + ) + at_two = find_agg(aggregator_cls()(scores), aggregation, 2) + + assert at_two is not None + assert at_two.num == 1 + assert at_two.metadata == {'eligible': 1, 'total': 2, 'coverage': 0.5, 'excluded': 1} diff --git a/tests/metrics/judge/test_score_extractors.py b/tests/metrics/judge/test_score_extractors.py deleted file mode 100644 index b9046a8aa..000000000 --- a/tests/metrics/judge/test_score_extractors.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Unit tests for LLM-judge score extraction strategies. - -These lock in the documented fail-close contract: a failed judge request -(``[ERROR] ...`` responses) and any unmatched/unmapped extraction must score -0, never full credit. Covers ``PatternScoreExtractor``, ``NumericScoreExtractor`` -and the ``[ERROR]`` guard in ``LLMJudge.get_score``. -""" - -from evalscope.metrics.judge.llm_judge import JUDGE_ERROR_PREFIX, LLMJudge -from evalscope.metrics.judge.score_extractors import NumericScoreExtractor, PatternScoreExtractor - - -class TestPatternScoreExtractor: - - def test_mapped_answer_returns_mapping_value(self) -> None: - extractor = PatternScoreExtractor(pattern=r'\b([AB])\b', score_mapping={'A': 1.0, 'B': 0.5}) - assert extractor.extract('The answer is A.') == 1.0 - assert extractor.extract('The answer is B.') == 0.5 - - def test_no_match_returns_zero(self) -> None: - extractor = PatternScoreExtractor(pattern=r'\b([AB])\b') - assert extractor.extract('The answer is C.') == 0.0 - - def test_matched_but_unmapped_returns_zero(self) -> None: - # 'B' matches the pattern but is not in the mapping: fail closed at 0.0 - extractor = PatternScoreExtractor(pattern=r'\b([AB])\b', score_mapping={'A': 1.0}) - assert extractor.extract('The answer is B.') == 0.0 - - def test_error_response_returns_zero_for_letter_pattern(self) -> None: - # An [ERROR] response carries no letter token; extraction fails closed. - extractor = PatternScoreExtractor(pattern=r'\b([AB])\b') - assert extractor.extract(f'{JUDGE_ERROR_PREFIX} Qwen3-235B@http://127.0.0.1:8000/v1/ HTTP 500') == 0.0 - - -class TestNumericScoreExtractor: - - def test_in_range_value_passes_through(self) -> None: - extractor = NumericScoreExtractor(pattern=r'\[\[(.*?)\]\]') - assert extractor.extract('[[0.5]]') == 0.5 - - def test_out_of_range_high_clamps_to_max(self) -> None: - # A 0-100-scale judge reporting [[90]] must not become full credit silently. - extractor = NumericScoreExtractor(pattern=r'\[\[(.*?)\]\]') - assert extractor.extract('[[90]]') == 1.0 - - def test_out_of_range_low_clamps_to_min(self) -> None: - extractor = NumericScoreExtractor(pattern=r'\[\[(.*?)\]\]') - assert extractor.extract('[[-1]]') == 0.0 - - def test_nan_fails_closed_to_min(self) -> None: - # float() accepts 'nan', and NaN compares false against both bounds, so without an - # explicit check it escapes clamping. One NaN sample would poison every aggregate - # built on this metric, since nothing downstream filters it. - extractor = NumericScoreExtractor(pattern=r'\[\[(.*?)\]\]') - assert extractor.extract('[[nan]]') == 0.0 - assert extractor.extract('[[NaN]]') == 0.0 - - def test_infinity_clamps_to_bounds(self) -> None: - extractor = NumericScoreExtractor(pattern=r'\[\[(.*?)\]\]') - assert extractor.extract('[[inf]]') == 1.0 - assert extractor.extract('[[-inf]]') == 0.0 - - def test_no_match_returns_zero(self) -> None: - extractor = NumericScoreExtractor(pattern=r'\[\[(.*?)\]\]') - assert extractor.extract('no brackets here') == 0.0 - - -class TestLLMJudgeErrorGuard: - - def make_judge(self, score_type: str, **kwargs) -> LLMJudge: - return LLMJudge(score_type=score_type, **kwargs) - - def test_none_response_returns_zero(self) -> None: - judge = self.make_judge('pattern', score_pattern=r'\b([AB])\b') - assert judge.get_score(None) == 0.0 - - def test_error_response_returns_zero_even_with_loose_numeric_pattern(self) -> None: - # Without the [ERROR] guard, a loose numeric pattern would parse the - # embedded digits (500, 8000, 127) from a failed judge response and clamp - # them to full credit — the opposite of the fail-close contract. - judge = self.make_judge('numeric', score_pattern=r'(\d+)') - response = f'{JUDGE_ERROR_PREFIX} Qwen3-235B@http://127.0.0.1:8000/v1/ HTTP 500' - assert judge.get_score(response) == 0.0 - - def test_normal_response_still_extracts(self) -> None: - judge = self.make_judge('numeric', score_pattern=r'\[\[(.*?)\]\]') - assert judge.get_score('[[0.75]]') == 0.75 - - def test_judge_returns_error_prefix_when_generate_raises(self) -> None: - # Locks the producer contract: on a failed request, ``judge`` returns a - # ``JUDGE_ERROR_PREFIX``-prefixed string that ``get_score`` then fails closed on. - judge = self.make_judge('pattern', score_pattern=r'\b([AB])\b') - - class _FailingModel: - - def generate(self, messages): - raise RuntimeError('connection refused') - - judge.model = _FailingModel() - result = judge.judge(prompt='question') - assert result.startswith(JUDGE_ERROR_PREFIX) diff --git a/tests/perf/test_arguments_secrets.py b/tests/perf/test_arguments_secrets.py index 45e6ffe43..b1dd7f7bf 100644 --- a/tests/perf/test_arguments_secrets.py +++ b/tests/perf/test_arguments_secrets.py @@ -2,7 +2,6 @@ from evalscope.api.model import model as model_module from evalscope.api.model.generate_config import GenerateConfig from evalscope.config import TaskConfig -from evalscope.constants import JudgeStrategy from evalscope.models.utils.openai import openai_completion_params from evalscope.models.utils.openai_responses import openai_response_params from evalscope.perf.arguments import Arguments @@ -50,12 +49,11 @@ def test_task_config_string_omits_api_keys(): api_url='http://localhost:8080/v1/chat/completions', api_key='secret-token', datasets=['gsm8k'], - judge_model_args={ + judge={'models': { + 'model_id': 'judge-model', 'api_key': 'judge-secret-token', - 'nested': { - 'api_key': 'nested-judge-secret-token' - }, - }, + 'model_args': {'nested': {'api_key': 'nested-judge-secret-token'}}, + }}, ) task_config_text = str(task_config) @@ -65,8 +63,9 @@ def test_task_config_string_omits_api_keys(): assert 'judge-secret-token' not in task_config_text assert 'nested-judge-secret-token' not in task_config_text assert task_config_dict['api_key'] == '**********' - assert task_config_dict['judge_model_args']['api_key'] == '**********' - assert task_config_dict['judge_model_args']['nested']['api_key'] == '**********' + judge_model = task_config_dict['judge']['models'][0] + assert judge_model['api_key'] == '**********' + assert judge_model['model_args']['nested']['api_key'] == '**********' def test_task_config_string_omits_extra_auth_headers(): @@ -106,7 +105,7 @@ def fake_get_model(**kwargs): assert captured['api_key'] == 'secret-token' -def test_llm_judge_mixin_uses_raw_nested_judge_model_args(monkeypatch): +def test_llm_judge_mixin_uses_raw_nested_judge_model_config(monkeypatch): captured = {} class FakeLLMJudge: @@ -119,19 +118,18 @@ def __init__(self, **kwargs): model='test-model', api_url='http://localhost:8080/v1/chat/completions', datasets=['gsm8k'], - judge_strategy=JudgeStrategy.LLM, - judge_model_args={ + judge={'strategy': 'llm', 'models': { + 'model_id': 'judge-model', 'api_key': 'judge-secret-token', - 'nested': { - 'api_key': 'nested-judge-secret-token' - }, - }, + 'model_args': {'nested': {'api_key': 'nested-judge-secret-token'}}, + }}, ) - llm_judge_mixin.LLMJudgeMixin(benchmark_meta=object(), task_config=task_config).init_llm_judge() + judges = llm_judge_mixin.LLMJudgeMixin(benchmark_meta=object(), task_config=task_config).init_llm_judges() + assert judges[0].judge_id == 'judge-model' assert captured['api_key'] == 'judge-secret-token' - assert captured['nested']['api_key'] == 'nested-judge-secret-token' + assert captured['model_args']['nested']['api_key'] == 'nested-judge-secret-token' def test_openai_params_use_raw_extra_auth_headers(): diff --git a/tests/report/semantics/golden_samples.json b/tests/report/semantics/golden_samples.json index 11074a5da..1a9532dbf 100644 --- a/tests/report/semantics/golden_samples.json +++ b/tests/report/semantics/golden_samples.json @@ -3,6 +3,7 @@ "accuracy": { "semantic_id": "quality.accuracy.ratio", "metric_name": "Accuracy", + "display_name": null, "kind": "quality", "direction": "higher_is_better", "raw_unit": null, @@ -18,6 +19,7 @@ "accuracy_precision_0": { "semantic_id": "quality.accuracy.ratio", "metric_name": "Accuracy", + "display_name": null, "kind": "quality", "direction": "higher_is_better", "raw_unit": null, @@ -33,6 +35,7 @@ "f1": { "semantic_id": "quality.f1.ratio", "metric_name": "F1", + "display_name": null, "kind": "quality", "direction": "higher_is_better", "raw_unit": null, @@ -48,6 +51,7 @@ "points_100": { "semantic_id": "quality.score.points_100", "metric_name": "Score", + "display_name": null, "kind": "quality", "direction": "higher_is_better", "raw_unit": null, @@ -63,6 +67,7 @@ "wer": { "semantic_id": "quality.wer.ratio", "metric_name": "WER", + "display_name": null, "kind": "quality", "direction": "lower_is_better", "raw_unit": null, @@ -78,6 +83,23 @@ "parse_status": { "semantic_id": "diagnostic.parse_status.ratio", "metric_name": "Parse Status", + "display_name": null, + "kind": "diagnostic", + "direction": "none", + "raw_unit": null, + "value_range": { + "min": 0.0, + "max": 1.0 + }, + "display_kind": "percent", + "display_multiplier": 100.0, + "display_unit": "%", + "display_precision": 1 + }, + "incorrect_rate": { + "semantic_id": "diagnostic.parse_status.ratio", + "metric_name": "Parse Status", + "display_name": "Incorrect rate", "kind": "diagnostic", "direction": "none", "raw_unit": null, @@ -93,6 +115,7 @@ "seconds": { "semantic_id": "perf.latency.seconds", "metric_name": "Latency", + "display_name": null, "kind": "quality", "direction": "lower_is_better", "raw_unit": "s", @@ -105,6 +128,7 @@ "milliseconds": { "semantic_id": "perf.latency.milliseconds", "metric_name": "Latency", + "display_name": null, "kind": "quality", "direction": "lower_is_better", "raw_unit": "ms", @@ -117,6 +141,7 @@ "ttft_ms": { "semantic_id": "perf.latency.seconds", "metric_name": "TTFT", + "display_name": null, "kind": "quality", "direction": "lower_is_better", "raw_unit": "s", @@ -129,6 +154,7 @@ "token_throughput": { "semantic_id": "perf.throughput.tokens_per_second", "metric_name": "Token Throughput", + "display_name": null, "kind": "quality", "direction": "higher_is_better", "raw_unit": "tok/s", @@ -141,6 +167,7 @@ "judge_score": { "semantic_id": "quality.judge_score.unbounded", "metric_name": "Judge Score", + "display_name": null, "kind": "quality", "direction": "higher_is_better", "raw_unit": null, @@ -153,6 +180,7 @@ "count": { "semantic_id": "diagnostic.count.items", "metric_name": "Count", + "display_name": null, "kind": "diagnostic", "direction": "none", "raw_unit": null, @@ -165,6 +193,7 @@ "diagnostic_precision_2": { "semantic_id": "diagnostic.unspecified", "metric_name": "Value", + "display_name": null, "kind": "diagnostic", "direction": "none", "raw_unit": null, @@ -177,6 +206,7 @@ "diagnostic_precision_0": { "semantic_id": "diagnostic.unspecified", "metric_name": "Value", + "display_name": null, "kind": "diagnostic", "direction": "none", "raw_unit": null, @@ -189,6 +219,7 @@ "percent_pp": { "semantic_id": "quality.score.ratio", "metric_name": "Change", + "display_name": null, "kind": "quality", "direction": "higher_is_better", "raw_unit": null, @@ -204,6 +235,7 @@ "number_percent": { "semantic_id": "diagnostic.unspecified", "metric_name": "Cache Hit Rate", + "display_name": null, "kind": "diagnostic", "direction": "none", "raw_unit": "%", @@ -216,6 +248,7 @@ "rouge": { "semantic_id": "quality.rouge.ratio", "metric_name": "ROUGE", + "display_name": null, "kind": "quality", "direction": "higher_is_better", "raw_unit": null, @@ -231,6 +264,7 @@ "similarity": { "semantic_id": "quality.similarity.ratio", "metric_name": "Similarity", + "display_name": null, "kind": "quality", "direction": "higher_is_better", "raw_unit": null, @@ -246,6 +280,7 @@ "legacy_diagnostic": { "semantic_id": "diagnostic.unspecified", "metric_name": "Weird.Metric", + "display_name": null, "kind": "diagnostic", "direction": "none", "raw_unit": null, @@ -258,6 +293,7 @@ "total_tokens": { "semantic_id": "diagnostic.unspecified", "metric_name": "total_tokens", + "display_name": null, "kind": "diagnostic", "direction": "none", "raw_unit": null, @@ -541,6 +577,19 @@ "expected_raw": "12.5", "expected_label": "Weird.Metric" }, + { + "id": "label-diagnostic-display-name", + "semantics_ref": "incorrect_rate", + "identity": { + "name": "is_incorrect", + "aggregation": "mean", + "dimensions": {} + }, + "value": 0.25, + "expected_primary": "25%", + "expected_raw": "0.25", + "expected_label": "Incorrect rate" + }, { "id": "label-diagnostic-without-legacy-name", "semantics_ref": "total_tokens", diff --git a/tests/report/semantics/test_formatting.py b/tests/report/semantics/test_formatting.py index c155431c3..100d4b6ff 100644 --- a/tests/report/semantics/test_formatting.py +++ b/tests/report/semantics/test_formatting.py @@ -218,6 +218,13 @@ def test_diagnostic_and_missing_semantics_keep_final_name(self) -> None: assert format_metric_label('judge_failed', diagnostic) == 'judge_failed' assert format_metric_label('unknown_metric', None) == 'unknown_metric' + def test_declared_diagnostic_display_name_overrides_identity(self) -> None: + diagnostic = make_percent_semantics(kind=MetricKind.DIAGNOSTIC, direction=MetricDirection.NONE).model_copy( + update={'display_name': 'Incorrect rate'} + ) + + assert format_metric_label('is_incorrect:mean', diagnostic, 'mean_is_incorrect') == 'Incorrect rate' + def test_duplicate_display_names_are_disambiguated(self) -> None: labels = format_metric_labels([ ('mean_acc', make_percent_semantics()), diff --git a/tests/report/semantics/test_report_model.py b/tests/report/semantics/test_report_model.py index 5f19277a2..9f4bb8f98 100644 --- a/tests/report/semantics/test_report_model.py +++ b/tests/report/semantics/test_report_model.py @@ -3,8 +3,10 @@ from pydantic import ValidationError from typing import Dict, List, Optional -from evalscope.api.metric import AggScore +from evalscope.api.judge import summarize_judge_runs +from evalscope.api.metric import AggScore, JudgeSummary, SampleScore, Score from evalscope.api.metric.semantics import MetricIdentity, MetricKind, MetricSelector +from evalscope.constants import ScoreStatus from evalscope.metrics.semantics.catalog import LEGACY_METRIC_MIGRATIONS from evalscope.metrics.semantics.entry import MetricEntry from evalscope.metrics.semantics.migration import migrate_legacy_report_identity @@ -97,7 +99,8 @@ def test_report_score_compatibility_prefers_primary_then_first_metric() -> None: 'model', ) assert collection.score == 0.6 - assert Report().score == 0.0 + # A report with no metric produced no score; it did not score zero. + assert Report().score is None assert 'score' not in report.to_dict() @@ -126,6 +129,61 @@ def test_v2_serialization_contains_no_v1_metric_fields() -> None: assert set(('identity', 'semantics')).issubset(metric) +def test_report_persists_first_class_judge_summary() -> None: + report = _report() + report.judge_summary = JudgeSummary(status=ScoreStatus.DEGRADED, scored=8, total=10, coverage=0.8) + + assert report.to_dict()['judge_summary']['coverage'] == 0.8 + + +def test_run_judge_summary_keeps_unavailable_samples_out_of_scores() -> None: + usable = SampleScore(score=Score(judge_summary=JudgeSummary( + status=ScoreStatus.SUCCESS, + scored=1, + total=1, + coverage=1.0, + judge_models=['primary'], + valid_observations=1, + total_observations=1, + ))) + unavailable = SampleScore(score=Score(judge_summary=JudgeSummary( + status=ScoreStatus.EXCLUDED, + scored=0, + total=1, + coverage=0.0, + judge_models=['primary'], + total_observations=1, + failures={'parse_error': 1}, + ))) + + summary = summarize_judge_runs([[usable, unavailable]]) + + assert summary.status is ScoreStatus.DEGRADED + assert (summary.scored, summary.total, summary.coverage) == (1, 2, 0.5) + assert summary.failures == {'parse_error': 1} + + +def test_run_judge_summary_preserves_degradation_and_rolls_up_disagreement() -> None: + degraded = SampleScore(score=Score(judge_summary=JudgeSummary( + status=ScoreStatus.DEGRADED, + scored=1, + total=1, + coverage=1.0, + disagreement={ + 'numeric': {'all_observations': {'acc': {'std': 0.2, 'range': 0.5}}}, + 'categorical': {'pair': {'agreement_ratio': 0.5, 'vote_entropy': 1.0}}, + 'position_consistency': 0.5, + 'swap_flip_count': 1, + }, + ))) + + summary = summarize_judge_runs([[degraded]]) + + assert summary.status is ScoreStatus.DEGRADED + assert summary.disagreement['numeric']['acc'] == {'mean_std': 0.2, 'max_range': 0.5, 'samples': 1} + assert summary.disagreement['position_consistency']['swap_flip_count'] == 1 + + def test_v2_round_trip_uses_persisted_semantics_without_resolution() -> None: original = _report() restored = Report.from_dict(original.to_dict()) diff --git a/tests/report/semantics/test_resolver.py b/tests/report/semantics/test_resolver.py index 2bf7d7791..a6391d40a 100644 --- a/tests/report/semantics/test_resolver.py +++ b/tests/report/semantics/test_resolver.py @@ -21,6 +21,18 @@ def test_unknown_metric_degrades_to_diagnostic() -> None: assert resolved.semantics.kind is MetricKind.DIAGNOSTIC +@pytest.mark.parametrize( + ('name', 'expected_display_name'), + [('is_incorrect', 'Incorrect rate'), ('is_not_attempted', 'Not attempted rate')], +) +def test_three_way_grading_diagnostics_have_explicit_display_names(name: str, expected_display_name: str) -> None: + identity = MetricIdentity(name=name, aggregation='mean') + resolved = get_semantics_resolver().resolve('chinese_simpleqa', identity) + + assert resolved.semantics.kind is MetricKind.DIAGNOSTIC + assert resolved.semantics.display_name == expected_display_name + + def test_pass_at_k_dimensions_share_one_aggregation_semantics() -> None: resolver = get_semantics_resolver() for k in (1, 7, 137): diff --git a/tests/report/semantics/test_semantics_model.py b/tests/report/semantics/test_semantics_model.py index 14abbf78c..0fcce8b18 100644 --- a/tests/report/semantics/test_semantics_model.py +++ b/tests/report/semantics/test_semantics_model.py @@ -34,6 +34,7 @@ def test_quality_and_diagnostic_semantics_are_valid() -> None: [ {'kind': MetricKind.QUALITY, 'direction': MetricDirection.NONE}, {'kind': MetricKind.DIAGNOSTIC, 'direction': MetricDirection.HIGHER_IS_BETTER}, + {'display_name': 'Incorrect rate'}, {'value_range': None}, {'display_multiplier': None}, {'display_multiplier': 0.0},