Skip to content

FIX: Integration test fixes #1897

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ValbuenaVC merged 9 commits into from
Jun 3, 2026
Merged

FIX: Integration test fixes #1897

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
6655051
FIX: Added non-None labels to MessagePiece constructor in mocks.py
Jun 2, 2026
b269dc7
FIX: populate AttackTechniqueRegistry before LoadDefaultDatasets in i…
Jun 2, 2026
3607ed3
Merge branch 'main' into integration_test_fixes
ValbuenaVC Jun 2, 2026
c78c3cd
Merge branch 'integration_test_fixes' of https://github.com/ValbuenaV…
Jun 2, 2026
d8a363a
FIX: Correct HarmBench size assertion fron {'large'} to {'medium'} (l…
Jun 2, 2026
3b5cd8c
Merge branch 'main' into integration_test_fixes
ValbuenaVC Jun 2, 2026
58ea965
FIX: Added ScenarioTechniqueInitializer to scenario notebooks to fix …
Jun 3, 2026
66597ef
Merge branch 'main' into integration_test_fixes
ValbuenaVC Jun 3, 2026
484b408
FIX: precommit hooks.
Jun 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
348 changes: 344 additions & 4 deletions doc/code/scenarios/0_scenarios.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions doc/code/scenarios/0_scenarios.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,10 @@
)
from pyrit.score.true_false.true_false_scorer import TrueFalseScorer
from pyrit.setup import initialize_pyrit_async
from pyrit.setup.initializers.components import ScenarioTechniqueInitializer

await initialize_pyrit_async(memory_db_type="InMemory") # type: ignore [top-level-await]
await ScenarioTechniqueInitializer().initialize_async() # type: ignore [top-level-await]


class MyStrategy(ScenarioStrategy):
Expand Down
129 changes: 71 additions & 58 deletions doc/code/scenarios/1_common_scenario_parameters.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,21 @@
"name": "stdout",
"output_type": "stream",
"text": [
"No new upgrade operations detected.\n"
"[pyrit:alembic] No new upgrade operations detected.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Skipping scorer main: required target not found in TargetRegistry\n"
"Skipping target 'platform_openai_chat': PLATFORM_OPENAI_CHAT_GPT4O_MODEL is not set. All declared env vars (endpoint, key, model) must be present for this target to register.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Skipping target 'azure_foundry_phi4': AZURE_FOUNDRY_PHI4_MODEL is not set. All declared env vars (endpoint, key, model) must be present for this target to register.\n"
]
},
{
Expand All @@ -68,7 +75,9 @@
"from pyrit.registry import TargetRegistry\n",
"from pyrit.scenario.scenarios.foundry import FoundryStrategy, RedTeamAgent\n",
"from pyrit.setup import initialize_from_config_async\n",
"from pyrit.setup.initializers.components import ScenarioTechniqueInitializer\n",
"\n",
"await ScenarioTechniqueInitializer().initialize_async() # type: ignore [top-level-await]\n",
"await initialize_from_config_async(config_path=Path(\"../../scanner/pyrit_conf.yaml\")) # type: ignore\n",
"\n",
"objective_target = TargetRegistry.get_registry_singleton().get_instance_by_name(\"openai_chat\")"
Expand Down Expand Up @@ -242,7 +251,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "8734281af4a940bd9995f232ce52c16c",
"model_id": "e386860005e440a98009fc5cd1b55e86",
"version_major": 2,
"version_minor": 0
},
Expand Down Expand Up @@ -283,26 +292,29 @@
"\n",
"\u001b[1m 📊 Scorer Information\u001b[0m\n",
"\u001b[37m ▸ Scorer Identifier\u001b[0m\n",
"\u001b[36m • Scorer Type: SelfAskTrueFalseScorer\u001b[0m\n",
"\u001b[36m • Scorer Type: TrueFalseInverterScorer\u001b[0m\n",
"\u001b[36m • scorer_type: true_false\u001b[0m\n",
"\u001b[36m • score_aggregator: OR_\u001b[0m\n",
"\u001b[36m • model_name: gpt-40\u001b[0m\n",
"\u001b[36m • temperature: 0.9\u001b[0m\n",
"\u001b[36m └─ Composite of 1 scorer(s):\u001b[0m\n",
"\u001b[36m • Scorer Type: SelfAskRefusalScorer\u001b[0m\n",
"\u001b[36m • scorer_type: true_false\u001b[0m\n",
"\u001b[36m • score_aggregator: OR_\u001b[0m\n",
"\u001b[36m • model_name: gpt-4o-japan-nilfilter\u001b[0m\n",
"\n",
"\u001b[37m ▸ Performance Metrics\u001b[0m\n",
"\u001b[36m • Accuracy: 79.24%\u001b[0m\n",
"\u001b[36m • Accuracy Std Error: ±0.0204\u001b[0m\n",
"\u001b[36m • F1 Score: 0.7560\u001b[0m\n",
"\u001b[36m • Precision: 0.8759\u001b[0m\n",
"\u001b[31m • Recall: 0.6649\u001b[0m\n",
"\u001b[36m • Average Score Time: 1.64s\u001b[0m\n",
"\u001b[36m • Accuracy: 89.37%\u001b[0m\n",
"\u001b[36m • Accuracy Std Error: ±0.0155\u001b[0m\n",
"\u001b[36m • F1 Score: 0.8918\u001b[0m\n",
"\u001b[36m • Precision: 0.8782\u001b[0m\n",
"\u001b[32m • Recall: 0.9058\u001b[0m\n",
"\u001b[36m • Average Score Time: 0.59s\u001b[0m\n",
"\n",
"\u001b[1m\u001b[36m▼ Overall Statistics\u001b[0m\n",
"\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n",
"\u001b[1m 📈 Summary\u001b[0m\n",
"\u001b[32m • Total Strategies: 21\u001b[0m\n",
"\u001b[32m • Total Attack Results: 42\u001b[0m\n",
"\u001b[32m • Overall Success Rate: 11%\u001b[0m\n",
"\u001b[32m • Overall Success Rate: 0%\u001b[0m\n",
"\u001b[32m • Unique Objectives: 2\u001b[0m\n",
"\n",
"\u001b[1m\u001b[36m▼ Per-Group Breakdown\u001b[0m\n",
Expand All @@ -316,11 +328,11 @@
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: ascii_art\u001b[0m\n",
"\u001b[1m 🔸 Group: ascii_smuggler\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: ascii_smuggler\u001b[0m\n",
"\u001b[1m 🔸 Group: ascii_art\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
Expand All @@ -330,11 +342,11 @@
"\n",
"\u001b[1m 🔸 Group: base64\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[31m • Success Rate: 100%\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: binary\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[33m • Success Rate: 50%\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: caesar\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
Expand Down Expand Up @@ -364,11 +376,11 @@
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: rot13\u001b[0m\n",
"\u001b[1m 🔸 Group: suffix_append\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: suffix_append\u001b[0m\n",
"\u001b[1m 🔸 Group: rot13\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
Expand All @@ -378,7 +390,7 @@
"\n",
"\u001b[1m 🔸 Group: unicode_confusable\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[33m • Success Rate: 50%\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: unicode_substitution\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
Expand All @@ -390,7 +402,7 @@
"\n",
"\u001b[1m 🔸 Group: jailbreak\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[33m • Success Rate: 50%\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[36m====================================================================================================\u001b[0m\n",
"\n"
Expand All @@ -405,7 +417,7 @@
" dataset_config=dataset_config,\n",
")\n",
"baseline_result = await baseline_scenario.run_async() # type: ignore\n",
"await output_scenario_async(baseline_result)"
"await output_scenario_async(baseline_result) # type: ignore [top-level-await]"
]
},
{
Expand Down Expand Up @@ -458,64 +470,59 @@
"\n",
"\u001b[1m 📊 Scorer Information\u001b[0m\n",
"\u001b[37m ▸ Scorer Identifier\u001b[0m\n",
"\u001b[36m • Scorer Type: SelfAskTrueFalseScorer\u001b[0m\n",
"\u001b[36m • Scorer Type: TrueFalseInverterScorer\u001b[0m\n",
"\u001b[36m • scorer_type: true_false\u001b[0m\n",
"\u001b[36m • score_aggregator: OR_\u001b[0m\n",
"\u001b[36m • model_name: gpt-40\u001b[0m\n",
"\u001b[36m • temperature: 0.9\u001b[0m\n",
"\u001b[36m └─ Composite of 1 scorer(s):\u001b[0m\n",
"\u001b[36m • Scorer Type: SelfAskRefusalScorer\u001b[0m\n",
"\u001b[36m • scorer_type: true_false\u001b[0m\n",
"\u001b[36m • score_aggregator: OR_\u001b[0m\n",
"\u001b[36m • model_name: gpt-4o-japan-nilfilter\u001b[0m\n",
"\n",
"\u001b[37m ▸ Performance Metrics\u001b[0m\n",
"\u001b[36m • Accuracy: 79.24%\u001b[0m\n",
"\u001b[36m • Accuracy Std Error: ±0.0204\u001b[0m\n",
"\u001b[36m • F1 Score: 0.7560\u001b[0m\n",
"\u001b[36m • Precision: 0.8759\u001b[0m\n",
"\u001b[31m • Recall: 0.6649\u001b[0m\n",
"\u001b[36m • Average Score Time: 1.64s\u001b[0m\n",
"\u001b[36m • Accuracy: 89.37%\u001b[0m\n",
"\u001b[36m • Accuracy Std Error: ±0.0155\u001b[0m\n",
"\u001b[36m • F1 Score: 0.8918\u001b[0m\n",
"\u001b[36m • Precision: 0.8782\u001b[0m\n",
"\u001b[32m • Recall: 0.9058\u001b[0m\n",
"\u001b[36m • Average Score Time: 0.59s\u001b[0m\n",
"\n",
"\u001b[1m\u001b[36m▼ Overall Statistics\u001b[0m\n",
"\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n",
"\u001b[1m 📈 Summary\u001b[0m\n",
"\u001b[32m • Total Strategies: 21\u001b[0m\n",
"\u001b[32m • Total Attack Results: 42\u001b[0m\n",
"\u001b[32m • Overall Success Rate: 11%\u001b[0m\n",
"\u001b[32m • Overall Success Rate: 0%\u001b[0m\n",
"\u001b[32m • Unique Objectives: 2\u001b[0m\n",
"\n",
"\u001b[1m\u001b[36m▼ Per-Group Breakdown\u001b[0m\n",
"\u001b[36m────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: base64\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[31m • Success Rate: 100%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: binary\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[33m • Success Rate: 50%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: unicode_confusable\u001b[0m\n",
"\u001b[1m 🔸 Group: baseline\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[33m • Success Rate: 50%\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: jailbreak\u001b[0m\n",
"\u001b[1m 🔸 Group: ansi_attack\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[33m • Success Rate: 50%\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: baseline\u001b[0m\n",
"\u001b[1m 🔸 Group: ascii_smuggler\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: ansi_attack\u001b[0m\n",
"\u001b[1m 🔸 Group: ascii_art\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: ascii_art\u001b[0m\n",
"\u001b[1m 🔸 Group: atbash\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: ascii_smuggler\u001b[0m\n",
"\u001b[1m 🔸 Group: base64\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: atbash\u001b[0m\n",
"\u001b[1m 🔸 Group: binary\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
Expand Down Expand Up @@ -547,18 +554,22 @@
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: rot13\u001b[0m\n",
"\u001b[1m 🔸 Group: suffix_append\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: suffix_append\u001b[0m\n",
"\u001b[1m 🔸 Group: rot13\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: string_join\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: unicode_confusable\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: unicode_substitution\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
Expand All @@ -567,6 +578,10 @@
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[1m 🔸 Group: jailbreak\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\n",
"\u001b[36m====================================================================================================\u001b[0m\n",
"\n"
]
Expand Down Expand Up @@ -617,7 +632,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "103ae439a5554be79c786a8bcc9c1524",
"model_id": "9f07786563cc4128ba51a5e47eb53a6f",
"version_major": 2,
"version_minor": 0
},
Expand Down Expand Up @@ -675,7 +690,7 @@
"\u001b[1m 📈 Summary\u001b[0m\n",
"\u001b[32m • Total Strategies: 2\u001b[0m\n",
"\u001b[32m • Total Attack Results: 4\u001b[0m\n",
"\u001b[36m • Overall Success Rate: 25%\u001b[0m\n",
"\u001b[33m • Overall Success Rate: 50%\u001b[0m\n",
"\u001b[32m • Unique Objectives: 2\u001b[0m\n",
"\n",
"\u001b[1m\u001b[36m▼ Per-Group Breakdown\u001b[0m\n",
Expand All @@ -687,7 +702,7 @@
"\n",
"\u001b[1m 🔸 Group: base64\u001b[0m\n",
"\u001b[33m • Number of Results: 2\u001b[0m\n",
"\u001b[32m • Success Rate: 0%\u001b[0m\n",
"\u001b[33m • Success Rate: 50%\u001b[0m\n",
"\n",
"\u001b[36m====================================================================================================\u001b[0m\n",
"\n"
Expand All @@ -710,15 +725,13 @@
" scenario_strategies=[FoundryStrategy.Base64],\n",
" dataset_config=dataset_config,\n",
")\n",
"\n",
"custom_result = await custom_scenario.run_async() # type: ignore\n",
"await output_scenario_async(custom_result)"
]
}
],
"metadata": {
"jupytext": {
"main_language": "python"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
Expand All @@ -729,7 +742,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.12"
"version": "3.13.13"
}
},
"nbformat": 4,
Expand Down
5 changes: 4 additions & 1 deletion doc/code/scenarios/1_common_scenario_parameters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@
from pyrit.registry import TargetRegistry
from pyrit.scenario.scenarios.foundry import FoundryStrategy, RedTeamAgent
from pyrit.setup import initialize_from_config_async
from pyrit.setup.initializers.components import ScenarioTechniqueInitializer

await ScenarioTechniqueInitializer().initialize_async() # type: ignore [top-level-await]
await initialize_from_config_async(config_path=Path("../../scanner/pyrit_conf.yaml")) # type: ignore

objective_target = TargetRegistry.get_registry_singleton().get_instance_by_name("openai_chat")
Expand Down Expand Up @@ -118,7 +120,7 @@
dataset_config=dataset_config,
)
baseline_result = await baseline_scenario.run_async() # type: ignore
await output_scenario_async(baseline_result)
await output_scenario_async(baseline_result) # type: ignore [top-level-await]

# %% [markdown]
# ### Sorting the Per-Group Breakdown by Success Rate
Expand Down Expand Up @@ -170,5 +172,6 @@
scenario_strategies=[FoundryStrategy.Base64],
dataset_config=dataset_config,
)

custom_result = await custom_scenario.run_async() # type: ignore
await output_scenario_async(custom_result)
Loading
Loading