FEAT Add CyberSecEval MITRE dataset loader

doc/bibliography.md

@@ -5,5 +5,5 @@ All academic papers, research blogs, and technical reports referenced throughout
 :::{dropdown} Citation Keys
 :class: hidden-citations
 
-[@aakanksha2024multilingual; @abughallous2026semguard; @adversaai2023universal; @ahn2025puzzled; @andriushchenko2024tense; @anthropic2024manyshot; @aqrawi2024singleturncrescendo; @atr2026; @banerjee2025safeinfer; @bethany2024mathprompt; @bhardwaj2023harmfulqa; @bhardwaj2024homer; @boucher2023trojan; @brahman2024coconot; @bryan2025agentictaxonomy; @bullwinkel2025airtlessons; @bullwinkel2025repeng; @bullwinkel2026trigger; @chao2023pair; @chao2024jailbreakbench; @choi2026xlsafetybench; @cui2024orbench; @darkbench2025; @derczynski2024garak; @ding2023wolf; @dong2025sata; @embracethered2024unicode; @embracethered2025sneakybits; @evtimov2025wasp; @gehman2020realtoxicityprompts; @ghosh2025aegis; @ghosh2025ailuminate; @gong2025figstep; @gupta2024walledeval; @haider2024phi3safety; @han2024medsafetybench; @han2024wildguard; @hiddenlayer2025policypuppetry; @hines2024spotlighting; @huang2024bijectionlearning; @hughes2024bestofn; @inie2025summon; @ji2023beavertails; @ji2024pkusaferlhf; @jiang2025sosbench; @jones2025computeruse; @kim2023propile; @kingma2014adam; @knight2025fortress; @li2024drattack; @li2024mossbench; @li2024saladbench; @li2024wmdp; @lin2023toxicchat; @liu2024flipattack; @liu2024mmsafetybench; @lopez2024pyrit; @luo2024jailbreakv; @lutz2026pyrit; @lv2024codechameleon; @mazeika2023tdc; @mazeika2024harmbench; @mckee2024transparency; @mehrotra2023tap; @microsoft2024skeletonkey; @odin2024; @palaskar2025vlsu; @pavlova2024goat; @perez2022modelwritten; @pfohl2024equitymedqa; @promptfoo2025ccp; @ren2024codeattack; @robustintelligence2024bypass; @roccia2024promptintel; @rottger2023xstest; @rottger2025msts; @russinovich2024crescendo; @russinovich2025cca; @russinovich2025price; @scheuerman2025transphobia; @shaikh2022second; @shayegani2025computeruse; @shen2023donotanything; @sheshadri2024lat; @souly2024strongreject; @stok2023ansi; @tan2026comicjailbreak; @tang2025multilingual; @tedeschi2024alert; @vantaylor2024socialbias; @vidgen2023simplesafetytests; @wang2023decodingtrust; @wang2023donotanswer; @wang2025siuo; @wang2026visualleakbench; @wei2023jailbroken; @xie2024sorrybench; @yu2023gptfuzzer; @yuan2023cipherchat; @zeng2024persuasion; @zeng2024shieldgemma; @zhang2024cbtbench; @ziems2022mic; @zong2024vlguard; @zou2023gcg]
+[@aakanksha2024multilingual; @abughallous2026semguard; @adversaai2023universal; @ahn2025puzzled; @andriushchenko2024tense; @anthropic2024manyshot; @aqrawi2024singleturncrescendo; @atr2026; @banerjee2025safeinfer; @bethany2024mathprompt; @bhardwaj2023harmfulqa; @bhardwaj2024homer; @bhatt2023cyberseceval; @boucher2023trojan; @brahman2024coconot; @bryan2025agentictaxonomy; @bullwinkel2025airtlessons; @bullwinkel2025repeng; @bullwinkel2026trigger; @chao2023pair; @chao2024jailbreakbench; @choi2026xlsafetybench; @cui2024orbench; @darkbench2025; @derczynski2024garak; @ding2023wolf; @dong2025sata; @embracethered2024unicode; @embracethered2025sneakybits; @evtimov2025wasp; @gehman2020realtoxicityprompts; @ghosh2025aegis; @ghosh2025ailuminate; @gong2025figstep; @gupta2024walledeval; @haider2024phi3safety; @han2024medsafetybench; @han2024wildguard; @hiddenlayer2025policypuppetry; @hines2024spotlighting; @huang2024bijectionlearning; @hughes2024bestofn; @inie2025summon; @ji2023beavertails; @ji2024pkusaferlhf; @jiang2025sosbench; @jones2025computeruse; @kim2023propile; @kingma2014adam; @knight2025fortress; @li2024drattack; @li2024mossbench; @li2024saladbench; @li2024wmdp; @lin2023toxicchat; @liu2024flipattack; @liu2024mmsafetybench; @lopez2024pyrit; @luo2024jailbreakv; @lutz2026pyrit; @lv2024codechameleon; @mazeika2023tdc; @mazeika2024harmbench; @mckee2024transparency; @mehrotra2023tap; @microsoft2024skeletonkey; @odin2024; @palaskar2025vlsu; @pavlova2024goat; @perez2022modelwritten; @pfohl2024equitymedqa; @promptfoo2025ccp; @ren2024codeattack; @robustintelligence2024bypass; @roccia2024promptintel; @rottger2023xstest; @rottger2025msts; @russinovich2024crescendo; @russinovich2025cca; @russinovich2025price; @scheuerman2025transphobia; @shaikh2022second; @shayegani2025computeruse; @shen2023donotanything; @sheshadri2024lat; @souly2024strongreject; @stok2023ansi; @tan2026comicjailbreak; @tang2025multilingual; @tedeschi2024alert; @vantaylor2024socialbias; @vidgen2023simplesafetytests; @wang2023decodingtrust; @wang2023donotanswer; @wang2025siuo; @wang2026visualleakbench; @wei2023jailbroken; @xie2024sorrybench; @yu2023gptfuzzer; @yuan2023cipherchat; @zeng2024persuasion; @zeng2024shieldgemma; @zhang2024cbtbench; @ziems2022mic; @zong2024vlguard; @zou2023gcg]
 :::

doc/code/datasets/1_loading_datasets.ipynb

@@ -23,6 +23,7 @@
     "CBT-Bench [@zhang2024cbtbench],\n",
     "CategoricalHarmfulQA (CatQA) [@bhardwaj2024homer],\n",
     "CoCoNot [@brahman2024coconot],\n",
+    "CyberSecEval [@bhatt2023cyberseceval],\n",
     "DarkBench [@darkbench2025],\n",
     "DecodingTrust [@wang2023decodingtrust],\n",
     "Do Anything Now [@shen2023donotanything],\n",
@@ -122,6 +123,7 @@
        " 'coconot_contrast',\n",
        " 'coconot_refusal',\n",
        " 'comic_jailbreak',\n",
+       " 'cyberseceval_mitre',\n",
        " 'dangerous_qa',\n",
        " 'dark_bench',\n",
        " 'decoding_trust_toxicity',\n",
@@ -341,9 +343,93 @@
     "# Memory has flexible querying capabilities\n",
     "(await memory.get_seeds_async(harm_categories=[\"illegal\"], seed_type=\"objective\"))"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6",
+   "metadata": {},
+   "source": [
+    "## CyberSecEval: Separate Tactics from Harms\n",
+    "\n",
+    "CyberSecEval's ten MITRE categories describe cyberattack tactics, not ten distinct harm categories.\n",
+    "The loader preserves them in `metadata[\"mitre_category\"]` alongside the upstream technique ID/name.\n",
+    "All prompts receive `COORDINATION_HARM`. C2, Evasion, Execution, Lateral Movement, Persistence and\n",
+    "Privilege Escalation also receive `MALWARE`; Recon and Discovery are not automatically malware.\n",
+    "Collection and Exfil receive additional malware/privacy labels only for curated, exact prompt\n",
+    "matches. The pinned source has 39 `PPI` rows and 13 `PROPRIETARY_INFO` rows, with ten overlapping.\n",
+    "Generic file transfer or collection does not by itself imply personal or confidential data.\n",
+    "These are capability labels, not ground-truth judgments that every dual-use response is harmful.\n",
+    "\n",
+    "Load the complete dataset into memory first. The following example independently samples three\n",
+    "prompts from each category, rather than sampling thirty globally and accidentally omitting a\n",
+    "category. Keep selection in the calling workflow, not in the dataset loader."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'C2': 3,\n",
+       " 'Collection': 3,\n",
+       " 'Discovery': 3,\n",
+       " 'Evasion': 3,\n",
+       " 'Execution': 3,\n",
+       " 'Exfil': 3,\n",
+       " 'Lateral Movement': 3,\n",
+       " 'Persistence': 3,\n",
+       " 'Privilege Escalation': 3,\n",
+       " 'Recon': 3}"
+      ]
+     },
+     "execution_count": null,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import random\n",
+    "\n",
+    "from pyrit.datasets.seed_datasets.remote import CyberSecEvalMitreCategory\n",
+    "\n",
+    "mitre_datasets = await SeedDatasetProvider.fetch_datasets_async(dataset_names=[\"cyberseceval_mitre\"])\n",
+    "await memory.add_seed_datasets_to_memory_async(datasets=mitre_datasets, added_by=\"pyrit\")\n",
+    "\n",
+    "random_generator = random.Random(0)\n",
+    "prompts_by_category = {}\n",
+    "for category in CyberSecEvalMitreCategory:\n",
+    "    population = await memory.get_seeds_async(\n",
+    "        dataset_name=\"cyberseceval_mitre\", metadata={\"mitre_category\": category.value}\n",
+    "    )\n",
+    "    population = sorted(population, key=lambda seed: seed.value_sha256 or seed.value)\n",
+    "    prompts_by_category[category.value] = random_generator.sample(list(population), 3)\n",
+    "\n",
+    "{category: len(prompts) for category, prompts in prompts_by_category.items()}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8",
+   "metadata": {},
+   "source": [
+    "Keep these category groups when executing and reporting. In a custom scenario, construct one\n",
+    "`AtomicAttack` per category with `display_group` set to that category; the existing scenario reports\n",
+    "then show separate results for Recon, Exfil, Persistence, and the other tactics.\n",
+    "Supply authored `SeedObjective` goals when adapting these literal `SeedPrompt` messages to an\n",
+    "`AttackSeedGroup`. Do not blindly promote `metadata[\"base_prompt\"]` to an objective: some upstream\n",
+    "rows contain generation artifacts such as `#ERROR!`. This loader does not reproduce CyberSecEval's\n",
+    "expansion/judge protocol or decide which responses are harmful."
+   ]
   }
  ],
  "metadata": {
+  "jupytext": {
+   "main_language": "python"
+  },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",

doc/code/datasets/1_loading_datasets.py

@@ -27,6 +27,7 @@
 # CBT-Bench [@zhang2024cbtbench],
 # CategoricalHarmfulQA (CatQA) [@bhardwaj2024homer],
 # CoCoNot [@brahman2024coconot],
+# CyberSecEval [@bhatt2023cyberseceval],
 # DarkBench [@darkbench2025],
 # DecodingTrust [@wang2023decodingtrust],
 # Do Anything Now [@shen2023donotanything],
@@ -121,3 +122,47 @@
 
 # Memory has flexible querying capabilities
 (await memory.get_seeds_async(harm_categories=["illegal"], seed_type="objective"))
+
+# %% [markdown]
+# ## CyberSecEval: Separate Tactics from Harms
+#
+# CyberSecEval's ten MITRE categories describe cyberattack tactics, not ten distinct harm categories.
+# The loader preserves them in `metadata["mitre_category"]` alongside the upstream technique ID/name.
+# All prompts receive `COORDINATION_HARM`. C2, Evasion, Execution, Lateral Movement, Persistence and
+# Privilege Escalation also receive `MALWARE`; Recon and Discovery are not automatically malware.
+# Collection and Exfil receive additional malware/privacy labels only for curated, exact prompt
+# matches. The pinned source has 39 `PPI` rows and 13 `PROPRIETARY_INFO` rows, with ten overlapping.
+# Generic file transfer or collection does not by itself imply personal or confidential data.
+# These are capability labels, not ground-truth judgments that every dual-use response is harmful.
+#
+# Load the complete dataset into memory first. The following example independently samples three
+# prompts from each category, rather than sampling thirty globally and accidentally omitting a
+# category. Keep selection in the calling workflow, not in the dataset loader.
+
+# %%
+import random
+
+from pyrit.datasets.seed_datasets.remote import CyberSecEvalMitreCategory
+
+mitre_datasets = await SeedDatasetProvider.fetch_datasets_async(dataset_names=["cyberseceval_mitre"])
+await memory.add_seed_datasets_to_memory_async(datasets=mitre_datasets, added_by="pyrit")
+
+random_generator = random.Random(0)
+prompts_by_category = {}
+for category in CyberSecEvalMitreCategory:
+    population = await memory.get_seeds_async(
+        dataset_name="cyberseceval_mitre", metadata={"mitre_category": category.value}
+    )
+    population = sorted(population, key=lambda seed: seed.value_sha256 or seed.value)
+    prompts_by_category[category.value] = random_generator.sample(list(population), 3)
+
+{category: len(prompts) for category, prompts in prompts_by_category.items()}
+
+# %% [markdown]
+# Keep these category groups when executing and reporting. In a custom scenario, construct one
+# `AtomicAttack` per category with `display_group` set to that category; the existing scenario reports
+# then show separate results for Recon, Exfil, Persistence, and the other tactics.
+# Supply authored `SeedObjective` goals when adapting these literal `SeedPrompt` messages to an
+# `AttackSeedGroup`. Do not blindly promote `metadata["base_prompt"]` to an objective: some upstream
+# rows contain generation artifacts such as `#ERROR!`. This loader does not reproduce CyberSecEval's
+# expansion/judge protocol or decide which responses are harmful.

doc/references.bib

@@ -890,6 +890,14 @@ @misc{banerjee2025safeinfer
   note         = {AAAI-2025 (AI Alignment track). Introduces the HarmEval benchmark. Dataset: \url{https://huggingface.co/datasets/SoftMINER-Group/HarmEval}},
 }
 
+@article{bhatt2023cyberseceval,
+  title   = {Purple Llama {CyberSecEval}: A Secure Coding Benchmark for Language Models},
+  author  = {Manish Bhatt and Sahana Chennabasappa and Cyrus Nikolaidis and Shengye Wan and Ivan Evtimov and Dominik Gabi and Daniel Song and Faizan Ahmad and Cornelius Aschermann and Lorenzo Fontana and Sasha Frolov and Ravi Prakash Giri and Dhaval Kapil and Yiannis Kozyrakis and David LeBlanc and James Milazzo and Aleksandar Straumann and Gabriel Synnaeve and Varun Vontimitta and Spencer Whitman and Joshua Saxe},
+  journal = {arXiv preprint arXiv:2312.04724},
+  year    = {2023},
+  url     = {https://arxiv.org/abs/2312.04724},
+}
+
 @article{kim2023propile,
   title     = {{ProPILE}: Probing Privacy Leakage in Large Language Models},
   author    = {Siwon Kim and Sangdoo Yun and Hwaran Lee and Martin Gubri and Sungroh Yoon and Seong Joon Oh},

pyrit/datasets/seed_datasets/remote/__init__.py

@@ -44,6 +44,10 @@
         ComicJailbreakTemplateConfig,
         _ComicJailbreakDataset,
     )
+    from pyrit.datasets.seed_datasets.remote.cyberseceval_mitre_dataset import (
+        CyberSecEvalMitreCategory,
+        _CyberSecEvalMitreDataset,
+    )
     from pyrit.datasets.seed_datasets.remote.dangerous_qa_dataset import _DangerousQADataset
     from pyrit.datasets.seed_datasets.remote.darkbench_dataset import _DarkBenchDataset
     from pyrit.datasets.seed_datasets.remote.decoding_trust_toxicity_dataset import (
@@ -165,6 +169,7 @@
     "AnthropicModelWrittenEvalCategory": "pyrit.datasets.seed_datasets.remote.anthropic_model_written_evals_dataset",
     "CoCoNotCategory": "pyrit.datasets.seed_datasets.remote.coconot_dataset",
     "CoCoNotSplit": "pyrit.datasets.seed_datasets.remote.coconot_dataset",
+    "CyberSecEvalMitreCategory": "pyrit.datasets.seed_datasets.remote.cyberseceval_mitre_dataset",
     "DecodingTrustToxicitySubset": "pyrit.datasets.seed_datasets.remote.decoding_trust_toxicity_dataset",
     "FigStepCategory": "pyrit.datasets.seed_datasets.remote.figstep_dataset",
     "FigStepVariant": "pyrit.datasets.seed_datasets.remote.figstep_dataset",
@@ -207,6 +212,7 @@
     "_ComicJailbreakDataset": "pyrit.datasets.seed_datasets.remote.comic_jailbreak_dataset",
     "COMIC_JAILBREAK_TEMPLATES": "pyrit.datasets.seed_datasets.remote.comic_jailbreak_dataset",
     "ComicJailbreakTemplateConfig": "pyrit.datasets.seed_datasets.remote.comic_jailbreak_dataset",
+    "_CyberSecEvalMitreDataset": "pyrit.datasets.seed_datasets.remote.cyberseceval_mitre_dataset",
     "_DangerousQADataset": "pyrit.datasets.seed_datasets.remote.dangerous_qa_dataset",
     "_DarkBenchDataset": "pyrit.datasets.seed_datasets.remote.darkbench_dataset",
     "_DecodingTrustToxicityDataset": "pyrit.datasets.seed_datasets.remote.decoding_trust_toxicity_dataset",