feat(eval): combine sec code oracles in one pipeline

eval/compile_xscode/README.md

@@ -0,0 +1,18 @@
+### XSCode
+[XSCode](https://huggingface.co/datasets/purpcode/XSCode) is an overrefusal benchmark for secure code generation. While benchmarks like CyberSecEval FRR prompts are lengthy and specifically target malicious cyber activities, XSCode contains 589 short and harmless code-generation prompts that do not contain any built-in code security bias.
+
+### XSCode Generation
+
+```
+export PYTHONPATH=$PYTHONPATH:$(pwd)
+python eval/compile_xscode/main.py
+```
+
+Generation requires AWS bedrock access.
+
+### XSCode Eval
+
+```
+export PYTHONPATH=$PYTHONPATH:$(pwd)
+python eval/main.py --task "purpcode/xscode" --model <MODEL_NAME>
+```

eval/compile_xscode/annotate.py → eval/compile_xscode/annotate_utils/annotate.py

@@ -2,7 +2,6 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-# pip install rich datasets
 import argparse
 import json
 import os

eval/compile_xscode/gather.py → eval/compile_xscode/annotate_utils/gather.py

@@ -2,11 +2,6 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-"""
-Script to process JSONL files and filter records where malicious, unnatural,
-and too_simple are all "disagree", then display statistics.
-"""
-
 import json
 from collections import Counter
 from pathlib import Path
@@ -20,7 +15,7 @@ def analyze_records(records):
         return None
     print(f"Total records: {len(records)}")
     print(
-        f"{len(set([ record['original_prompt']['additional_context']['cwe_id'] for record in records ]))} unique CWEs"
+        f"{len({record['original_prompt']['additional_context']['cwe_id'] for record in records})} unique CWEs"
     )
 
     # distribution per language

eval/compile_xscode/split.py → eval/compile_xscode/annotate_utils/split.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+
 import argparse
 import json
 from pathlib import Path