Repository navigation
Expand file tree
/
Copy pathscore_v2.py
More file actions
73 lines (60 loc) · 2.83 KB
/
Copy pathscore_v2.py
File metadata and controls
73 lines (60 loc) · 2.83 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
"""
score_v2.py — score any prompt version and save versioned outputs.
Usage:
python score_v2.py v1 # uses prompts/classifier_v1.txt → data/eval_results_v1.csv
python score_v2.py v2 # uses prompts/classifier_v2.txt → data/eval_results_v2.csv
python score_v2.py v3 # uses prompts/classifier_v3.txt → data/eval_results_v3.csv
"""
import sys, csv, os, pathlib, datetime
sys.path.insert(0, "src")
from dotenv import load_dotenv; load_dotenv()
from openai import OpenAI
from classifier import classify_email, load_prompt, NEBIUS_BASE_URL, NEBIUS_MODEL
version = sys.argv[1] if len(sys.argv) > 1 else "v3"
base = pathlib.Path(".")
prompt_path = base / "prompts" / f"classifier_{version}.txt"
out_path = base / "data" / f"eval_results_{version}.csv"
summary_path = base / "data" / "eval_summary.csv"
if not prompt_path.exists():
print(f"ERROR: {prompt_path} not found"); sys.exit(1)
client = OpenAI(base_url=NEBIUS_BASE_URL, api_key=os.environ["NEBIUS_API_KEY"])
system_prompt = load_prompt(str(prompt_path))
results = []
with open("data/dataset.csv", newline="", encoding="utf-8") as f:
rows = list(csv.DictReader(f))
print(f"Scoring {version} on {len(rows)} emails...")
for i, row in enumerate(rows, 1):
result = classify_email(client, system_prompt, row["query"])
match = "PASS" if result["predicted"] == row["intent"] else "FAIL"
results.append({"id": row["id"], "query": row["query"],
"ground_truth": row["intent"], "predicted": result["predicted"], "match": match})
if i % 10 == 0:
print(f" {i}/100 done...")
with open(out_path, "w", newline="", encoding="utf-8") as f:
w = csv.DictWriter(f, fieldnames=["id","query","ground_truth","predicted","match"])
w.writeheader(); w.writerows(results)
passes = sum(1 for r in results if r["match"] == "PASS")
fails = [r for r in results if r["match"] == "FAIL"]
today = datetime.date.today().isoformat()
print(f"\n{version} Accuracy: {passes}/100 = {passes}%")
if fails:
print("Failures:")
for r in fails:
print(f" {r['id']}: truth={r['ground_truth']} predicted={r['predicted']}")
else:
print("All 100 PASS!")
# Append/update eval_summary.csv
existing = []
if summary_path.exists():
with open(summary_path, newline="", encoding="utf-8") as f:
existing = list(csv.DictReader(f))
# Remove old row for this version if it exists
existing = [r for r in existing if r["version"] != version]
existing.append({"version": version, "accuracy": passes, "passes": passes, "total": len(rows), "timestamp": today})
# Sort by version
existing.sort(key=lambda r: r["version"])
with open(summary_path, "w", newline="", encoding="utf-8") as f:
w = csv.DictWriter(f, fieldnames=["version","accuracy","passes","total","timestamp"])
w.writeheader(); w.writerows(existing)
print(f"\nSaved: {out_path}")
print(f"Updated: {summary_path}")