Seed-Anchored Budgeted Graph Context / mini_experiment.py

Mechanism confirmed, baseline not beaten

Raw ⬇ ZIP
 1import json, random
 2from pathlib import Path
 3from experiment import build_graph, name_match, distances, units_for, render
 4
 5
 6def run():
 7    names, desc, edges = build_graph(n=48, seed=23)
 8    queries = [
 9        f"Explain {names[i]} and {names[j]}" for i, j in
10        [(1, 18), (4, 27), (8, 35), (12, 43), (20, 31), (2, 40), (15, 29), (6, 22)]
11    ]
12    rows = []
13    for qi, q in enumerate(queries):
14        seeds = name_match(q, names)
15        d = distances(names, edges, seeds, 2)
16        units = units_for(names, desc, edges, d)
17        # Relevant evidence is the complete seed-local one-hop region.
18        gold = {u.ident for u in units if u.hop <= 1}
19        gold_len = sum(u.length for u in units if u.hop <= 1)
20        D = sum(u.length for u in units)
21        # A budget that fits the relevant region but not generally the full region.
22        budget = gold_len
23        row = {"query": q, "D": D, "gold_units": len(gold), "budget": budget}
24        for mode in ("anchored", "global", "random"):
25            _, chosen, used = render(units, budget, mode, random.Random(700 + qi))
26            got = {u.ident for u in chosen}
27            row[mode] = {"recall": len(got & gold) / len(gold), "units": len(chosen), "used": used}
28        rows.append(row)
29    avg = {m: sum(r[m]["recall"] for r in rows) / len(rows) for m in ("anchored", "global", "random")}
30    exact = {m: sum(r[m]["recall"] == 1.0 for r in rows) for m in avg}
31    report = {"queries": len(rows), "rows": rows, "average_relevant_recall": avg,
32              "queries_with_full_relevant_recall": exact}
33    Path("mini_results.json").write_text(json.dumps(report, indent=2))
34    print(json.dumps(report, indent=2))
35
36if __name__ == "__main__":
37    run()