Seed-Anchored Budgeted Graph Context / experiment.py

Mechanism confirmed, baseline not beaten

Raw ⬇ ZIP
  1import json, random, hashlib
  2from collections import deque
  3from dataclasses import dataclass
  4from pathlib import Path
  5
  6@dataclass(frozen=True)
  7class Unit:
  8    kind: str
  9    ident: str
 10    hop: int
 11    text: str
 12    length: int
 13
 14
 15def build_graph(n=36, seed=17):
 16    rng = random.Random(seed)
 17    names = [f"Entity_{i:02d}" for i in range(n)]
 18    desc = {x: f"{x} is a synthetic entity with property {((i*7)%19)+3} and category C{i%5}." for i, x in enumerate(names)}
 19    edges = []
 20    # A connected backbone plus deterministic extra links.
 21    for i in range(n - 1):
 22        edges.append((names[i], "related_to", names[i+1]))
 23    seen = {(u, w) for u, _, w in edges}
 24    for _ in range(42):
 25        u, w = rng.sample(names, 2)
 26        if (u, w) not in seen and (w, u) not in seen:
 27            edges.append((u, "linked_with", w)); seen.add((u, w))
 28    edges.sort(key=lambda x: (x[0], x[1], x[2]))
 29    return names, desc, edges
 30
 31
 32def name_match(question, names):
 33    # deterministic exact entity-name matching, with stable identifier order
 34    return sorted([x for x in names if x in question])
 35
 36
 37def distances(names, edges, seeds, k):
 38    adj = {x: [] for x in names}
 39    for u, _, w in edges:
 40        adj[u].append(w); adj[w].append(u)
 41    d = {s: 0 for s in seeds}
 42    q = deque(sorted(seeds))
 43    while q:
 44        v = q.popleft()
 45        if d[v] >= k: continue
 46        for w in sorted(adj[v]):
 47            if w not in d:
 48                d[w] = d[v] + 1; q.append(w)
 49    return d
 50
 51
 52def units_for(names, desc, edges, d, separator="\n"):
 53    units = []
 54    for v in sorted(d):
 55        text = f"NODE {v}: {desc[v]}"
 56        units.append(Unit("node", v, d[v], text, len(text) + len(separator)))
 57    for i, (u, r, w) in enumerate(edges):
 58        if u in d and w in d:
 59            ident = f"edge:{u}|{r}|{w}|{i:03d}"
 60            text = f"EDGE {u} -[{r}]-> {w}"
 61            units.append(Unit("edge", ident, max(d[u], d[w]), text, len(text) + len(separator)))
 62    return units
 63
 64
 65def render(units, budget, mode="anchored", rng=None):
 66    if mode == "anchored":
 67        ordered = sorted(units, key=lambda x: (x.hop, x.ident))
 68    elif mode == "global":
 69        ordered = sorted(units, key=lambda x: x.ident)
 70    elif mode == "random":
 71        ordered = list(units); rng.shuffle(ordered)
 72    else:
 73        raise ValueError(mode)
 74    chosen, used = [], 0
 75    for u in ordered:
 76        if used + u.length > budget: break
 77        chosen.append(u); used += u.length
 78    return separator_join(chosen), chosen, used
 79
 80
 81def separator_join(chosen):
 82    return "\n".join(u.text for u in chosen) + ("\n" if chosen else "")
 83
 84
 85def evaluate(units, budgets, gold_ids):
 86    out = {}
 87    for mode in ("anchored", "global", "random"):
 88        rows = []
 89        for b in budgets:
 90            rng = random.Random(1000 + b)
 91            context, chosen, used = render(units, b, mode, rng)
 92            ids = {u.ident for u in chosen}
 93            rows.append({"budget": b, "recall": len(ids & gold_ids) / len(gold_ids),
 94                         "units": len(chosen), "used": used, "context_sha256": hashlib.sha256(context.encode()).hexdigest()})
 95        out[mode] = rows
 96    return out
 97
 98
 99def main():
100    names, desc, edges = build_graph()
101    question = "Compare Entity_03 and Entity_11 and explain their local connections."
102    seeds = name_match(question, names)
103    k = 2
104    d = distances(names, edges, seeds, k)
105    units = units_for(names, desc, edges, d)
106    D = sum(u.length for u in units)
107    # The candidate set itself is the annotated gold region for the formal claim.
108    gold = {u.ident for u in units}
109    # Include values around the mathematically predicted transition and a wide range.
110    budgets = sorted(set([1, D//4, D//2, max(1, D-20), D-1, D, D+1, D+100, 2*D]))
111    results = evaluate(units, budgets, gold)
112    # Verify exact implication and stable contexts under repeated execution.
113    full_below = all(next(r for r in results["anchored"] if r["budget"] == b)["recall"] < 1.0 for b in budgets if b < D)
114    full_at = next(r for r in results["anchored"] if r["budget"] == D)["recall"] == 1.0
115    full_above = all(next(r for r in results["anchored"] if r["budget"] == b)["recall"] == 1.0 for b in budgets if b >= D)
116    c1 = render(units, max(1, D//2), "anchored", random.Random(4))[0]
117    c2 = render(units, max(1, D//2), "anchored", random.Random(999))[0]
118    stable = c1 == c2
119    # Small useful summary at budgets where truncation is meaningful.
120    summary = []
121    for b in [D//4, D//2, D-1, D, D+100]:
122        row = {"budget": b}
123        for mode in ("anchored", "global", "random"):
124            row[mode] = next(x["recall"] for x in results[mode] if x["budget"] == b)
125        summary.append(row)
126    report = {"seeds": seeds, "k": k, "candidate_units": len(units), "D_chars": D,
127              "math_check": {"all_budgets_below_D_incomplete": full_below,
128                             "at_D_full": full_at, "all_tested_budgets_at_or_above_D_full": full_above,
129                             "repeated_context_identical": stable},
130              "summary": summary, "all_results": results}
131    Path("results.json").write_text(json.dumps(report, indent=2))
132    print(json.dumps(report, indent=2))
133
134if __name__ == "__main__":
135    main()