Age-conditioned semi-Markov router / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "sequence",
4 "model": "transformer_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.01,
10 "temperature": 0.8,
11 "hazard_bias": 0.0
12 },
13 "sweep": [
14 {
15 "cfg": {
16 "lr": 0.001,
17 "temperature": 0.8,
18 "hazard_bias": 0.0
19 },
20 "mean": 1.0667133182287216
21 },
22 {
23 "cfg": {
24 "lr": 0.001,
25 "temperature": 1.2,
26 "hazard_bias": 0.0
27 },
28 "mean": 1.0630156695842743
29 },
30 {
31 "cfg": {
32 "lr": 0.003,
33 "temperature": 0.8,
34 "hazard_bias": 0.0
35 },
36 "mean": 1.07551509141922
37 },
38 {
39 "cfg": {
40 "lr": 0.003,
41 "temperature": 1.2,
42 "hazard_bias": 0.0
43 },
44 "mean": 1.0769807696342468
45 },
46 {
47 "cfg": {
48 "lr": 0.01,
49 "temperature": 0.8,
50 "hazard_bias": 0.0
51 },
52 "mean": 1.0265950411558151
53 },
54 {
55 "cfg": {
56 "lr": 0.01,
57 "temperature": 1.2,
58 "hazard_bias": 0.0
59 },
60 "mean": 1.0694856494665146
61 }
62 ],
63 "full": {
64 "mean": 1.0413843393325806,
65 "std": 0.07579068628025519,
66 "per_seed": [
67 1.030417799949646,
68 0.8935723900794983,
69 1.0985037088394165,
70 1.0838862657546997,
71 0.9647193551063538,
72 1.0631779432296753,
73 1.1534550189971924,
74 1.0433422327041626
75 ],
76 "n": 8
77 }
78 },
79 "idea": {
80 "mean": 1.0507365465164185,
81 "std": 0.07984042752181207,
82 "per_seed": [
83 1.0360348224639893,
84 0.9041898846626282,
85 1.1562472581863403,
86 1.0952754020690918,
87 0.9761890769004822,
88 1.0409249067306519,
89 1.1531434059143066,
90 1.0438876152038574
91 ],
92 "n": 8,
93 "chosen_cfg": {
94 "lr": 0.01,
95 "temperature": 0.8,
96 "hazard_bias": 0.0
97 },
98 "sweep": [
99 {
100 "cfg": {
101 "lr": 0.01,
102 "temperature": 0.8,
103 "hazard_bias": -1.0
104 },
105 "result": {
106 "mean": 1.0596621185541153,
107 "std": 0.07361314701141786,
108 "per_seed": [
109 1.071687936782837,
110 0.9170374870300293,
111 1.136078119277954,
112 1.1125483512878418,
113 0.9964454174041748,
114 1.02909517288208,
115 1.15607750415802,
116 1.0583269596099854
117 ],
118 "n": 8
119 }
120 },
121 {
122 "cfg": {
123 "lr": 0.01,
124 "temperature": 0.8,
125 "hazard_bias": 0.0
126 },
127 "result": {
128 "mean": 1.0507365465164185,
129 "std": 0.07984042752181207,
130 "per_seed": [
131 1.0360348224639893,
132 0.9041898846626282,
133 1.1562472581863403,
134 1.0952754020690918,
135 0.9761890769004822,
136 1.0409249067306519,
137 1.1531434059143066,
138 1.0438876152038574
139 ],
140 "n": 8
141 }
142 },
143 {
144 "cfg": {
145 "lr": 0.01,
146 "temperature": 0.8,
147 "hazard_bias": 1.0
148 },
149 "result": {
150 "mean": 1.0628163740038872,
151 "std": 0.06874408412093434,
152 "per_seed": [
153 1.0399799346923828,
154 0.9146118760108948,
155 1.1155239343643188,
156 1.1016066074371338,
157 1.0239859819412231,
158 1.0964356660842896,
159 1.1542212963104248,
160 1.0561656951904297
161 ],
162 "n": 8
163 }
164 }
165 ]
166 },
167 "comparison": {
168 "delta_mean": 0.00935220718383789,
169 "idea_wins": 2,
170 "n_pairs": 8,
171 "per_seed_diffs": [
172 0.005617022514343262,
173 0.010617494583129883,
174 0.05774354934692383,
175 0.01138913631439209,
176 0.011469721794128418,
177 -0.022253036499023438,
178 -0.0003116130828857422,
179 0.0005453824996948242
180 ],
181 "p_value": 0.3073,
182 "mde": 0.018792208195343812,
183 "mde_rel_pct": 1.8045410791742527,
184 "verdict": "no measurable effect",
185 "system_worked": false
186 },
187 "track_justification": "Sequence forecasting has multi-token correlations and regime persistence, matching semi-Markov routing.",
188 "mechanism_signature": {
189 "predicted_mean_hazard": 0.5708998311311007,
190 "observed_trace_change": 0.009716346365166828,
191 "predicted_vs_observed_ratio": 58.75663646345304,
192 "age_mean": 0.9239334501326084,
193 "per_seed": [
194 {
195 "predicted_mean_hazard": 0.7784030437469482,
196 "observed_trace_change": 0.016614258289337158,
197 "age_mean": 0.2825595736503601
198 },
199 {
200 "predicted_mean_hazard": 0.6146835684776306,
201 "observed_trace_change": 0.0018246846739202738,
202 "age_mean": 0.6145921945571899
203 },
204 {
205 "predicted_mean_hazard": 0.622718870639801,
206 "observed_trace_change": 0.004094474017620087,
207 "age_mean": 0.594530463218689
208 },
209 {
210 "predicted_mean_hazard": 0.5631765127182007,
211 "observed_trace_change": 0.00499045429751277,
212 "age_mean": 0.7603605389595032
213 },
214 {
215 "predicted_mean_hazard": 0.5672649145126343,
216 "observed_trace_change": 0.012080281972885132,
217 "age_mean": 0.7463456392288208
218 },
219 {
220 "predicted_mean_hazard": 0.5145835280418396,
221 "observed_trace_change": 0.006644372828304768,
222 "age_mean": 0.9115980863571167
223 },
224 {
225 "predicted_mean_hazard": 0.23928262293338776,
226 "observed_trace_change": 0.017612455412745476,
227 "age_mean": 2.9931130409240723
228 },
229 {
230 "predicted_mean_hazard": 0.667085587978363,
231 "observed_trace_change": 0.01386978942900896,
232 "age_mean": 0.48836806416511536
233 }
234 ],
235 "confirmed": false
236 }
237}