Orthogonal-Rank Contextual Memory / bench_report.json
Failed on benchmark
1{
2 "bench_version": 1,
3 "track": "sparse_cycle_compatibility",
4 "model": "custom_gru",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.01
10 },
11 "sweep": [
12 {
13 "cfg": {
14 "lr": 0.001
15 },
16 "mean": 0.6574999913573265
17 },
18 {
19 "cfg": {
20 "lr": 0.003
21 },
22 "mean": 0.5949999913573265
23 },
24 {
25 "cfg": {
26 "lr": 0.01
27 },
28 "mean": 0.4999999776482582
29 }
30 ],
31 "full": {
32 "mean": 0.5137499831616879,
33 "std": 0.1000234298210712,
34 "per_seed": [
35 0.5999999642372131,
36 0.5399999618530273,
37 0.5699999928474426,
38 0.28999999165534973,
39 0.4699999988079071,
40 0.48499998450279236,
41 0.6399999856948853,
42 0.5149999856948853
43 ],
44 "n": 8
45 }
46 },
47 "idea": {
48 "mean": 0.55375000461936,
49 "std": 0.09809911813863173,
50 "per_seed": [
51 0.6650000214576721,
52 0.574999988079071,
53 0.6299999952316284,
54 0.33000001311302185,
55 0.5199999809265137,
56 0.5400000214576721,
57 0.6349999904632568,
58 0.5350000262260437
59 ],
60 "n": 8,
61 "lr": 0.01,
62 "lambda_orth": 0.0,
63 "edge_violation_mean": 0.8190760239958763
64 },
65 "comparison": {
66 "delta_mean": 0.04000002145767212,
67 "idea_wins": 1,
68 "n_pairs": 8,
69 "per_seed_diffs": [
70 0.06500005722045898,
71 0.0350000262260437,
72 0.06000000238418579,
73 0.04000002145767212,
74 0.04999998211860657,
75 0.05500003695487976,
76 -0.004999995231628418,
77 0.020000040531158447
78 ],
79 "p_value": 0.0169,
80 "mde": 0.019481796815583544,
81 "mde_rel_pct": 3.792077363329512,
82 "verdict": "idea worse (significant)",
83 "system_worked": false
84 },
85 "custom_track": {
86 "name": "sparse_cycle_compatibility",
87 "file": "bench/custom_tracks/sparse_cycle_compatibility.py",
88 "domain": "masked_categorical_compatibility"
89 },
90 "protocol_notes": "8 paired seeds; registered graph-compatible track; identical GRU hidden width, epochs, batch, and lr union.",
91 "mechanism_signature": {
92 "predicted_effect": "orthogonality penalty lowers exclusivity-edge embedding inner products",
93 "baseline_edge_violation": null,
94 "idea_edge_violation": 0.8190760239958763,
95 "observed_reduction": null,
96 "confirmed": false
97 },
98 "idea_sweep": [
99 {
100 "lr": 0.001,
101 "lambda_orth": 0.0,
102 "mean": 0.7193749994039536,
103 "std": 0.14231209834077435,
104 "per_seed": [
105 0.7799999713897705,
106 0.8500000238418579,
107 0.7400000095367432,
108 0.36000001430511475,
109 0.800000011920929,
110 0.7799999713897705,
111 0.6949999928474426,
112 0.75
113 ],
114 "edge_violation_mean": 0.9312416091561317
115 },
116 {
117 "lr": 0.001,
118 "lambda_orth": 1.0,
119 "mean": 0.7375000044703484,
120 "std": 0.14866068540331184,
121 "per_seed": [
122 0.7799999713897705,
123 0.8500000238418579,
124 0.8550000190734863,
125 0.36000001430511475,
126 0.800000011920929,
127 0.7799999713897705,
128 0.7250000238418579,
129 0.75
130 ],
131 "edge_violation_mean": 0.8652326688170433
132 },
133 {
134 "lr": 0.001,
135 "lambda_orth": 3.0,
136 "mean": 0.7375000044703484,
137 "std": 0.14866068540331184,
138 "per_seed": [
139 0.7799999713897705,
140 0.8500000238418579,
141 0.8550000190734863,
142 0.36000001430511475,
143 0.800000011920929,
144 0.7799999713897705,
145 0.7250000238418579,
146 0.75
147 ],
148 "edge_violation_mean": 0.8657455369830132
149 },
150 {
151 "lr": 0.003,
152 "lambda_orth": 0.0,
153 "mean": 0.6524999961256981,
154 "std": 0.14166421585966793,
155 "per_seed": [
156 0.8050000071525574,
157 0.7850000262260437,
158 0.7749999761581421,
159 0.36000001430511475,
160 0.5199999809265137,
161 0.6449999809265137,
162 0.6899999976158142,
163 0.6399999856948853
164 ],
165 "edge_violation_mean": 0.8980525210499763
166 },
167 {
168 "lr": 0.003,
169 "lambda_orth": 1.0,
170 "mean": 0.7012499943375587,
171 "std": 0.12996994825186,
172 "per_seed": [
173 0.8050000071525574,
174 0.8650000095367432,
175 0.824999988079071,
176 0.42500001192092896,
177 0.7049999833106995,
178 0.6449999809265137,
179 0.6899999976158142,
180 0.6499999761581421
181 ],
182 "edge_violation_mean": 0.6635167449712753
183 },
184 {
185 "lr": 0.003,
186 "lambda_orth": 3.0,
187 "mean": 0.7012499943375587,
188 "std": 0.12996994825186,
189 "per_seed": [
190 0.8050000071525574,
191 0.8650000095367432,
192 0.824999988079071,
193 0.42500001192092896,
194 0.7049999833106995,
195 0.6449999809265137,
196 0.6899999976158142,
197 0.6499999761581421
198 ],
199 "edge_violation_mean": 0.6590283587574959
200 },
201 {
202 "lr": 0.01,
203 "lambda_orth": 0.0,
204 "mean": 0.55375000461936,
205 "std": 0.09809911813863173,
206 "per_seed": [
207 0.6650000214576721,
208 0.574999988079071,
209 0.6299999952316284,
210 0.33000001311302185,
211 0.5199999809265137,
212 0.5400000214576721,
213 0.6349999904632568,
214 0.5350000262260437
215 ],
216 "edge_violation_mean": 0.8190760239958763
217 },
218 {
219 "lr": 0.01,
220 "lambda_orth": 1.0,
221 "mean": 0.5756249986588955,
222 "std": 0.12212793455754742,
223 "per_seed": [
224 0.675000011920929,
225 0.7699999809265137,
226 0.6549999713897705,
227 0.33000001311302185,
228 0.5199999809265137,
229 0.5400000214576721,
230 0.5600000023841858,
231 0.5550000071525574
232 ],
233 "edge_violation_mean": 0.22121139988303185
234 },
235 {
236 "lr": 0.01,
237 "lambda_orth": 3.0,
238 "mean": 0.6031250022351742,
239 "std": 0.11712539810482167,
240 "per_seed": [
241 0.675000011920929,
242 0.7400000095367432,
243 0.675000011920929,
244 0.33000001311302185,
245 0.6200000047683716,
246 0.574999988079071,
247 0.6549999713897705,
248 0.5550000071525574
249 ],
250 "edge_violation_mean": 0.15788554586470127
251 }
252 ]
253}