Task-Tangent Capture Pruning / bench_report.json
Mechanism confirmed, baseline not beaten
1{
2 "bench_version": 1,
3 "track": "tabular",
4 "model": "mlp_tiny",
5 "metric_direction": "lower is better",
6 "n_seeds": 8,
7 "baseline": {
8 "best_cfg": {
9 "lr": 0.01
10 },
11 "sweep": [
12 {
13 "cfg": {
14 "lr": 0.001
15 },
16 "mean": 95.01333999633789
17 },
18 {
19 "cfg": {
20 "lr": 0.003
21 },
22 "mean": 9.572970628738403
23 },
24 {
25 "cfg": {
26 "lr": 0.01
27 },
28 "mean": 7.331724643707275
29 }
30 ],
31 "full": {
32 "mean": 7.261052072048187,
33 "std": 0.9775059897186311,
34 "per_seed": [
35 6.919011116027832,
36 5.700474262237549,
37 7.776523113250732,
38 8.930890083312988,
39 7.3590850830078125,
40 6.166253566741943,
41 8.154062271118164,
42 7.082117080688477
43 ],
44 "n": 8
45 }
46 },
47 "idea": {
48 "mean": 7.309336423873901,
49 "std": 0.9807488529523701,
50 "per_seed": [
51 6.58059549331665,
52 5.827126979827881,
53 7.845516681671143,
54 8.944876670837402,
55 7.448084354400635,
56 6.235833644866943,
57 8.219693183898926,
58 7.372964382171631
59 ],
60 "n": 8
61 },
62 "comparison": {
63 "delta_mean": 0.04828435182571411,
64 "idea_wins": 1,
65 "n_pairs": 8,
66 "per_seed_diffs": [
67 -0.33841562271118164,
68 0.12665271759033203,
69 0.06899356842041016,
70 0.013986587524414062,
71 0.08899927139282227,
72 0.069580078125,
73 0.06563091278076172,
74 0.2908473014831543
75 ],
76 "p_value": 0.4521,
77 "mde": 0.1477390247386159,
78 "mde_rel_pct": 2.034677940230525,
79 "verdict": "no measurable effect",
80 "system_worked": false
81 },
82 "mechanism_signature": {
83 "track_choice": "tabular is structurally matched because this intervention is pruning/training-dynamics rather than convolution, attention, or control.",
84 "sparsity": 0.5,
85 "calibration_examples": 96,
86 "idea_config": {
87 "lr": 0.01
88 },
89 "idea_sweep": [
90 {
91 "cfg": {
92 "lr": 0.001
93 },
94 "result": {
95 "mean": 31.27217388153076,
96 "std": 7.421801352093722,
97 "per_seed": [
98 36.63642120361328,
99 29.88167953491211,
100 25.0460147857666,
101 29.383060455322266,
102 19.669511795043945,
103 34.03684616088867,
104 29.361820220947266,
105 46.16203689575195
106 ],
107 "n": 8
108 }
109 },
110 {
111 "cfg": {
112 "lr": 0.003
113 },
114 "result": {
115 "mean": 8.934857428073883,
116 "std": 1.2454062470166958,
117 "per_seed": [
118 8.492694854736328,
119 8.243812561035156,
120 10.493083000183105,
121 8.787611961364746,
122 10.205215454101562,
123 6.738030910491943,
124 10.40708065032959,
125 8.111330032348633
126 ],
127 "n": 8
128 }
129 },
130 {
131 "cfg": {
132 "lr": 0.01
133 },
134 "result": {
135 "mean": 7.309336423873901,
136 "std": 0.9807488529523701,
137 "per_seed": [
138 6.58059549331665,
139 5.827126979827881,
140 7.845516681671143,
141 8.944876670837402,
142 7.448084354400635,
143 6.235833644866943,
144 8.219693183898926,
145 7.372964382171631
146 ],
147 "n": 8
148 }
149 }
150 ],
151 "mechanism_signature": {
152 "prediction": "task-tangent mask discards a small fraction of calibration tangent energy",
153 "predicted_ratio_bound": 1.0,
154 "observed_mean_ratio": 0.002711589972022921,
155 "observed_max_ratio": 0.015607444569468498,
156 "model_rows": [
157 {
158 "seed": 0,
159 "baseline_final": 10.01066780090332,
160 "idea_final": 8.492694854736328,
161 "baseline_immediate": 18.023481369018555,
162 "idea_immediate": 17.420024871826172,
163 "predicted_tangent_ratio_bound": 1.0,
164 "observed_tangent_ratio": 0.0
165 },
166 {
167 "seed": 1,
168 "baseline_final": 8.832267761230469,
169 "idea_final": 8.243812561035156,
170 "baseline_immediate": 15.395939826965332,
171 "idea_immediate": 15.474164962768555,
172 "predicted_tangent_ratio_bound": 1.0,
173 "observed_tangent_ratio": 0.0
174 },
175 {
176 "seed": 2,
177 "baseline_final": 10.172430038452148,
178 "idea_final": 10.493083000183105,
179 "baseline_immediate": 20.724096298217773,
180 "idea_immediate": 27.978755950927734,
181 "predicted_tangent_ratio_bound": 1.0,
182 "observed_tangent_ratio": 0.015607444569468498
183 },
184 {
185 "seed": 3,
186 "baseline_final": 9.276516914367676,
187 "idea_final": 8.787611961364746,
188 "baseline_immediate": 16.58881378173828,
189 "idea_immediate": 17.414724349975586,
190 "predicted_tangent_ratio_bound": 1.0,
191 "observed_tangent_ratio": 0.0
192 },
193 {
194 "seed": 4,
195 "baseline_final": 9.78347110748291,
196 "idea_final": 10.205215454101562,
197 "baseline_immediate": 20.978132247924805,
198 "idea_immediate": 32.86971664428711,
199 "predicted_tangent_ratio_bound": 1.0,
200 "observed_tangent_ratio": 0.0060852752067148685
201 },
202 {
203 "seed": 5,
204 "baseline_final": 9.389260292053223,
205 "idea_final": 6.738030910491943,
206 "baseline_immediate": 17.720703125,
207 "idea_immediate": 16.962766647338867,
208 "predicted_tangent_ratio_bound": 1.0,
209 "observed_tangent_ratio": 0.0
210 },
211 {
212 "seed": 6,
213 "baseline_final": 10.443191528320312,
214 "idea_final": 10.40708065032959,
215 "baseline_immediate": 19.673473358154297,
216 "idea_immediate": 22.77062225341797,
217 "predicted_tangent_ratio_bound": 1.0,
218 "observed_tangent_ratio": 0.0
219 },
220 {
221 "seed": 7,
222 "baseline_final": 12.183694839477539,
223 "idea_final": 8.111330032348633,
224 "baseline_immediate": 24.06068992614746,
225 "idea_immediate": 21.5717716217041,
226 "predicted_tangent_ratio_bound": 1.0,
227 "observed_tangent_ratio": 0.0
228 }
229 ],
230 "confirmed": true
231 }
232 }
233}