import os, json, math, random, time import numpy as np import torch import torch.nn as nn SEED = 558 random.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED) torch.set_num_threads(min(12, os.cpu_count() or 1)) try: device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') if device.type == 'cuda': torch.cuda.reset_peak_memory_stats() except Exception: device = torch.device('cpu') def L_min(n, d, m, eps, c0=0.25, C0=2.0, delta=1e-6): eps = max(float(eps), delta) q = (m + 1) * math.log(max(C0 * (m + 1) * n * d / eps, 1.000001)) return c0 * eps * math.sqrt(n / q) class MLP(nn.Module): def __init__(self, d, width): super().__init__() self.d, self.width = d, width self.net = nn.Sequential(nn.Linear(d, width), nn.ReLU(), nn.Linear(width, 1)) def forward(self, x): return self.net(x).squeeze(-1) def widen(self, new_width): old = self.net new = MLP(self.d, new_width).to(next(self.parameters()).device) with torch.no_grad(): new.net[0].weight[:self.width].copy_(old[0].weight) new.net[0].bias[:self.width].copy_(old[0].bias) new.net[2].weight[:, :self.width].copy_(old[2].weight) new.net[2].bias.copy_(old[2].bias) # New units have small output coefficients, as prescribed. new.net[2].weight[:, self.width:].mul_(0.02) return new def jacobian_lip(model, x): # Exact scalar-output Jacobian row norm on a small probe set. vals = [] for z in x: zz = z.detach().clone().requires_grad_(True) y = model(zz[None])[0] g = torch.autograd.grad(y, zz, retain_graph=False)[0] vals.append(g.norm().item()) return max(vals) if vals else 0.0 def run(controller, seed=SEED): torch.manual_seed(seed); np.random.seed(seed) n, d, noise = 320, 5, 0.35 x = torch.randn(n, d, device=device) clean = torch.sin(x[:, 0]) + 0.35 * x[:, 1] - 0.2 * x[:, 2]**2 y = clean + noise * torch.randn(n, device=device) perm = torch.randperm(n, device=device) tr, va = perm[:240], perm[240:] model = MLP(d, 8).to(device) opt = torch.optim.Adam(model.parameters(), lr=0.015) history=[]; widen_count=0; t0=time.time() # Same number of update intervals for both conditions; widening adds capacity, not extra steps. for step in range(1, 801): model.train(); opt.zero_grad() loss = ((model(x[tr]) - y[tr])**2).mean() loss.backward(); opt.step() if step % 80 == 0: model.eval() with torch.no_grad(): train_mse = ((model(x[tr])-y[tr])**2).mean().item() val_noisy = ((model(x[va])-y[va])**2).mean().item() val_clean = ((model(x[va])-clean[va])**2).mean().item() # Probe sensitivity on held-out points. The theorem's constants are empirical, # deliberately explicit and fixed across both runs. lip = jacobian_lip(model, x[va[:32]]) eps = math.sqrt(max(train_mse, 1e-12)) bound = L_min(len(tr), d, model.width, eps) if controller and train_mse < noise**2 and lip < 0.9 * bound and model.width < 64: oldw=model.width; model=model.widen(min(64, max(oldw+1, int(math.ceil(oldw*1.5))))) opt=torch.optim.Adam(model.parameters(), lr=0.015) widen_count += 1 history.append({'step':step,'width':model.width,'train_mse':train_mse,'val_noisy_mse':val_noisy,'val_clean_mse':val_clean,'lip':lip,'L_min':bound}) # Noise robustness is measured against the clean target on fresh Gaussian probes. with torch.no_grad(): xte=torch.randn(160,d,device=device) clean_te=torch.sin(xte[:,0])+0.35*xte[:,1]-0.2*xte[:,2]**2 pred=model(xte) clean_mse=((pred-clean_te)**2).mean().item() return {'history':history,'final_width':model.width,'widen_count':widen_count,'test_clean_mse':clean_mse,'seconds':time.time()-t0} def math_check(): # Claimed scaling: increasing m lowers the necessary lower bound, while increasing # epsilon raises it. Ratios are checked against monotonicity, not a fitted theorem. widths=[2,4,8,16,32,64] by_width=[L_min(240,5,m,0.35) for m in widths] errs=[0.08,0.16,0.32,0.64] by_error=[L_min(240,5,16,e) for e in errs] return {'widths':widths,'L_by_width':by_width,'errors':errs,'L_by_error':by_error, 'width_monotone':all(a>b for a,b in zip(by_width,by_width[1:])), 'error_monotone':all(a