| { |
| "objective": { |
| "conv": 1.0, |
| "expr": 1.0, |
| "speed": 0.3, |
| "novelty": 1.0, |
| "invalid": -1.0 |
| }, |
| "layers": [ |
| { |
| "niche": [ |
| false, |
| false, |
| true, |
| true |
| ], |
| "reward": 5.821100126778342, |
| "formula": "x + gelu(glu2(gelu(neg(tanh(gelu(dwconv_k3(relu(linear(gelu(gelu(gelu(rmsnorm(x)))))))))))))", |
| "metrics": { |
| "early": 4.507159852981568, |
| "final": 3.2640185594558715, |
| "step_ms": 50.704002380371094, |
| "peak_gb": 11.556068864, |
| "params": 6248832, |
| "improved": true |
| } |
| }, |
| { |
| "niche": [ |
| false, |
| false, |
| false, |
| true |
| ], |
| "reward": 5.745956145798423, |
| "formula": "x + mlp2(silu(x))", |
| "metrics": { |
| "early": 4.346919012069702, |
| "final": 3.783213210105896, |
| "step_ms": 33.315181732177734, |
| "peak_gb": 8.055067648, |
| "params": 4918272, |
| "improved": true |
| } |
| }, |
| { |
| "niche": [ |
| true, |
| false, |
| false, |
| false |
| ], |
| "reward": 5.486117064034202, |
| "formula": "x + gelu((gelu((x \u2299\u03c3 (gelu((x \u2299\u03c3 scale(x))) \u2299\u03c3 scale(x)))) \u2299\u03c3 attn_h8(x)))", |
| "metrics": { |
| "early": 4.561727523803711, |
| "final": 3.775619101524353, |
| "step_ms": 59.35382843017578, |
| "peak_gb": 11.285428736, |
| "params": 4920576, |
| "improved": true |
| } |
| }, |
| { |
| "niche": [ |
| true, |
| false, |
| true, |
| false |
| ], |
| "reward": 5.154641150509574, |
| "formula": "x + gelu(gelu(gelu((x \u2299\u03c3 scale(relu(linear(gelu(dwconv_k3(gelu(gelu((scale(relu(linear(gelu(x)))) \u2299\u03c3 scale(relu(linear(gelu((scale(relu(linear(gelu((scale(relu(linear(gelu((attn_h8(linear(gelu((scale(relu(linear(gelu(dwconv_k3(x))))) \u2299\u03c3 scale(x))))) \u2299\u03c3 x))))) \u2299\u03c3 x))))) \u2299\u03c3 x)))))))))))))))))", |
| "metrics": { |
| "early": 4.642396020889282, |
| "final": 3.6228970527648925, |
| "step_ms": 120.60427665710449, |
| "peak_gb": 17.224617984, |
| "params": 8029824, |
| "improved": true |
| } |
| }, |
| { |
| "niche": [ |
| false, |
| false, |
| true, |
| false |
| ], |
| "reward": 5.121382938896873, |
| "formula": "x + gelu(gelu((scale(relu(linear(gelu((scale(relu(linear(x))) \u2299\u03c3 scale(relu(linear(gelu(silu(x)))))))))) \u2299\u03c3 scale(relu(linear(gelu(neg(dwconv_k3(linear(x))))))))))", |
| "metrics": { |
| "early": 3.988122344017029, |
| "final": 3.1123839378356934, |
| "step_ms": 56.54621124267578, |
| "peak_gb": 11.528669696, |
| "params": 5368704, |
| "improved": true |
| } |
| }, |
| { |
| "niche": [ |
| false, |
| true, |
| false, |
| true |
| ], |
| "reward": 4.835630094086387, |
| "formula": "x + expc(neg(gelu(gelu((scale(relu(linear(sigmoid(mlp4(gelu(linear(gelu(neg(tanh(tanh(scale(relu(linear(gelu(rmsnorm(x)))))))))))))))) \u2299\u03c3 scale(relu(linear(cummean(scale(relu(neg((gelu(glu2(gelu(tanh(scale(relu(linear(gelu(x)))))))) - scale(relu(linear(gelu((scale(relu(x)) \u2299\u03c3 x))))))))))))))))))", |
| "metrics": { |
| "early": 5.217595434188842, |
| "final": 3.9673903226852416, |
| "step_ms": 97.13172912597656, |
| "peak_gb": 17.459894272, |
| "params": 12004224, |
| "improved": true |
| } |
| }, |
| { |
| "niche": [ |
| false, |
| false, |
| false, |
| false |
| ], |
| "reward": 4.823786722695091, |
| "formula": "x + gelu((scale(relu(gelu(neg(tanh(x))))) \u2299\u03c3 scale(relu(rmsnorm(x)))))", |
| "metrics": { |
| "early": 4.0697242498397825, |
| "final": 3.7892616271972654, |
| "step_ms": 42.938947677612305, |
| "peak_gb": 9.84554752, |
| "params": 3151104, |
| "improved": true |
| } |
| }, |
| { |
| "niche": [ |
| true, |
| false, |
| true, |
| true |
| ], |
| "reward": 4.598309086834647, |
| "formula": "x + gelu((scale(relu(linear(x))) \u2299\u03c3 scale(relu(linear(gelu(gelu((x \u2299\u03c3 mlp4(attn_h8((x - scale(relu(abs(relu(linear(gelu((dwconv_k3(silu(x)) \u2299\u03c3 scale(relu(linear(gelu(abs(dwconv_k7(rmsnorm(x)))))))))))))))))))))))))", |
| "metrics": { |
| "early": 3.9827035903930663, |
| "final": 3.7270321369171144, |
| "step_ms": 92.9560661315918, |
| "peak_gb": 15.793202688, |
| "params": 10242816, |
| "improved": true |
| } |
| }, |
| { |
| "niche": [ |
| false, |
| true, |
| false, |
| false |
| ], |
| "reward": 4.508914195572593, |
| "formula": "x + expc((relu(linear(gelu((scale(relu(linear(gelu(neg(gelu(x)))))) \u2299\u03c3 scale(relu(linear(gelu(cummean(scale(relu(linear(gelu((scale(relu(linear(gelu(cummean(gelu(((scale(relu(linear(gelu(gelu((scale(relu(linear(gelu(gelu(x))))) \u2299\u03c3 x)))))) \u2299\u03c3 x) + x))))))) \u2299\u03c3 x)))))))))))))) - x))", |
| "metrics": { |
| "early": 4.0373770236969, |
| "final": 3.7183741331100464, |
| "step_ms": 104.02059555053711, |
| "peak_gb": 16.417043968, |
| "params": 6252288, |
| "improved": true |
| } |
| }, |
| { |
| "niche": [ |
| false, |
| true, |
| true, |
| true |
| ], |
| "reward": 4.334455405747153, |
| "formula": "x + gelu((scale(relu(linear(gelu((scale(relu(linear(dwconv_k3(relu(gelu(dwconv_k3(relu(linear(gelu(neg(cummean(relu(linear(gelu(linear(gelu(linear(silu(dwconv_k7(gelu(rmsnorm(x)))))))))))))))))))))) \u2299\u03c3 glu2(rmsnorm(x))))))) \u2299\u03c3 scale(relu(linear(gelu(rmsnorm(x)))))))", |
| "metrics": { |
| "early": 4.589245700836182, |
| "final": 3.7722170114517213, |
| "step_ms": 82.62109756469727, |
| "peak_gb": 14.808715264, |
| "params": 8918016, |
| "improved": true |
| } |
| }, |
| { |
| "niche": [ |
| true, |
| true, |
| true, |
| true |
| ], |
| "reward": 4.187065421616294, |
| "formula": "x + gelu(mlp4(scale(gelu((dwconv_k3(((linear(gelu(scale(dwconv_k3(rmsnorm(dwconv_k7(dwconv_k7(gelu(dwconv_k3(relu(rmsnorm(mlp4(linear(x))))))))))))) \u2299\u03c3 scale(relu(neg(cummean(attn_h8(tanh(attn_h8(tanh(tanh(abs(glu2(glu2(neg(dwconv_k7((rmsnorm(neg(neg(relu(dwconv_k5(attn_h2(scale(x))))))) + x)))))))))))))))) \u2299\u03c3 x)) + x)))))", |
| "metrics": { |
| "early": 4.385487127304077, |
| "final": 3.1699397802352904, |
| "step_ms": 174.35574531555176, |
| "peak_gb": 22.777659904, |
| "params": 21773184, |
| "improved": true |
| } |
| }, |
| { |
| "niche": [ |
| true, |
| false, |
| false, |
| true |
| ], |
| "reward": 4.0684950223319305, |
| "formula": "x + gelu((gelu((scale(relu(linear(gelu((scale(relu(linear(gelu(attn_h4(x))))) \u2299\u03c3 scale(x)))))) \u2299\u03c3 (scale(relu(linear(gelu(glu2(gelu(x)))))) \u2299\u03c3 scale(relu(linear(gelu((scale(relu(linear(gelu((x \u2299\u03c3 scale(relu(linear(gelu((scale(relu(linear(x))) \u2299\u03c3 x)))))))))) \u2299\u03c3 x)))))))) \u2299\u03c3 x))", |
| "metrics": { |
| "early": 4.622024250030518, |
| "final": 3.7339990377426147, |
| "step_ms": 127.40111351013184, |
| "peak_gb": 19.977999872, |
| "params": 10678272, |
| "improved": true |
| } |
| }, |
| { |
| "niche": [ |
| true, |
| true, |
| false, |
| false |
| ], |
| "reward": 3.7166457643859165, |
| "formula": "x + cummean(relu(gelu(scale(relu(attn_h4(x))))))", |
| "metrics": { |
| "early": 5.998728942871094, |
| "final": 5.8545290470123295, |
| "step_ms": 61.27309799194336, |
| "peak_gb": 9.077064704, |
| "params": 4919424, |
| "improved": true |
| } |
| } |
| ] |
| } |
