{
  "objective": {
    "conv": 1.0,
    "expr": 1.0,
    "speed": 0.3,
    "novelty": 1.0,
    "invalid": -1.0
  },
  "layers": [
    {
      "niche": [
        false,
        false,
        true,
        true
      ],
      "reward": 5.821100126778342,
      "formula": "x + gelu(glu2(gelu(neg(tanh(gelu(dwconv_k3(relu(linear(gelu(gelu(gelu(rmsnorm(x)))))))))))))",
      "metrics": {
        "early": 4.507159852981568,
        "final": 3.2640185594558715,
        "step_ms": 50.704002380371094,
        "peak_gb": 11.556068864,
        "params": 6248832,
        "improved": true
      }
    },
    {
      "niche": [
        false,
        false,
        false,
        true
      ],
      "reward": 5.745956145798423,
      "formula": "x + mlp2(silu(x))",
      "metrics": {
        "early": 4.346919012069702,
        "final": 3.783213210105896,
        "step_ms": 33.315181732177734,
        "peak_gb": 8.055067648,
        "params": 4918272,
        "improved": true
      }
    },
    {
      "niche": [
        true,
        false,
        false,
        false
      ],
      "reward": 5.486117064034202,
      "formula": "x + gelu((gelu((x \u2299\u03c3 (gelu((x \u2299\u03c3 scale(x))) \u2299\u03c3 scale(x)))) \u2299\u03c3 attn_h8(x)))",
      "metrics": {
        "early": 4.561727523803711,
        "final": 3.775619101524353,
        "step_ms": 59.35382843017578,
        "peak_gb": 11.285428736,
        "params": 4920576,
        "improved": true
      }
    },
    {
      "niche": [
        true,
        false,
        true,
        false
      ],
      "reward": 5.154641150509574,
      "formula": "x + gelu(gelu(gelu((x \u2299\u03c3 scale(relu(linear(gelu(dwconv_k3(gelu(gelu((scale(relu(linear(gelu(x)))) \u2299\u03c3 scale(relu(linear(gelu((scale(relu(linear(gelu((scale(relu(linear(gelu((attn_h8(linear(gelu((scale(relu(linear(gelu(dwconv_k3(x))))) \u2299\u03c3 scale(x))))) \u2299\u03c3 x))))) \u2299\u03c3 x))))) \u2299\u03c3 x)))))))))))))))))",
      "metrics": {
        "early": 4.642396020889282,
        "final": 3.6228970527648925,
        "step_ms": 120.60427665710449,
        "peak_gb": 17.224617984,
        "params": 8029824,
        "improved": true
      }
    },
    {
      "niche": [
        false,
        false,
        true,
        false
      ],
      "reward": 5.121382938896873,
      "formula": "x + gelu(gelu((scale(relu(linear(gelu((scale(relu(linear(x))) \u2299\u03c3 scale(relu(linear(gelu(silu(x)))))))))) \u2299\u03c3 scale(relu(linear(gelu(neg(dwconv_k3(linear(x))))))))))",
      "metrics": {
        "early": 3.988122344017029,
        "final": 3.1123839378356934,
        "step_ms": 56.54621124267578,
        "peak_gb": 11.528669696,
        "params": 5368704,
        "improved": true
      }
    },
    {
      "niche": [
        false,
        true,
        false,
        true
      ],
      "reward": 4.835630094086387,
      "formula": "x + expc(neg(gelu(gelu((scale(relu(linear(sigmoid(mlp4(gelu(linear(gelu(neg(tanh(tanh(scale(relu(linear(gelu(rmsnorm(x)))))))))))))))) \u2299\u03c3 scale(relu(linear(cummean(scale(relu(neg((gelu(glu2(gelu(tanh(scale(relu(linear(gelu(x)))))))) - scale(relu(linear(gelu((scale(relu(x)) \u2299\u03c3 x))))))))))))))))))",
      "metrics": {
        "early": 5.217595434188842,
        "final": 3.9673903226852416,
        "step_ms": 97.13172912597656,
        "peak_gb": 17.459894272,
        "params": 12004224,
        "improved": true
      }
    },
    {
      "niche": [
        false,
        false,
        false,
        false
      ],
      "reward": 4.823786722695091,
      "formula": "x + gelu((scale(relu(gelu(neg(tanh(x))))) \u2299\u03c3 scale(relu(rmsnorm(x)))))",
      "metrics": {
        "early": 4.0697242498397825,
        "final": 3.7892616271972654,
        "step_ms": 42.938947677612305,
        "peak_gb": 9.84554752,
        "params": 3151104,
        "improved": true
      }
    },
    {
      "niche": [
        true,
        false,
        true,
        true
      ],
      "reward": 4.598309086834647,
      "formula": "x + gelu((scale(relu(linear(x))) \u2299\u03c3 scale(relu(linear(gelu(gelu((x \u2299\u03c3 mlp4(attn_h8((x - scale(relu(abs(relu(linear(gelu((dwconv_k3(silu(x)) \u2299\u03c3 scale(relu(linear(gelu(abs(dwconv_k7(rmsnorm(x)))))))))))))))))))))))))",
      "metrics": {
        "early": 3.9827035903930663,
        "final": 3.7270321369171144,
        "step_ms": 92.9560661315918,
        "peak_gb": 15.793202688,
        "params": 10242816,
        "improved": true
      }
    },
    {
      "niche": [
        false,
        true,
        false,
        false
      ],
      "reward": 4.508914195572593,
      "formula": "x + expc((relu(linear(gelu((scale(relu(linear(gelu(neg(gelu(x)))))) \u2299\u03c3 scale(relu(linear(gelu(cummean(scale(relu(linear(gelu((scale(relu(linear(gelu(cummean(gelu(((scale(relu(linear(gelu(gelu((scale(relu(linear(gelu(gelu(x))))) \u2299\u03c3 x)))))) \u2299\u03c3 x) + x))))))) \u2299\u03c3 x)))))))))))))) - x))",
      "metrics": {
        "early": 4.0373770236969,
        "final": 3.7183741331100464,
        "step_ms": 104.02059555053711,
        "peak_gb": 16.417043968,
        "params": 6252288,
        "improved": true
      }
    },
    {
      "niche": [
        false,
        true,
        true,
        true
      ],
      "reward": 4.334455405747153,
      "formula": "x + gelu((scale(relu(linear(gelu((scale(relu(linear(dwconv_k3(relu(gelu(dwconv_k3(relu(linear(gelu(neg(cummean(relu(linear(gelu(linear(gelu(linear(silu(dwconv_k7(gelu(rmsnorm(x)))))))))))))))))))))) \u2299\u03c3 glu2(rmsnorm(x))))))) \u2299\u03c3 scale(relu(linear(gelu(rmsnorm(x)))))))",
      "metrics": {
        "early": 4.589245700836182,
        "final": 3.7722170114517213,
        "step_ms": 82.62109756469727,
        "peak_gb": 14.808715264,
        "params": 8918016,
        "improved": true
      }
    },
    {
      "niche": [
        true,
        true,
        true,
        true
      ],
      "reward": 4.187065421616294,
      "formula": "x + gelu(mlp4(scale(gelu((dwconv_k3(((linear(gelu(scale(dwconv_k3(rmsnorm(dwconv_k7(dwconv_k7(gelu(dwconv_k3(relu(rmsnorm(mlp4(linear(x))))))))))))) \u2299\u03c3 scale(relu(neg(cummean(attn_h8(tanh(attn_h8(tanh(tanh(abs(glu2(glu2(neg(dwconv_k7((rmsnorm(neg(neg(relu(dwconv_k5(attn_h2(scale(x))))))) + x)))))))))))))))) \u2299\u03c3 x)) + x)))))",
      "metrics": {
        "early": 4.385487127304077,
        "final": 3.1699397802352904,
        "step_ms": 174.35574531555176,
        "peak_gb": 22.777659904,
        "params": 21773184,
        "improved": true
      }
    },
    {
      "niche": [
        true,
        false,
        false,
        true
      ],
      "reward": 4.0684950223319305,
      "formula": "x + gelu((gelu((scale(relu(linear(gelu((scale(relu(linear(gelu(attn_h4(x))))) \u2299\u03c3 scale(x)))))) \u2299\u03c3 (scale(relu(linear(gelu(glu2(gelu(x)))))) \u2299\u03c3 scale(relu(linear(gelu((scale(relu(linear(gelu((x \u2299\u03c3 scale(relu(linear(gelu((scale(relu(linear(x))) \u2299\u03c3 x)))))))))) \u2299\u03c3 x)))))))) \u2299\u03c3 x))",
      "metrics": {
        "early": 4.622024250030518,
        "final": 3.7339990377426147,
        "step_ms": 127.40111351013184,
        "peak_gb": 19.977999872,
        "params": 10678272,
        "improved": true
      }
    },
    {
      "niche": [
        true,
        true,
        false,
        false
      ],
      "reward": 3.7166457643859165,
      "formula": "x + cummean(relu(gelu(scale(relu(attn_h4(x))))))",
      "metrics": {
        "early": 5.998728942871094,
        "final": 5.8545290470123295,
        "step_ms": 61.27309799194336,
        "peak_gb": 9.077064704,
        "params": 4919424,
        "improved": true
      }
    }
  ]
}