add mtp layer

.gitattributes

@@ -33,5 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-model.safetensors.index.json filter=lfs diff=lfs merge=lfs -text
 tokenizer.json filter=lfs diff=lfs merge=lfs -text
+model.safetensors.index.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -13,6 +13,7 @@
   ],
   "ep_size": 1,
   "first_k_dense_replace": 3,
+  "head_dim": 64,
   "hidden_act": "silu",
   "hidden_size": 6144,
   "index_head_dim": 128,
@@ -854,7 +855,7 @@
     "quant_method": "quark",
     "quant_mode": "eager_mode",
     "softmax_quant_spec": null,
-    "version": "0.12+bfe174cea91"
+    "version": "0.13+488baccced9"
   },
   "rms_norm_eps": 1e-05,
   "rope_interleave": true,
@@ -867,7 +868,7 @@
   "tie_word_embeddings": false,
   "topk_group": 1,
   "topk_method": "noaux_tc",
-  "transformers_version": "5.4.0",
+  "transformers_version": "5.2.0",
   "use_cache": true,
   "v_head_dim": 256,
   "vocab_size": 154880

generation_config.json

@@ -9,5 +9,5 @@
   "pad_token_id": 154820,
   "temperature": 1.0,
   "top_p": 0.95,
-  "transformers_version": "5.4.0"
+  "transformers_version": "5.2.0"
 }

model-00001-of-00009.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7eb7ff455bc396cb93f3f0006d0b893bca2ab24a488936489c7ed2b9a437b6bc
+oid sha256:d056160e3a2d43d308a2083d9ec18ac40e8bf39bd4cc2d692009b4e5cf749b68
 size 49999051524

model-00002-of-00009.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37b540c98269297f48afb26badef4b29d9d0517206b8daf2543103f46db3ddfd
+oid sha256:a1ac481d97576b9b12ffa708bc2debd34e82245980826e0de77ada48d6cf40df
 size 49998227584

model-00003-of-00009.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8a38862fa733644d473e6b96d9ffbdd2c1938befa71c0eb79fbd6223e0bdec75
+oid sha256:0faf16ca5fb77ebb86d54c7ede11b1278621aa594a3225ad0bd5daea7f319d80
 size 50000184968

model-00004-of-00009.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:acd19130767acadebb47b7c0bb2c5063f17d65dab7a5eebeb584a2b3d9b0f4cf
+oid sha256:9fb3bdacd0b7dbd3226d09d696f46f1200c8e025b37f91d03ed88a67063718df
 size 49998227296

model-00005-of-00009.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f13eda43abc898fa9adb9727b0bb470fe3326b42fd32da59d5338422cfe64d4e
+oid sha256:263454897962bb1d539595be86526636df7cccf64d2010da853ade746b615b35
 size 49916961116

model-00006-of-00009.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d62da9cf4f135006074ad75dd952f9888e42648181855b388873c210d815d51
+oid sha256:14b70f238a1313bfd607fad830c7fe5d6148d00a547be223d8262856d10c4923
 size 50002802316

model-00007-of-00009.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75fc66a1007b11486180c2efd68243c6ab615e21fceb5b971bbcfcc0e279af9a
+oid sha256:f2db7e723a8a2eafd213141eb90cfff7189c3d84b9ad8c25248b31e5e51efe80
 size 49999014384

model-00008-of-00009.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c5fa30f6e744f74c25655958d3c284bd4018b869df00ce13e752e297307fb63
+oid sha256:3ba72559cf58cb0e6bd04af6d4aafcc10487999f42a3e388853f6740c0e6ae4a
 size 50000185032

model-00009-of-00009.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e680692d2efe667ef366badbe661962909ab605e5fb5e0c36620e49fa0a1c11e
-size 42001612804
+oid sha256:fd05cefd4fa774ff4c09a15ae45ec61d5b1b1c2e7032bdcc733f0fd4ae11de75
+size 61907566876

model.safetensors.index.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c7553414afe173015b77b566daae8680f2f1ed8f6840dc79f09e28d436d82770
-size 22332014
+oid sha256:0f23d65b82e78a3c64d765704f0f473a4792760186fbb191ea8bec8c05c65ee3
+size 22403819