diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..9546427ecfa969c490c4b431d743434f82ff0a0a 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
+trainer_state.json filter=lfs diff=lfs merge=lfs -text
diff --git a/config.json b/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..ea55150afe5789c6b87df8d97f268725e26d1393
--- /dev/null
+++ b/config.json
@@ -0,0 +1,65 @@
+{
+ "_name_or_path": "/fs/archive/share/yulan/data/aa_mini/output/miniyulan-2B-final-stage23/checkpoint-223732-rms_norm",
+ "architectures": [
+ "MiniYuLanModelForCausalLM"
+ ],
+ "attention_bias": true,
+ "attention_dropout": 0.0,
+ "bos_token_id": 1,
+ "dim_model_base": 1920,
+ "dim_model_base_attn": 64,
+ "dim_model_base_init": null,
+ "dim_model_base_lmh": 1,
+ "dim_model_base_logits": 1920.0,
+ "dim_model_base_lr": 256.0,
+ "down_proj_alpha": 0.03450327796711771,
+ "embed_tokens_alpha": 1,
+ "embedding_ln": false,
+ "embedding_rmsln": false,
+ "eos_token_id": 2,
+ "gate_up_proj_alpha": 0.3651483716701107,
+ "gradient_checkpointing_step": 11,
+ "hidden_act": "silu",
+ "hidden_size": 1920,
+ "hidden_states_shrink": 0.18708286933869706,
+ "init_scale_o": 1,
+ "initializer_range": 5e-05,
+ "input_layernorm_alpha": 1.0,
+ "intermediate_size": 4800,
+ "k_proj_alpha": 0.3651483716701107,
+ "layer_norm_eps": 1e-06,
+ "lm_head_alpha": 1.0,
+ "ln_scale": 1,
+ "max_position_embeddings": 4096,
+ "model_reproduce": "transformer",
+ "model_type": "miniyulan",
+ "norm_alpha": 1.0,
+ "num_attention_heads": 30,
+ "num_epochs_trained_before_this_epoch": 23,
+ "num_hidden_layers": 56,
+ "num_key_value_heads": 6,
+ "num_steps_trained_before_this_epoch": 223732,
+ "o_proj_alpha": 0.03450327796711771,
+ "post_attention_layernorm_alpha": 1.0,
+ "q_proj_alpha": 0.3651483716701107,
+ "qk_layernorm": false,
+ "rms_norm_eps": 1e-06,
+ "rms_type": "llama",
+ "rope_scaling": null,
+ "rope_theta": 10000.0,
+ "scale_emb": 10.0,
+ "shrink_alpha": 1,
+ "sliding_window": null,
+ "tie_word_embeddings": true,
+ "torch_dtype": "bfloat16",
+ "transformers_version": "4.44.0",
+ "use_cache": false,
+ "use_emb_alpha": true,
+ "use_liger": true,
+ "use_norm_alpha": true,
+ "use_sliding_window": false,
+ "v_proj_alpha": 0.3651483716701107,
+ "vocab_size": 99000,
+ "wesar_weights": true,
+ "z_loss": 0.0001
+}
diff --git a/global_step233470/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e491265284e22b50a4328dabdee572c5b0c44566
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f9e4d155a1a18af71553b262a6def9c890975391c5a147c924ae765a9324735
+size 558554482
diff --git a/global_step233470/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..93032d2284566a4d8cb140e2393cafc683ad6f05
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f42bc2fb958218cda725e7f2b6081d5a834ead1c0923fa6102d868ed5c179ae
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ab4183937e5f62576e483c3b0ed835705c6b8944
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ddb8111ea416ba4d32047d84f64aa5ef9e3b4c1bd7843dbfbd7a1ec4aba136e
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7324e9fdc733804a51bcfc274533cdced5798b55
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:948b43edc99728ce7347f84f69506419f165aadfc8223919f9ca0224ce793e7b
+size 558554370
diff --git a/global_step233470/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..985ccded4a63595e612c0747417af6b0af165e51
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d04db2cc67b5fb4594cad3e8cb17a54e8f3d6d7d97e3cf6385b0ecfc8955384e
+size 558554434
diff --git a/global_step233470/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..77c345bea95ae5a584f634a039b07b4f9e51a8c6
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c17606e18db1b3421798a62e309a9021cfcfb44c225dbea971b745a2adddd6a
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..661e9e3719afd31b929aa969490bb84f0ec5546a
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eda7e7f8c306f8789018c9ac6d69c2ae3ccac653e080b9e54e9181d883f4d759
+size 558554370
diff --git a/global_step233470/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8e73a72e404f0a0770905312cf47e0ea2d2cb82f
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f0fcb087ba15e133ef13fded58d9d72facf32a3880f4ca46c8bd57511d61973
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..46b1ad0e8e3e695c50fdccfe47e69ba546b11ce2
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:178cc4ead6dee7ff2f715425837b20e35e497479b966f23c1f4121325450c7ce
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..427e71ad75d78cb88cfa08bed37094725cd5203f
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:904db425c1086623eae1f9a685e8fbcdc628f3ec6f4723ad4e1a1aa402df203c
+size 558554370
diff --git a/global_step233470/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..12db5b8937f196b40cc125581cbc5f125d2270ea
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b79254d4dcc40f7f80deeca906acd7a5ef03dd60f63f6479cba779a5dc8bd322
+size 558554434
diff --git a/global_step233470/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4fc21ab359b9c660769bf9f769e120f7e12f7fa0
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12327567db2150b9f83c2a05cecdeba4ba7fe110db086a8deb20f62b73326887
+size 558554418
diff --git a/global_step233470/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..eb518edaee86efd9c2c600f754b394b24a3f1cfd
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f220514255f10628cecc928bf3ffd4226f50194e2fa25258a75f13cdfe3a075
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ac48ce7ec99eb17e182e3a851cc50608038f9f62
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:795d4b263bf6f9c47805bc1efb2151b53205d4c0eb026124736b6dd304dec04e
+size 558554370
diff --git a/global_step233470/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ebbb08ff54c59b739c18ca430dd7ee7145316ce3
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:694db15bc04c98306eb482372d862a694331c97c8ea2a7729bac44e7b5c0b80c
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..915b2d1e01e58d94a5c69f9b4fcb764f53cf5643
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55e924060ee8f9737a6b0b641915f9dfcf58f51355871e6d682baf647b8a66d7
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..425c4876fe744f23e0eb91d66ee9c34bcf8b1eda
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d278339f174fc7e7150542733cb921d971d516708013529635b37f2aeb33ba64
+size 558554370
diff --git a/global_step233470/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c886a0e5551db4f9d56248f9a888c38a5064959c
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78fce995cb74c433ed47122a0f8a54bb6bc45120c605008c5013472c33b4f98f
+size 558554434
diff --git a/global_step233470/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f054030e377fd908b426fbd71653726d047282e5
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de8727cdeef03f96c5a092d9a2f1ebaec6b4e57ed5a0bb1276bab1051a514308
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f5eed63dcc0c869e32d238b6c142872eff93e40f
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c12be39bfddb26bea82ee6a6eda9f8041635894a4b4469ff9b64341ff9212192
+size 558554370
diff --git a/global_step233470/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..807bd5313cb331c883daa139f898f4a6bb1a806b
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c9eb28298c702a4f68c1a01c37bab7d20c518d92ef645aa5d32e26e185adb53
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0b7cc2cc73624da3319855c85df720cf0996b95e
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63dc53fe96d14e149abdda9d8174a4164ddc16e9c2c32c4fb960be0b63bc7abd
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d5f6eede794e58596a134b045a5a90989fb248f9
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df8372fc0e54958c897a397e6f71dbe29756395803b9bb052d0038c3f212be5f
+size 558554290
diff --git a/global_step233470/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4d9976efbe1debb9f89eb8814e1d1d1ef4222108
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:348cd27825984c6b1d083d729842c43859b3803c1805755e491e4305049c90a2
+size 558554370
diff --git a/global_step233470/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d999ca849551c3e15bab2b1b162dc9d08e55b02c
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:593806922c35e61766f68229aa926c8680b1b977fe21a345cc59624d1847163c
+size 558554434
diff --git a/global_step233470/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e09389964f328f0c9c4267233f142bed203fb982
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3a2461e6b9d3f554fc2e4f1a99e0408a54e28a7ca6e13260ef85a32fd6b0d5e
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f2cbd07793efaf1c33f3eb357990a68db728ef08
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bddd16c27533f2a72b1aa9407ae2be8e5f4eb7086d74fb990db4e01965dcf07c
+size 558554370
diff --git a/global_step233470/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bb8662f9b1e92d4b42f21770d3d3a0523d66ac50
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:725702e387f1662e3647e9d2227bb04eef6460a1ff4460f758bb7edae7043f1d
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5cc4170a86c59ff848d95ca0127f8e27bf8e5abc
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bbeca52c9ddcc2b9e18be73e85649a724e817ef7710edbb87a4fb585ad80b9f
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c9af242268440cf4c93473120bdc93b51776669e
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf86d6c41e996acf5ac343a23fb2ab22d9238f780434ff44c06d0aee8dca3de7
+size 558554370
diff --git a/global_step233470/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9b09d4d9bad65b1cea6717a20f6fe7269070defc
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dd6fb503286050d643c09f46ad36333b572b5d47c55b1eb630aa8b334fcce8d
+size 558554434
diff --git a/global_step233470/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4e260b9f5da3943c20bed3e3d690b06b3af0b191
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dc6531e4ead894bba2a26048bbd4663fa820edf3b44ae7f27368bff3189a854
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..01b6b96c919dca6cad3af32caf8f02325102e2db
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e1af018a737ca9ddf951a6f06c6637fd2f919fe8a884e3709651bc4e080cb2d
+size 558554370
diff --git a/global_step233470/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a3bb3104bb2a54aa075aaaff2e922a0f1ddcfd6c
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2d314067233e2b6d1607b9cbb3c0d8e6253d147d50f84f6f13cfd323aee8161
+size 558554354
diff --git a/global_step233470/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..927c3eadf61cfb38819fafd0442937505ddf6e3a
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5a3df1708a9a5b564fb98aba6841448705d35721ab29234846121b89b153235
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b783068dbb739e3f937e5c65d0a89a23f320c02d
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dad68828aa68ef18181485e62c9d8b97b14a84e4e53e8047896deab14671677d
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..91228f92acb4ba923a4f4a542f61ba68bb9780b9
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0526cf8e4d92c0876535baff7ce6a4bbf3587d3643d468edf16c4395ebf609eb
+size 558554370
diff --git a/global_step233470/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a0215305077a04125934ae493a4556bea44361d9
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de6840c4841cc417e396f68264e2934e60ec3a39d42c00a496d5b8045bf84b2f
+size 558554434
diff --git a/global_step233470/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d1cab041deca7ddbeabd6dbd06bf7d5714820730
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:009312747af865f81e830c940a5e2722a87a74c34316323eb96fbc7fc5b45498
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..754d8b4b223074e6984395dc0ec694c0ccd7f723
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:625d5e97392c3995c966da01696cc2c0e51ad4f5c8f58cd9fa70bd1053f44567
+size 558554370
diff --git a/global_step233470/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c6a6247ce389dfb56ec38bb55030e4779fbe5a23
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7f5f99e5fe4fed31e20bb5776a79c3895a5d9afbc19944cd3348792bdc66978
+size 558554306
diff --git a/global_step233470/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..389cba873d8ce3fe36ef2d88f90318a21b21bb0f
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d76c9f32f0e324027b0ed398857f242fe5adadd972b795f308ea1316a934b1e
+size 558610626
diff --git a/global_step233470/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5a5011901b11eab0525841c8264b5c62b53abb90
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fc385be4bf6fef5910491bb3cd7da755b909b24cc610ef0d57a3aae0c7f1b49
+size 558554290
diff --git a/global_step233470/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cfa9fdbc71ca9955e14d0f3df33a246a8688c8ad
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef72b98662f649605e10b42b194507cb8343b1b8df6bb1d03ae78ebacd8aa887
+size 558554290
diff --git a/global_step233470/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a769ac0a5b24e9f5b1e00ee1c60f8ac710f7fe20
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00db41147de345173809c80e4ee1313381b7dd0616b7c859905d3cf234ab4537
+size 558554354
diff --git a/global_step233470/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5a1d9a7845ea1b67805af0b26c56e432844930a3
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2acf9af83a256bc5b7725b0df96147df563711841cf2ba67dee8e242f4815a8c
+size 558554418
diff --git a/global_step233470/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..41236f3f4511be12f5d73adfdbe2b988265610ac
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75a824650d476d30b450967461721d385a180c863410ac96e9d256334dd07131
+size 558554290
diff --git a/global_step233470/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/global_step233470/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a5d859bf73043f6527d713c42c9b3875adb91c45
--- /dev/null
+++ b/global_step233470/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:383428f4b5af6299cd862a16638f6cee9b63778f420eb00759cde8436b03d02d
+size 558554354
diff --git a/global_step233470/mp_rank_00_model_states.pt b/global_step233470/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..059f9766a2439a959fae38366b0905f5d70d5fda
--- /dev/null
+++ b/global_step233470/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ef40f4bbab90735cce5d23fe9f797929183c62c4fd22f9ac7f5994605dae052
+size 4468641136
diff --git a/latest b/latest
new file mode 100644
index 0000000000000000000000000000000000000000..e38bd6bbddff9df89ada978c6f43e0af3e1ddd6e
--- /dev/null
+++ b/latest
@@ -0,0 +1 @@
+global_step233470
\ No newline at end of file
diff --git a/model.safetensors b/model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..90e4a4ba94fe35e17c82ad440621d5d1948686a7
--- /dev/null
+++ b/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b342d3389134d9279ee70e8370f83846a7878599ba91ab1615197c0c19ab5ea
+size 4848661852
diff --git a/special_tokens_map.json b/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..a782b2f1cdab4d0bacb2dc0f85d02c4b1e31f0bd
--- /dev/null
+++ b/special_tokens_map.json
@@ -0,0 +1,30 @@
+{
+ "bos_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "eos_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "pad_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "unk_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ }
+}
diff --git a/tokenizer.json b/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..efc4b7ce3fdffa249828bf0f3fe2256cae304a3f
--- /dev/null
+++ b/tokenizer.json
@@ -0,0 +1,226306 @@
+{
+ "version": "1.0",
+ "truncation": null,
+ "padding": null,
+ "added_tokens": [
+ {
+ "id": 0,
+ "content": "",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 1,
+ "content": "",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 2,
+ "content": "",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 102,
+ "content": "",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 103,
+ "content": "",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 104,
+ "content": "<|start_header_id|>",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 105,
+ "content": "<|end_header_id|>",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 106,
+ "content": "<|eot_id|>",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ }
+ ],
+ "normalizer": {
+ "type": "Sequence",
+ "normalizers": [
+ {
+ "type": "Prepend",
+ "prepend": "▁"
+ },
+ {
+ "type": "Replace",
+ "pattern": {
+ "String": " "
+ },
+ "content": "▁"
+ },
+ {
+ "type": "Replace",
+ "pattern": {
+ "String": " "
+ },
+ "content": "▁"
+ }
+ ]
+ },
+ "pre_tokenizer": null,
+ "post_processor": {
+ "type": "TemplateProcessing",
+ "single": [
+ {
+ "SpecialToken": {
+ "id": "",
+ "type_id": 0
+ }
+ },
+ {
+ "Sequence": {
+ "id": "A",
+ "type_id": 0
+ }
+ }
+ ],
+ "pair": [
+ {
+ "SpecialToken": {
+ "id": "",
+ "type_id": 0
+ }
+ },
+ {
+ "Sequence": {
+ "id": "A",
+ "type_id": 0
+ }
+ },
+ {
+ "SpecialToken": {
+ "id": "",
+ "type_id": 1
+ }
+ },
+ {
+ "Sequence": {
+ "id": "B",
+ "type_id": 1
+ }
+ }
+ ],
+ "special_tokens": {
+ "": {
+ "id": "",
+ "ids": [
+ 1
+ ],
+ "tokens": [
+ ""
+ ]
+ }
+ }
+ },
+ "decoder": {
+ "type": "Sequence",
+ "decoders": [
+ {
+ "type": "Replace",
+ "pattern": {
+ "String": "▁"
+ },
+ "content": " "
+ },
+ {
+ "type": "ByteFallback"
+ },
+ {
+ "type": "Fuse"
+ },
+ {
+ "type": "Strip",
+ "content": " ",
+ "start": 1,
+ "stop": 0
+ }
+ ]
+ },
+ "model": {
+ "type": "BPE",
+ "dropout": null,
+ "unk_token": "",
+ "continuing_subword_prefix": null,
+ "end_of_word_suffix": null,
+ "fuse_unk": true,
+ "byte_fallback": true,
+ "ignore_merges": false,
+ "vocab": {
+ "": 0,
+ "": 1,
+ "": 2,
+ "": 3,
+ "": 4,
+ "\n": 5,
+ "\t": 6,
+ "
": 7,
+ "
": 8,
+ "": 9,
+ "": 10,
+ "": 11,
+ "
": 12,
+ "": 13,
+ " | | ": 14,
+ "": 15,
+ "": 16,
+ "": 17,
+ "": 18,
+ "": 21,
+ "": 22,
+ "
": 23,
+ "": 24,
+ "": 25,
+ "": 26,
+ "": 27,
+ "": 28,
+ "": 29,
+ "": 30,
+ "": 31,
+ "": 32,
+ "
": 33,
+ "
": 34,
+ "
": 35,
+ "": 36,
+ "": 37,
+ "": 38,
+ "
": 39,
+ "": 40,
+ "": 41,
+ "
": 42,
+ "": 43,
+ "
": 44,
+ "
": 45,
+ "": 46,
+ "": 47,
+ "
": 48,
+ "": 49,
+ "": 50,
+ "": 51,
+ "0": 52,
+ "1": 53,
+ "2": 54,
+ "3": 55,
+ "4": 56,
+ "5": 57,
+ "6": 58,
+ "7": 59,
+ "8": 60,
+ "9": 61,
+ "+": 62,
+ "-": 63,
+ "=": 64,
+ ",": 65,
+ "。": 66,
+ "!": 67,
+ "?": 68,
+ "、": 69,
+ ":": 70,
+ "¥": 71,
+ ".": 72,
+ "!": 73,
+ "?": 74,
+ "...": 75,
+ "。。。": 76,
+ "。。。。。。": 77,
+ "《": 78,
+ "》": 79,
+ "【": 80,
+ "】": 81,
+ "『": 82,
+ "』": 83,
+ "```": 84,
+ "": 86,
+ "---": 87,
+ "": 88,
+ ";": 89,
+ ".": 90,
+ "=": 91,
+ "<": 92,
+ ">": 93,
+ "-": 94,
+ "+": 95,
+ "%": 96,
+ "‼": 97,
+ "㊣": 98,
+ "/": 99,
+ "|": 100,
+ "": 101,
+ "": 102,
+ "": 103,
+ "<|start_header_id|>": 104,
+ "<|end_header_id|>": 105,
+ "<|eot_id|>": 106,
+ "": 107,
+ "": 108,
+ "": 109,
+ "": 110,
+ "": 111,
+ "": 112,
+ "": 113,
+ "": 114,
+ "": 115,
+ "": 116,
+ "": 117,
+ "": 118,
+ "": 119,
+ "": 120,
+ "": 121,
+ "": 122,
+ "": 123,
+ "": 124,
+ "": 125,
+ "": 126,
+ "": 127,
+ "": 128,
+ "": 129,
+ "": 130,
+ "": 131,
+ "": 132,
+ "": 133,
+ "": 134,
+ "": 135,
+ "": 136,
+ "": 137,
+ "": 138,
+ "": 139,
+ "": 140,
+ "": 141,
+ "": 142,
+ "": 143,
+ "": 144,
+ "": 145,
+ "": 146,
+ "": 147,
+ "": 148,
+ "": 149,
+ "": 150,
+ "": 151,
+ "": 152,
+ "": 153,
+ "": 154,
+ "": 155,
+ "": 156,
+ "": 157,
+ "": 158,
+ "": 159,
+ "": 160,
+ "": 161,
+ "": 162,
+ "": 163,
+ "": 164,
+ "": 165,
+ "": 166,
+ "": 167,
+ "": 168,
+ "": 169,
+ "": 170,
+ "": 171,
+ "": 172,
+ "": 173,
+ "": 174,
+ "": 175,
+ "": 176,
+ "": 177,
+ "": 178,
+ "": 179,
+ "": 180,
+ "": 181,
+ "": 182,
+ "": 183,
+ "": 184,
+ "": 185,
+ "": 186,
+ "": 187,
+ "": 188,
+ "": 189,
+ "": 190,
+ "": 191,
+ "": 192,
+ "": 193,
+ "": 194,
+ "": 195,
+ "": 196,
+ "": 197,
+ "": 198,
+ "": 199,
+ "": 200,
+ "": 201,
+ "": 202,
+ "": 203,
+ "": 204,
+ "": 205,
+ "": 206,
+ "": 207,
+ "": 208,
+ "": 209,
+ "": 210,
+ "": 211,
+ "": 212,
+ "": 213,
+ "": 214,
+ "": 215,
+ "": 216,
+ "": 217,
+ "": 218,
+ "": 219,
+ "": 220,
+ "": 221,
+ "": 222,
+ "": 223,
+ "": 224,
+ "": 225,
+ "": 226,
+ "": 227,
+ "": 228,
+ "": 229,
+ "": 230,
+ "": 231,
+ "": 232,
+ "": 233,
+ "": 234,
+ "": 235,
+ "": 236,
+ "": 237,
+ "": 238,
+ "": 239,
+ "": 240,
+ "": 241,
+ "": 242,
+ "": 243,
+ "": 244,
+ "": 245,
+ "": 246,
+ "": 247,
+ "": 248,
+ "": 249,
+ "": 250,
+ "": 251,
+ "": 252,
+ "": 253,
+ "": 254,
+ "": 255,
+ "": 256,
+ "": 257,
+ "": 258,
+ "": 259,
+ "": 260,
+ "": 261,
+ "": 262,
+ "": 263,
+ "": 264,
+ "": 265,
+ "": 266,
+ "": 267,
+ "": 268,
+ "": 269,
+ "": 270,
+ "": 271,
+ "": 272,
+ "": 273,
+ "": 274,
+ "": 275,
+ "": 276,
+ "": 277,
+ "": 278,
+ "": 279,
+ "": 280,
+ "": 281,
+ "": 282,
+ "": 283,
+ "": 284,
+ "": 285,
+ "": 286,
+ "": 287,
+ "": 288,
+ "": 289,
+ "": 290,
+ "": 291,
+ "": 292,
+ "": 293,
+ "": 294,
+ "": 295,
+ "": 296,
+ "": 297,
+ "": 298,
+ "": 299,
+ "": 300,
+ "": 301,
+ "": 302,
+ "": 303,
+ "": 304,
+ "": 305,
+ "": 306,
+ "": 307,
+ "": 308,
+ "": 309,
+ "": 310,
+ "": 311,
+ "": 312,
+ "": 313,
+ "": 314,
+ "": 315,
+ "": 316,
+ "": 317,
+ "": 318,
+ "": 319,
+ "": 320,
+ "": 321,
+ "": 322,
+ "": 323,
+ "": 324,
+ "": 325,
+ "": 326,
+ "": 327,
+ "": 328,
+ "": 329,
+ "": 330,
+ "": 331,
+ "": 332,
+ "": 333,
+ "": 334,
+ "": 335,
+ "": 336,
+ "": 337,
+ "": 338,
+ "": 339,
+ "": 340,
+ "": 341,
+ "": 342,
+ "": 343,
+ "": 344,
+ "": 345,
+ "": 346,
+ "": 347,
+ "": 348,
+ "": 349,
+ "": 350,
+ "": 351,
+ "": 352,
+ "": 353,
+ "": 354,
+ "": 355,
+ "": 356,
+ "