SlothE-T 25M ternary Zhuyin→zh-TW IME model: ternary GGUF (TQ2_0) + fp32 master + reproducibility toolchain
Browse files- .gitattributes +1 -0
- NAMES.md +286 -0
- README.md +125 -0
- REPRODUCE.md +83 -0
- extract_slothe.py +99 -0
- gate_slothe_ternary.py +99 -0
- model.safetensors +3 -0
- pack_gguf.py +194 -0
- run_ce_ls_32.sh +10 -0
- run_ternary_honest.sh +12 -0
- slothe-t-25m.gguf +3 -0
- slothe.pt +3 -0
- slothe_config.json +17 -0
- syl2legal.npz +3 -0
- syl_vocab.json +1 -0
- train_slothe_ternary.py +585 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
slothe-t-25m.gguf filter=lfs diff=lfs merge=lfs -text
|
NAMES.md
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# slothe GGUF tensor name map
|
| 2 |
+
|
| 3 |
+
Architecture: `slothe` | ternary in-features padded to multiple of 256 (recorded as `slothe.ternary_pad_to`).
|
| 4 |
+
|
| 5 |
+
Ternary linears (blocks 1..14 q/o/k/v/w1/w2/w3) are stored TQ2_0 with the in-features zero-padded to the next multiple of 256 (352->512, 960->1024). Strip padding using the original in-features listed below. All other tensors are F16.
|
| 6 |
+
|
| 7 |
+
| ggml name | original name | role | original shape | stored dtype | padded in-features |
|
| 8 |
+
|---|---|---|---|---|---|
|
| 9 |
+
| `token_embd.weight` | `embed.weight` | fp | (1539, 352) | F16 | |
|
| 10 |
+
| `token_embd_norm.weight` | `embed_norm.w` | fp | (352,) | F16 | |
|
| 11 |
+
| `output_norm.weight` | `norm.w` | fp | (352,) | F16 | |
|
| 12 |
+
| `output.weight` | `head.weight` | fp | (8342, 352) | F16 | |
|
| 13 |
+
| `blk.0.attn_k.weight` | `blocks.0.attn.k.weight` | fp | (88, 352) | F16 | |
|
| 14 |
+
| `blk.0.attn_k_rmsnorm.weight` | `blocks.0.attn.kn.w` | fp | (44,) | F16 | |
|
| 15 |
+
| `blk.0.attn_output.weight` | `blocks.0.attn.o.weight` | fp | (352, 352) | F16 | |
|
| 16 |
+
| `blk.0.attn_q.weight` | `blocks.0.attn.q.weight` | fp | (352, 352) | F16 | |
|
| 17 |
+
| `blk.0.attn_q_rmsnorm.weight` | `blocks.0.attn.qn.w` | fp | (44,) | F16 | |
|
| 18 |
+
| `blk.0.attn_v.weight` | `blocks.0.attn.v.weight` | fp | (88, 352) | F16 | |
|
| 19 |
+
| `blk.0.ffn_gate.weight` | `blocks.0.ffn.w1.weight` | fp | (960, 352) | F16 | |
|
| 20 |
+
| `blk.0.ffn_down.weight` | `blocks.0.ffn.w2.weight` | fp | (352, 960) | F16 | |
|
| 21 |
+
| `blk.0.ffn_up.weight` | `blocks.0.ffn.w3.weight` | fp | (960, 352) | F16 | |
|
| 22 |
+
| `blk.0.attn_norm.weight` | `blocks.0.n1.w` | fp | (352,) | F16 | |
|
| 23 |
+
| `blk.0.ffn_norm.weight` | `blocks.0.n2.w` | fp | (352,) | F16 | |
|
| 24 |
+
| `blk.1.attn_k_subln.weight` | `blocks.1.attn.k.pre.w` | fp | (352,) | F16 | |
|
| 25 |
+
| `blk.1.attn_k.weight` | `blocks.1.attn.k.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 26 |
+
| `blk.1.attn_k_rmsnorm.weight` | `blocks.1.attn.kn.w` | fp | (44,) | F16 | |
|
| 27 |
+
| `blk.1.attn_output_subln.weight` | `blocks.1.attn.o.pre.w` | fp | (352,) | F16 | |
|
| 28 |
+
| `blk.1.attn_output.weight` | `blocks.1.attn.o.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 29 |
+
| `blk.1.attn_q_subln.weight` | `blocks.1.attn.q.pre.w` | fp | (352,) | F16 | |
|
| 30 |
+
| `blk.1.attn_q.weight` | `blocks.1.attn.q.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 31 |
+
| `blk.1.attn_q_rmsnorm.weight` | `blocks.1.attn.qn.w` | fp | (44,) | F16 | |
|
| 32 |
+
| `blk.1.attn_v_subln.weight` | `blocks.1.attn.v.pre.w` | fp | (352,) | F16 | |
|
| 33 |
+
| `blk.1.attn_v.weight` | `blocks.1.attn.v.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 34 |
+
| `blk.1.ffn_gate_subln.weight` | `blocks.1.ffn.w1.pre.w` | fp | (352,) | F16 | |
|
| 35 |
+
| `blk.1.ffn_gate.weight` | `blocks.1.ffn.w1.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 36 |
+
| `blk.1.ffn_down_subln.weight` | `blocks.1.ffn.w2.pre.w` | fp | (960,) | F16 | |
|
| 37 |
+
| `blk.1.ffn_down.weight` | `blocks.1.ffn.w2.weight` | ternary | (352, 960) | TQ2_0 | 1024 |
|
| 38 |
+
| `blk.1.ffn_up_subln.weight` | `blocks.1.ffn.w3.pre.w` | fp | (352,) | F16 | |
|
| 39 |
+
| `blk.1.ffn_up.weight` | `blocks.1.ffn.w3.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 40 |
+
| `blk.1.attn_norm.weight` | `blocks.1.n1.w` | fp | (352,) | F16 | |
|
| 41 |
+
| `blk.1.ffn_norm.weight` | `blocks.1.n2.w` | fp | (352,) | F16 | |
|
| 42 |
+
| `blk.2.attn_k_subln.weight` | `blocks.2.attn.k.pre.w` | fp | (352,) | F16 | |
|
| 43 |
+
| `blk.2.attn_k.weight` | `blocks.2.attn.k.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 44 |
+
| `blk.2.attn_k_rmsnorm.weight` | `blocks.2.attn.kn.w` | fp | (44,) | F16 | |
|
| 45 |
+
| `blk.2.attn_output_subln.weight` | `blocks.2.attn.o.pre.w` | fp | (352,) | F16 | |
|
| 46 |
+
| `blk.2.attn_output.weight` | `blocks.2.attn.o.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 47 |
+
| `blk.2.attn_q_subln.weight` | `blocks.2.attn.q.pre.w` | fp | (352,) | F16 | |
|
| 48 |
+
| `blk.2.attn_q.weight` | `blocks.2.attn.q.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 49 |
+
| `blk.2.attn_q_rmsnorm.weight` | `blocks.2.attn.qn.w` | fp | (44,) | F16 | |
|
| 50 |
+
| `blk.2.attn_v_subln.weight` | `blocks.2.attn.v.pre.w` | fp | (352,) | F16 | |
|
| 51 |
+
| `blk.2.attn_v.weight` | `blocks.2.attn.v.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 52 |
+
| `blk.2.ffn_gate_subln.weight` | `blocks.2.ffn.w1.pre.w` | fp | (352,) | F16 | |
|
| 53 |
+
| `blk.2.ffn_gate.weight` | `blocks.2.ffn.w1.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 54 |
+
| `blk.2.ffn_down_subln.weight` | `blocks.2.ffn.w2.pre.w` | fp | (960,) | F16 | |
|
| 55 |
+
| `blk.2.ffn_down.weight` | `blocks.2.ffn.w2.weight` | ternary | (352, 960) | TQ2_0 | 1024 |
|
| 56 |
+
| `blk.2.ffn_up_subln.weight` | `blocks.2.ffn.w3.pre.w` | fp | (352,) | F16 | |
|
| 57 |
+
| `blk.2.ffn_up.weight` | `blocks.2.ffn.w3.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 58 |
+
| `blk.2.attn_norm.weight` | `blocks.2.n1.w` | fp | (352,) | F16 | |
|
| 59 |
+
| `blk.2.ffn_norm.weight` | `blocks.2.n2.w` | fp | (352,) | F16 | |
|
| 60 |
+
| `blk.3.attn_k_subln.weight` | `blocks.3.attn.k.pre.w` | fp | (352,) | F16 | |
|
| 61 |
+
| `blk.3.attn_k.weight` | `blocks.3.attn.k.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 62 |
+
| `blk.3.attn_k_rmsnorm.weight` | `blocks.3.attn.kn.w` | fp | (44,) | F16 | |
|
| 63 |
+
| `blk.3.attn_output_subln.weight` | `blocks.3.attn.o.pre.w` | fp | (352,) | F16 | |
|
| 64 |
+
| `blk.3.attn_output.weight` | `blocks.3.attn.o.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 65 |
+
| `blk.3.attn_q_subln.weight` | `blocks.3.attn.q.pre.w` | fp | (352,) | F16 | |
|
| 66 |
+
| `blk.3.attn_q.weight` | `blocks.3.attn.q.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 67 |
+
| `blk.3.attn_q_rmsnorm.weight` | `blocks.3.attn.qn.w` | fp | (44,) | F16 | |
|
| 68 |
+
| `blk.3.attn_v_subln.weight` | `blocks.3.attn.v.pre.w` | fp | (352,) | F16 | |
|
| 69 |
+
| `blk.3.attn_v.weight` | `blocks.3.attn.v.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 70 |
+
| `blk.3.ffn_gate_subln.weight` | `blocks.3.ffn.w1.pre.w` | fp | (352,) | F16 | |
|
| 71 |
+
| `blk.3.ffn_gate.weight` | `blocks.3.ffn.w1.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 72 |
+
| `blk.3.ffn_down_subln.weight` | `blocks.3.ffn.w2.pre.w` | fp | (960,) | F16 | |
|
| 73 |
+
| `blk.3.ffn_down.weight` | `blocks.3.ffn.w2.weight` | ternary | (352, 960) | TQ2_0 | 1024 |
|
| 74 |
+
| `blk.3.ffn_up_subln.weight` | `blocks.3.ffn.w3.pre.w` | fp | (352,) | F16 | |
|
| 75 |
+
| `blk.3.ffn_up.weight` | `blocks.3.ffn.w3.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 76 |
+
| `blk.3.attn_norm.weight` | `blocks.3.n1.w` | fp | (352,) | F16 | |
|
| 77 |
+
| `blk.3.ffn_norm.weight` | `blocks.3.n2.w` | fp | (352,) | F16 | |
|
| 78 |
+
| `blk.4.attn_k_subln.weight` | `blocks.4.attn.k.pre.w` | fp | (352,) | F16 | |
|
| 79 |
+
| `blk.4.attn_k.weight` | `blocks.4.attn.k.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 80 |
+
| `blk.4.attn_k_rmsnorm.weight` | `blocks.4.attn.kn.w` | fp | (44,) | F16 | |
|
| 81 |
+
| `blk.4.attn_output_subln.weight` | `blocks.4.attn.o.pre.w` | fp | (352,) | F16 | |
|
| 82 |
+
| `blk.4.attn_output.weight` | `blocks.4.attn.o.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 83 |
+
| `blk.4.attn_q_subln.weight` | `blocks.4.attn.q.pre.w` | fp | (352,) | F16 | |
|
| 84 |
+
| `blk.4.attn_q.weight` | `blocks.4.attn.q.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 85 |
+
| `blk.4.attn_q_rmsnorm.weight` | `blocks.4.attn.qn.w` | fp | (44,) | F16 | |
|
| 86 |
+
| `blk.4.attn_v_subln.weight` | `blocks.4.attn.v.pre.w` | fp | (352,) | F16 | |
|
| 87 |
+
| `blk.4.attn_v.weight` | `blocks.4.attn.v.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 88 |
+
| `blk.4.ffn_gate_subln.weight` | `blocks.4.ffn.w1.pre.w` | fp | (352,) | F16 | |
|
| 89 |
+
| `blk.4.ffn_gate.weight` | `blocks.4.ffn.w1.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 90 |
+
| `blk.4.ffn_down_subln.weight` | `blocks.4.ffn.w2.pre.w` | fp | (960,) | F16 | |
|
| 91 |
+
| `blk.4.ffn_down.weight` | `blocks.4.ffn.w2.weight` | ternary | (352, 960) | TQ2_0 | 1024 |
|
| 92 |
+
| `blk.4.ffn_up_subln.weight` | `blocks.4.ffn.w3.pre.w` | fp | (352,) | F16 | |
|
| 93 |
+
| `blk.4.ffn_up.weight` | `blocks.4.ffn.w3.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 94 |
+
| `blk.4.attn_norm.weight` | `blocks.4.n1.w` | fp | (352,) | F16 | |
|
| 95 |
+
| `blk.4.ffn_norm.weight` | `blocks.4.n2.w` | fp | (352,) | F16 | |
|
| 96 |
+
| `blk.5.attn_k_subln.weight` | `blocks.5.attn.k.pre.w` | fp | (352,) | F16 | |
|
| 97 |
+
| `blk.5.attn_k.weight` | `blocks.5.attn.k.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 98 |
+
| `blk.5.attn_k_rmsnorm.weight` | `blocks.5.attn.kn.w` | fp | (44,) | F16 | |
|
| 99 |
+
| `blk.5.attn_output_subln.weight` | `blocks.5.attn.o.pre.w` | fp | (352,) | F16 | |
|
| 100 |
+
| `blk.5.attn_output.weight` | `blocks.5.attn.o.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 101 |
+
| `blk.5.attn_q_subln.weight` | `blocks.5.attn.q.pre.w` | fp | (352,) | F16 | |
|
| 102 |
+
| `blk.5.attn_q.weight` | `blocks.5.attn.q.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 103 |
+
| `blk.5.attn_q_rmsnorm.weight` | `blocks.5.attn.qn.w` | fp | (44,) | F16 | |
|
| 104 |
+
| `blk.5.attn_v_subln.weight` | `blocks.5.attn.v.pre.w` | fp | (352,) | F16 | |
|
| 105 |
+
| `blk.5.attn_v.weight` | `blocks.5.attn.v.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 106 |
+
| `blk.5.ffn_gate_subln.weight` | `blocks.5.ffn.w1.pre.w` | fp | (352,) | F16 | |
|
| 107 |
+
| `blk.5.ffn_gate.weight` | `blocks.5.ffn.w1.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 108 |
+
| `blk.5.ffn_down_subln.weight` | `blocks.5.ffn.w2.pre.w` | fp | (960,) | F16 | |
|
| 109 |
+
| `blk.5.ffn_down.weight` | `blocks.5.ffn.w2.weight` | ternary | (352, 960) | TQ2_0 | 1024 |
|
| 110 |
+
| `blk.5.ffn_up_subln.weight` | `blocks.5.ffn.w3.pre.w` | fp | (352,) | F16 | |
|
| 111 |
+
| `blk.5.ffn_up.weight` | `blocks.5.ffn.w3.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 112 |
+
| `blk.5.attn_norm.weight` | `blocks.5.n1.w` | fp | (352,) | F16 | |
|
| 113 |
+
| `blk.5.ffn_norm.weight` | `blocks.5.n2.w` | fp | (352,) | F16 | |
|
| 114 |
+
| `blk.6.attn_k_subln.weight` | `blocks.6.attn.k.pre.w` | fp | (352,) | F16 | |
|
| 115 |
+
| `blk.6.attn_k.weight` | `blocks.6.attn.k.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 116 |
+
| `blk.6.attn_k_rmsnorm.weight` | `blocks.6.attn.kn.w` | fp | (44,) | F16 | |
|
| 117 |
+
| `blk.6.attn_output_subln.weight` | `blocks.6.attn.o.pre.w` | fp | (352,) | F16 | |
|
| 118 |
+
| `blk.6.attn_output.weight` | `blocks.6.attn.o.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 119 |
+
| `blk.6.attn_q_subln.weight` | `blocks.6.attn.q.pre.w` | fp | (352,) | F16 | |
|
| 120 |
+
| `blk.6.attn_q.weight` | `blocks.6.attn.q.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 121 |
+
| `blk.6.attn_q_rmsnorm.weight` | `blocks.6.attn.qn.w` | fp | (44,) | F16 | |
|
| 122 |
+
| `blk.6.attn_v_subln.weight` | `blocks.6.attn.v.pre.w` | fp | (352,) | F16 | |
|
| 123 |
+
| `blk.6.attn_v.weight` | `blocks.6.attn.v.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 124 |
+
| `blk.6.ffn_gate_subln.weight` | `blocks.6.ffn.w1.pre.w` | fp | (352,) | F16 | |
|
| 125 |
+
| `blk.6.ffn_gate.weight` | `blocks.6.ffn.w1.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 126 |
+
| `blk.6.ffn_down_subln.weight` | `blocks.6.ffn.w2.pre.w` | fp | (960,) | F16 | |
|
| 127 |
+
| `blk.6.ffn_down.weight` | `blocks.6.ffn.w2.weight` | ternary | (352, 960) | TQ2_0 | 1024 |
|
| 128 |
+
| `blk.6.ffn_up_subln.weight` | `blocks.6.ffn.w3.pre.w` | fp | (352,) | F16 | |
|
| 129 |
+
| `blk.6.ffn_up.weight` | `blocks.6.ffn.w3.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 130 |
+
| `blk.6.attn_norm.weight` | `blocks.6.n1.w` | fp | (352,) | F16 | |
|
| 131 |
+
| `blk.6.ffn_norm.weight` | `blocks.6.n2.w` | fp | (352,) | F16 | |
|
| 132 |
+
| `blk.7.attn_k_subln.weight` | `blocks.7.attn.k.pre.w` | fp | (352,) | F16 | |
|
| 133 |
+
| `blk.7.attn_k.weight` | `blocks.7.attn.k.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 134 |
+
| `blk.7.attn_k_rmsnorm.weight` | `blocks.7.attn.kn.w` | fp | (44,) | F16 | |
|
| 135 |
+
| `blk.7.attn_output_subln.weight` | `blocks.7.attn.o.pre.w` | fp | (352,) | F16 | |
|
| 136 |
+
| `blk.7.attn_output.weight` | `blocks.7.attn.o.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 137 |
+
| `blk.7.attn_q_subln.weight` | `blocks.7.attn.q.pre.w` | fp | (352,) | F16 | |
|
| 138 |
+
| `blk.7.attn_q.weight` | `blocks.7.attn.q.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 139 |
+
| `blk.7.attn_q_rmsnorm.weight` | `blocks.7.attn.qn.w` | fp | (44,) | F16 | |
|
| 140 |
+
| `blk.7.attn_v_subln.weight` | `blocks.7.attn.v.pre.w` | fp | (352,) | F16 | |
|
| 141 |
+
| `blk.7.attn_v.weight` | `blocks.7.attn.v.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 142 |
+
| `blk.7.ffn_gate_subln.weight` | `blocks.7.ffn.w1.pre.w` | fp | (352,) | F16 | |
|
| 143 |
+
| `blk.7.ffn_gate.weight` | `blocks.7.ffn.w1.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 144 |
+
| `blk.7.ffn_down_subln.weight` | `blocks.7.ffn.w2.pre.w` | fp | (960,) | F16 | |
|
| 145 |
+
| `blk.7.ffn_down.weight` | `blocks.7.ffn.w2.weight` | ternary | (352, 960) | TQ2_0 | 1024 |
|
| 146 |
+
| `blk.7.ffn_up_subln.weight` | `blocks.7.ffn.w3.pre.w` | fp | (352,) | F16 | |
|
| 147 |
+
| `blk.7.ffn_up.weight` | `blocks.7.ffn.w3.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 148 |
+
| `blk.7.attn_norm.weight` | `blocks.7.n1.w` | fp | (352,) | F16 | |
|
| 149 |
+
| `blk.7.ffn_norm.weight` | `blocks.7.n2.w` | fp | (352,) | F16 | |
|
| 150 |
+
| `blk.8.attn_k_subln.weight` | `blocks.8.attn.k.pre.w` | fp | (352,) | F16 | |
|
| 151 |
+
| `blk.8.attn_k.weight` | `blocks.8.attn.k.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 152 |
+
| `blk.8.attn_k_rmsnorm.weight` | `blocks.8.attn.kn.w` | fp | (44,) | F16 | |
|
| 153 |
+
| `blk.8.attn_output_subln.weight` | `blocks.8.attn.o.pre.w` | fp | (352,) | F16 | |
|
| 154 |
+
| `blk.8.attn_output.weight` | `blocks.8.attn.o.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 155 |
+
| `blk.8.attn_q_subln.weight` | `blocks.8.attn.q.pre.w` | fp | (352,) | F16 | |
|
| 156 |
+
| `blk.8.attn_q.weight` | `blocks.8.attn.q.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 157 |
+
| `blk.8.attn_q_rmsnorm.weight` | `blocks.8.attn.qn.w` | fp | (44,) | F16 | |
|
| 158 |
+
| `blk.8.attn_v_subln.weight` | `blocks.8.attn.v.pre.w` | fp | (352,) | F16 | |
|
| 159 |
+
| `blk.8.attn_v.weight` | `blocks.8.attn.v.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 160 |
+
| `blk.8.ffn_gate_subln.weight` | `blocks.8.ffn.w1.pre.w` | fp | (352,) | F16 | |
|
| 161 |
+
| `blk.8.ffn_gate.weight` | `blocks.8.ffn.w1.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 162 |
+
| `blk.8.ffn_down_subln.weight` | `blocks.8.ffn.w2.pre.w` | fp | (960,) | F16 | |
|
| 163 |
+
| `blk.8.ffn_down.weight` | `blocks.8.ffn.w2.weight` | ternary | (352, 960) | TQ2_0 | 1024 |
|
| 164 |
+
| `blk.8.ffn_up_subln.weight` | `blocks.8.ffn.w3.pre.w` | fp | (352,) | F16 | |
|
| 165 |
+
| `blk.8.ffn_up.weight` | `blocks.8.ffn.w3.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 166 |
+
| `blk.8.attn_norm.weight` | `blocks.8.n1.w` | fp | (352,) | F16 | |
|
| 167 |
+
| `blk.8.ffn_norm.weight` | `blocks.8.n2.w` | fp | (352,) | F16 | |
|
| 168 |
+
| `blk.9.attn_k_subln.weight` | `blocks.9.attn.k.pre.w` | fp | (352,) | F16 | |
|
| 169 |
+
| `blk.9.attn_k.weight` | `blocks.9.attn.k.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 170 |
+
| `blk.9.attn_k_rmsnorm.weight` | `blocks.9.attn.kn.w` | fp | (44,) | F16 | |
|
| 171 |
+
| `blk.9.attn_output_subln.weight` | `blocks.9.attn.o.pre.w` | fp | (352,) | F16 | |
|
| 172 |
+
| `blk.9.attn_output.weight` | `blocks.9.attn.o.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 173 |
+
| `blk.9.attn_q_subln.weight` | `blocks.9.attn.q.pre.w` | fp | (352,) | F16 | |
|
| 174 |
+
| `blk.9.attn_q.weight` | `blocks.9.attn.q.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 175 |
+
| `blk.9.attn_q_rmsnorm.weight` | `blocks.9.attn.qn.w` | fp | (44,) | F16 | |
|
| 176 |
+
| `blk.9.attn_v_subln.weight` | `blocks.9.attn.v.pre.w` | fp | (352,) | F16 | |
|
| 177 |
+
| `blk.9.attn_v.weight` | `blocks.9.attn.v.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 178 |
+
| `blk.9.ffn_gate_subln.weight` | `blocks.9.ffn.w1.pre.w` | fp | (352,) | F16 | |
|
| 179 |
+
| `blk.9.ffn_gate.weight` | `blocks.9.ffn.w1.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 180 |
+
| `blk.9.ffn_down_subln.weight` | `blocks.9.ffn.w2.pre.w` | fp | (960,) | F16 | |
|
| 181 |
+
| `blk.9.ffn_down.weight` | `blocks.9.ffn.w2.weight` | ternary | (352, 960) | TQ2_0 | 1024 |
|
| 182 |
+
| `blk.9.ffn_up_subln.weight` | `blocks.9.ffn.w3.pre.w` | fp | (352,) | F16 | |
|
| 183 |
+
| `blk.9.ffn_up.weight` | `blocks.9.ffn.w3.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 184 |
+
| `blk.9.attn_norm.weight` | `blocks.9.n1.w` | fp | (352,) | F16 | |
|
| 185 |
+
| `blk.9.ffn_norm.weight` | `blocks.9.n2.w` | fp | (352,) | F16 | |
|
| 186 |
+
| `blk.10.attn_k_subln.weight` | `blocks.10.attn.k.pre.w` | fp | (352,) | F16 | |
|
| 187 |
+
| `blk.10.attn_k.weight` | `blocks.10.attn.k.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 188 |
+
| `blk.10.attn_k_rmsnorm.weight` | `blocks.10.attn.kn.w` | fp | (44,) | F16 | |
|
| 189 |
+
| `blk.10.attn_output_subln.weight` | `blocks.10.attn.o.pre.w` | fp | (352,) | F16 | |
|
| 190 |
+
| `blk.10.attn_output.weight` | `blocks.10.attn.o.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 191 |
+
| `blk.10.attn_q_subln.weight` | `blocks.10.attn.q.pre.w` | fp | (352,) | F16 | |
|
| 192 |
+
| `blk.10.attn_q.weight` | `blocks.10.attn.q.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 193 |
+
| `blk.10.attn_q_rmsnorm.weight` | `blocks.10.attn.qn.w` | fp | (44,) | F16 | |
|
| 194 |
+
| `blk.10.attn_v_subln.weight` | `blocks.10.attn.v.pre.w` | fp | (352,) | F16 | |
|
| 195 |
+
| `blk.10.attn_v.weight` | `blocks.10.attn.v.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 196 |
+
| `blk.10.ffn_gate_subln.weight` | `blocks.10.ffn.w1.pre.w` | fp | (352,) | F16 | |
|
| 197 |
+
| `blk.10.ffn_gate.weight` | `blocks.10.ffn.w1.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 198 |
+
| `blk.10.ffn_down_subln.weight` | `blocks.10.ffn.w2.pre.w` | fp | (960,) | F16 | |
|
| 199 |
+
| `blk.10.ffn_down.weight` | `blocks.10.ffn.w2.weight` | ternary | (352, 960) | TQ2_0 | 1024 |
|
| 200 |
+
| `blk.10.ffn_up_subln.weight` | `blocks.10.ffn.w3.pre.w` | fp | (352,) | F16 | |
|
| 201 |
+
| `blk.10.ffn_up.weight` | `blocks.10.ffn.w3.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 202 |
+
| `blk.10.attn_norm.weight` | `blocks.10.n1.w` | fp | (352,) | F16 | |
|
| 203 |
+
| `blk.10.ffn_norm.weight` | `blocks.10.n2.w` | fp | (352,) | F16 | |
|
| 204 |
+
| `blk.11.attn_k_subln.weight` | `blocks.11.attn.k.pre.w` | fp | (352,) | F16 | |
|
| 205 |
+
| `blk.11.attn_k.weight` | `blocks.11.attn.k.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 206 |
+
| `blk.11.attn_k_rmsnorm.weight` | `blocks.11.attn.kn.w` | fp | (44,) | F16 | |
|
| 207 |
+
| `blk.11.attn_output_subln.weight` | `blocks.11.attn.o.pre.w` | fp | (352,) | F16 | |
|
| 208 |
+
| `blk.11.attn_output.weight` | `blocks.11.attn.o.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 209 |
+
| `blk.11.attn_q_subln.weight` | `blocks.11.attn.q.pre.w` | fp | (352,) | F16 | |
|
| 210 |
+
| `blk.11.attn_q.weight` | `blocks.11.attn.q.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 211 |
+
| `blk.11.attn_q_rmsnorm.weight` | `blocks.11.attn.qn.w` | fp | (44,) | F16 | |
|
| 212 |
+
| `blk.11.attn_v_subln.weight` | `blocks.11.attn.v.pre.w` | fp | (352,) | F16 | |
|
| 213 |
+
| `blk.11.attn_v.weight` | `blocks.11.attn.v.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 214 |
+
| `blk.11.ffn_gate_subln.weight` | `blocks.11.ffn.w1.pre.w` | fp | (352,) | F16 | |
|
| 215 |
+
| `blk.11.ffn_gate.weight` | `blocks.11.ffn.w1.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 216 |
+
| `blk.11.ffn_down_subln.weight` | `blocks.11.ffn.w2.pre.w` | fp | (960,) | F16 | |
|
| 217 |
+
| `blk.11.ffn_down.weight` | `blocks.11.ffn.w2.weight` | ternary | (352, 960) | TQ2_0 | 1024 |
|
| 218 |
+
| `blk.11.ffn_up_subln.weight` | `blocks.11.ffn.w3.pre.w` | fp | (352,) | F16 | |
|
| 219 |
+
| `blk.11.ffn_up.weight` | `blocks.11.ffn.w3.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 220 |
+
| `blk.11.attn_norm.weight` | `blocks.11.n1.w` | fp | (352,) | F16 | |
|
| 221 |
+
| `blk.11.ffn_norm.weight` | `blocks.11.n2.w` | fp | (352,) | F16 | |
|
| 222 |
+
| `blk.12.attn_k_subln.weight` | `blocks.12.attn.k.pre.w` | fp | (352,) | F16 | |
|
| 223 |
+
| `blk.12.attn_k.weight` | `blocks.12.attn.k.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 224 |
+
| `blk.12.attn_k_rmsnorm.weight` | `blocks.12.attn.kn.w` | fp | (44,) | F16 | |
|
| 225 |
+
| `blk.12.attn_output_subln.weight` | `blocks.12.attn.o.pre.w` | fp | (352,) | F16 | |
|
| 226 |
+
| `blk.12.attn_output.weight` | `blocks.12.attn.o.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 227 |
+
| `blk.12.attn_q_subln.weight` | `blocks.12.attn.q.pre.w` | fp | (352,) | F16 | |
|
| 228 |
+
| `blk.12.attn_q.weight` | `blocks.12.attn.q.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 229 |
+
| `blk.12.attn_q_rmsnorm.weight` | `blocks.12.attn.qn.w` | fp | (44,) | F16 | |
|
| 230 |
+
| `blk.12.attn_v_subln.weight` | `blocks.12.attn.v.pre.w` | fp | (352,) | F16 | |
|
| 231 |
+
| `blk.12.attn_v.weight` | `blocks.12.attn.v.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 232 |
+
| `blk.12.ffn_gate_subln.weight` | `blocks.12.ffn.w1.pre.w` | fp | (352,) | F16 | |
|
| 233 |
+
| `blk.12.ffn_gate.weight` | `blocks.12.ffn.w1.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 234 |
+
| `blk.12.ffn_down_subln.weight` | `blocks.12.ffn.w2.pre.w` | fp | (960,) | F16 | |
|
| 235 |
+
| `blk.12.ffn_down.weight` | `blocks.12.ffn.w2.weight` | ternary | (352, 960) | TQ2_0 | 1024 |
|
| 236 |
+
| `blk.12.ffn_up_subln.weight` | `blocks.12.ffn.w3.pre.w` | fp | (352,) | F16 | |
|
| 237 |
+
| `blk.12.ffn_up.weight` | `blocks.12.ffn.w3.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 238 |
+
| `blk.12.attn_norm.weight` | `blocks.12.n1.w` | fp | (352,) | F16 | |
|
| 239 |
+
| `blk.12.ffn_norm.weight` | `blocks.12.n2.w` | fp | (352,) | F16 | |
|
| 240 |
+
| `blk.13.attn_k_subln.weight` | `blocks.13.attn.k.pre.w` | fp | (352,) | F16 | |
|
| 241 |
+
| `blk.13.attn_k.weight` | `blocks.13.attn.k.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 242 |
+
| `blk.13.attn_k_rmsnorm.weight` | `blocks.13.attn.kn.w` | fp | (44,) | F16 | |
|
| 243 |
+
| `blk.13.attn_output_subln.weight` | `blocks.13.attn.o.pre.w` | fp | (352,) | F16 | |
|
| 244 |
+
| `blk.13.attn_output.weight` | `blocks.13.attn.o.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 245 |
+
| `blk.13.attn_q_subln.weight` | `blocks.13.attn.q.pre.w` | fp | (352,) | F16 | |
|
| 246 |
+
| `blk.13.attn_q.weight` | `blocks.13.attn.q.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 247 |
+
| `blk.13.attn_q_rmsnorm.weight` | `blocks.13.attn.qn.w` | fp | (44,) | F16 | |
|
| 248 |
+
| `blk.13.attn_v_subln.weight` | `blocks.13.attn.v.pre.w` | fp | (352,) | F16 | |
|
| 249 |
+
| `blk.13.attn_v.weight` | `blocks.13.attn.v.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 250 |
+
| `blk.13.ffn_gate_subln.weight` | `blocks.13.ffn.w1.pre.w` | fp | (352,) | F16 | |
|
| 251 |
+
| `blk.13.ffn_gate.weight` | `blocks.13.ffn.w1.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 252 |
+
| `blk.13.ffn_down_subln.weight` | `blocks.13.ffn.w2.pre.w` | fp | (960,) | F16 | |
|
| 253 |
+
| `blk.13.ffn_down.weight` | `blocks.13.ffn.w2.weight` | ternary | (352, 960) | TQ2_0 | 1024 |
|
| 254 |
+
| `blk.13.ffn_up_subln.weight` | `blocks.13.ffn.w3.pre.w` | fp | (352,) | F16 | |
|
| 255 |
+
| `blk.13.ffn_up.weight` | `blocks.13.ffn.w3.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 256 |
+
| `blk.13.attn_norm.weight` | `blocks.13.n1.w` | fp | (352,) | F16 | |
|
| 257 |
+
| `blk.13.ffn_norm.weight` | `blocks.13.n2.w` | fp | (352,) | F16 | |
|
| 258 |
+
| `blk.14.attn_k_subln.weight` | `blocks.14.attn.k.pre.w` | fp | (352,) | F16 | |
|
| 259 |
+
| `blk.14.attn_k.weight` | `blocks.14.attn.k.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 260 |
+
| `blk.14.attn_k_rmsnorm.weight` | `blocks.14.attn.kn.w` | fp | (44,) | F16 | |
|
| 261 |
+
| `blk.14.attn_output_subln.weight` | `blocks.14.attn.o.pre.w` | fp | (352,) | F16 | |
|
| 262 |
+
| `blk.14.attn_output.weight` | `blocks.14.attn.o.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 263 |
+
| `blk.14.attn_q_subln.weight` | `blocks.14.attn.q.pre.w` | fp | (352,) | F16 | |
|
| 264 |
+
| `blk.14.attn_q.weight` | `blocks.14.attn.q.weight` | ternary | (352, 352) | TQ2_0 | 512 |
|
| 265 |
+
| `blk.14.attn_q_rmsnorm.weight` | `blocks.14.attn.qn.w` | fp | (44,) | F16 | |
|
| 266 |
+
| `blk.14.attn_v_subln.weight` | `blocks.14.attn.v.pre.w` | fp | (352,) | F16 | |
|
| 267 |
+
| `blk.14.attn_v.weight` | `blocks.14.attn.v.weight` | ternary | (88, 352) | TQ2_0 | 512 |
|
| 268 |
+
| `blk.14.ffn_gate_subln.weight` | `blocks.14.ffn.w1.pre.w` | fp | (352,) | F16 | |
|
| 269 |
+
| `blk.14.ffn_gate.weight` | `blocks.14.ffn.w1.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 270 |
+
| `blk.14.ffn_down_subln.weight` | `blocks.14.ffn.w2.pre.w` | fp | (960,) | F16 | |
|
| 271 |
+
| `blk.14.ffn_down.weight` | `blocks.14.ffn.w2.weight` | ternary | (352, 960) | TQ2_0 | 1024 |
|
| 272 |
+
| `blk.14.ffn_up_subln.weight` | `blocks.14.ffn.w3.pre.w` | fp | (352,) | F16 | |
|
| 273 |
+
| `blk.14.ffn_up.weight` | `blocks.14.ffn.w3.weight` | ternary | (960, 352) | TQ2_0 | 512 |
|
| 274 |
+
| `blk.14.attn_norm.weight` | `blocks.14.n1.w` | fp | (352,) | F16 | |
|
| 275 |
+
| `blk.14.ffn_norm.weight` | `blocks.14.n2.w` | fp | (352,) | F16 | |
|
| 276 |
+
| `blk.15.attn_k.weight` | `blocks.15.attn.k.weight` | fp | (88, 352) | F16 | |
|
| 277 |
+
| `blk.15.attn_k_rmsnorm.weight` | `blocks.15.attn.kn.w` | fp | (44,) | F16 | |
|
| 278 |
+
| `blk.15.attn_output.weight` | `blocks.15.attn.o.weight` | fp | (352, 352) | F16 | |
|
| 279 |
+
| `blk.15.attn_q.weight` | `blocks.15.attn.q.weight` | fp | (352, 352) | F16 | |
|
| 280 |
+
| `blk.15.attn_q_rmsnorm.weight` | `blocks.15.attn.qn.w` | fp | (44,) | F16 | |
|
| 281 |
+
| `blk.15.attn_v.weight` | `blocks.15.attn.v.weight` | fp | (88, 352) | F16 | |
|
| 282 |
+
| `blk.15.ffn_gate.weight` | `blocks.15.ffn.w1.weight` | fp | (960, 352) | F16 | |
|
| 283 |
+
| `blk.15.ffn_down.weight` | `blocks.15.ffn.w2.weight` | fp | (352, 960) | F16 | |
|
| 284 |
+
| `blk.15.ffn_up.weight` | `blocks.15.ffn.w3.weight` | fp | (960, 352) | F16 | |
|
| 285 |
+
| `blk.15.attn_norm.weight` | `blocks.15.n1.w` | fp | (352,) | F16 | |
|
| 286 |
+
| `blk.15.ffn_norm.weight` | `blocks.15.n2.w` | fp | (352,) | F16 | |
|
README.md
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
language:
|
| 4 |
+
- zh
|
| 5 |
+
tags:
|
| 6 |
+
- input-method
|
| 7 |
+
- zhuyin
|
| 8 |
+
- bopomofo
|
| 9 |
+
- traditional-chinese
|
| 10 |
+
- ternary
|
| 11 |
+
- bitnet
|
| 12 |
+
- gguf
|
| 13 |
+
library_name: gguf
|
| 14 |
+
pipeline_tag: text2text-generation
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
# SlothE-T 25M — Ternary Zhuyin → Traditional Chinese IME model
|
| 18 |
+
|
| 19 |
+
The conversion model behind **Slothing (懶音輸入法)**: a libchewing-free, on-device
|
| 20 |
+
LLM-powered 注音 (Bopomofo) input method that does **免選字** (candidate-free)
|
| 21 |
+
whole-sentence conversion — you type the phonetic keystream, it emits the sentence,
|
| 22 |
+
with no candidate list to scroll.
|
| 23 |
+
|
| 24 |
+
- **25M parameters**, ternary weights (W1.58A8), bidirectional encoder.
|
| 25 |
+
- **~7 MB** ternary-packed GGUF (TQ2_0) · 99 MB fp32 master.
|
| 26 |
+
- Runs on-device across four frontends: **fcitx5, IBus, Android, web**.
|
| 27 |
+
|
| 28 |
+
## What it does
|
| 29 |
+
|
| 30 |
+
Given a sequence of Zhuyin **syllables** (the input keystream, e.g. `ㄋㄧˇ ㄏㄠˇ`),
|
| 31 |
+
the model emits the **Traditional Chinese characters** for each position as an
|
| 32 |
+
aligned sequence-labeling task. The output head is **phonetic-legality-masked**:
|
| 33 |
+
at each position only the ~1–50 characters that are legal readings of that
|
| 34 |
+
syllable are scored, out of the full 8342-char vocabulary. This is what replaces
|
| 35 |
+
the traditional IME 選字 (candidate-selection) step.
|
| 36 |
+
|
| 37 |
+
## Architecture
|
| 38 |
+
|
| 39 |
+
| | |
|
| 40 |
+
|---|---|
|
| 41 |
+
| encoder | bidirectional (BERT-like), 16 layers |
|
| 42 |
+
| dim / ffn | 352 / 960 |
|
| 43 |
+
| heads | 8 query, 2 KV (GQA), head_dim 44, QK-norm |
|
| 44 |
+
| norm | RMSNorm, embed-norm, **SubLN** pre-norm before each ternary linear |
|
| 45 |
+
| quantization | ternary weights {−1, 0, +1} × per-output-channel **absmedian** scale; **int8** activations; QAT via STE |
|
| 46 |
+
| fp islands | boundary blocks (0 and 15) kept fp16 for stability (`fp_boundary=1`) |
|
| 47 |
+
| vocab | 1539 syllables (in) · 8342 characters (out) |
|
| 48 |
+
|
| 49 |
+
## Evaluation — honest held-out
|
| 50 |
+
|
| 51 |
+
Measured on **500 fresh zh-TW sentences** (C4-zh-TW, offset far past the training
|
| 52 |
+
window, filtered to be **absent from the training corpus**), then g2pW-labeled.
|
| 53 |
+
|
| 54 |
+
| metric | **this model** | 12M int8 (previous ship) | 32M fp teacher |
|
| 55 |
+
|---|---|---|---|
|
| 56 |
+
| **免選字** (whole-sentence exact) | **76 %** | 72 % | — |
|
| 57 |
+
| **homophone-hard** | **86 %** | 82 % | 83 % |
|
| 58 |
+
| **toneless** | **77 %** | 79 % | 81 % |
|
| 59 |
+
|
| 60 |
+
> **Note on 免選字.** Earlier project numbers (~84 %) were inflated by a benchmark
|
| 61 |
+
> leak — the reference set had been sampled from the training corpus, rewarding
|
| 62 |
+
> memorization. **76 %** is the honest whole-sentence held-out figure.
|
| 63 |
+
> *homophone-hard* and *toneless* are leak-clean throughout. On honest data this
|
| 64 |
+
> 25M ternary model **beats the previously-shipped 12M int8 on 免選字 and
|
| 65 |
+
> homophone-hard** while being smaller and faster.
|
| 66 |
+
|
| 67 |
+
## Training recipe (teacher-free)
|
| 68 |
+
|
| 69 |
+
- **Direct cross-entropy** on a g2pW-labeled zh-TW corpus, with **label smoothing 0.1**.
|
| 70 |
+
- **Long OneCycleLR schedule (32 epochs), early-stopped at the peak (epoch 24)** —
|
| 71 |
+
the model overfits after (73 % 免選字 by ep32).
|
| 72 |
+
- **8-adjacency keyboard-error noise** (TAAI-2024 error model): simulated mis-keys
|
| 73 |
+
are constrained to the QWERTY physical 8-neighbourhood rather than any
|
| 74 |
+
edit-distance-1 syllable.
|
| 75 |
+
- DDP on 2× RTX 5090.
|
| 76 |
+
|
| 77 |
+
Knowledge distillation from a 32M teacher was tried and only **matched** (never
|
| 78 |
+
beat) this teacher-free recipe, so the teacher was dropped — simpler and ~2.4×
|
| 79 |
+
faster to train. The two levers that carry the quality are (a) label smoothing
|
| 80 |
+
(a teacher-free regularizer; +5 免選字 over plain CE) and (b) enough epochs to
|
| 81 |
+
exploit the no-overfit headroom.
|
| 82 |
+
|
| 83 |
+
## Deployment / inference
|
| 84 |
+
|
| 85 |
+
The ternary weights are shipped as a **GGUF** using ggml's mainline **TQ2_0**
|
| 86 |
+
ternary type, so the model runs on stock llama.cpp CPU kernels (ARM NEON, x86
|
| 87 |
+
AVX2, and the generic/WASM path). Measured `vec_dot` throughput on x86 AVX2:
|
| 88 |
+
**TQ2_0 ≈ 2.3× int8 (Q8_0)** — the fastest quant type in the table — and ~2× on
|
| 89 |
+
a no-dotprod ARM (Snapdragon 662). The ternary win is partly memory-bandwidth
|
| 90 |
+
(2 bits/weight = 4× fewer weight bytes), so it holds on both weak and
|
| 91 |
+
accelerator-equipped CPUs.
|
| 92 |
+
|
| 93 |
+
> **256-alignment note.** TQ2_0 uses 256-element blocks, so the model's
|
| 94 |
+
> in-features (352, 960) are zero-padded up to the next multiple of 256 in the
|
| 95 |
+
> GGUF (352 → 512, 960 → 1024). The padding is exact ternary-zero (loss-free) but
|
| 96 |
+
> costs some MACs; a future model with 256-aligned dims removes the tax.
|
| 97 |
+
|
| 98 |
+
## Files
|
| 99 |
+
|
| 100 |
+
| file | what |
|
| 101 |
+
|---|---|
|
| 102 |
+
| `slothe-t-25m.gguf` | ternary GGUF — TQ2_0 blocks (layers 1–14) + fp16 islands/embed/head |
|
| 103 |
+
| `model.safetensors` | fp32 master weights (HF-native, non-pickle) |
|
| 104 |
+
| `slothe.pt` | fp32 master checkpoint with embedded `config` |
|
| 105 |
+
| `syl_vocab.json` | syllable tokenizer (input vocab) |
|
| 106 |
+
| `syl2legal.npz` | phonetic legality mask, 1539 syllables × 8342 chars (bool) |
|
| 107 |
+
| `train_slothe_ternary.py` | training script |
|
| 108 |
+
| `gate_slothe_ternary.py` | evaluation / gating script |
|
| 109 |
+
| `REPRODUCE.md` | end-to-end reproduction recipe |
|
| 110 |
+
| `NAMES.md` | GGUF tensor-name ↔ checkpoint-name map |
|
| 111 |
+
|
| 112 |
+
## Intended use & limits
|
| 113 |
+
|
| 114 |
+
- **Intended:** on-device Traditional-Chinese Bopomofo input (Taiwan readings).
|
| 115 |
+
- **Out of scope:** Simplified Chinese, mainland pinyin readings, free-form
|
| 116 |
+
generation. The model only scores per-position legal characters — it is a
|
| 117 |
+
converter, not a chat model.
|
| 118 |
+
- The whole-sentence (免選字) metric is honest-held-out and modest (76 %) by
|
| 119 |
+
design: it is measured on unseen sentences, not the training distribution.
|
| 120 |
+
|
| 121 |
+
## Citation / related work
|
| 122 |
+
|
| 123 |
+
The keyboard-error model and the 免選字-vs-選字 framing are discussed in the
|
| 124 |
+
project's `docs/RELATED-WORK.md`, which positions this work against the TAAI-2024
|
| 125 |
+
cross-multi-IME system (李偉安, 葉展維, 張嘉惠, National Central University).
|
REPRODUCE.md
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Reproducing SlothE-T 25M
|
| 2 |
+
|
| 3 |
+
End-to-end recipe for the ternary Zhuyin→Traditional-Chinese model
|
| 4 |
+
`slothe_t_25m_ce_ls32_ep24` and its GGUF. Trained on 2× RTX 5090 (DDP).
|
| 5 |
+
|
| 6 |
+
## 0. Inputs
|
| 7 |
+
|
| 8 |
+
| artifact | role |
|
| 9 |
+
|---|---|
|
| 10 |
+
| `train_e_g2pw.bin` | packed training set: zh-TW sentences, g2pW-labeled (syllable→char aligned) |
|
| 11 |
+
| `syl_vocab.json` | 1539-entry syllable (input) vocab |
|
| 12 |
+
| `tokenizer/` | char tokenizer (8342 chars) |
|
| 13 |
+
| `phonetic_table.tsv` | syllable→legal-character table (Taiwan readings) |
|
| 14 |
+
| `syl2legal.npz` | the same table as a dense `[1539 × 8342]` bool mask used by the legality-masked head |
|
| 15 |
+
|
| 16 |
+
Data prep: raw zh-TW corpus → g2pW phonetic labeling → aligned `(syllable, char)`
|
| 17 |
+
pairs packed into `train_e_g2pw.bin`. The held-out eval sets
|
| 18 |
+
(`eval/reference_heldout.tsv`, `eval/testset.tsv`) are **filtered to exclude any
|
| 19 |
+
sentence present in the training corpus** — this is what makes the reported
|
| 20 |
+
numbers honest (see the leakage note in the model card).
|
| 21 |
+
|
| 22 |
+
## 1. Train (teacher-free CE + label smoothing, long schedule)
|
| 23 |
+
|
| 24 |
+
`run_ce_ls_32.sh`:
|
| 25 |
+
|
| 26 |
+
```bash
|
| 27 |
+
python3 -m torch.distributed.run --nproc_per_node=2 train_slothe_ternary.py \
|
| 28 |
+
--data train_e_g2pw.bin --vocab syl_vocab.json --tokenizer tokenizer \
|
| 29 |
+
--out slothe_t_25m_ce_ls32 \
|
| 30 |
+
--dim 352 --depth 16 --heads 8 --kv-heads 2 --ffn 960 --embed-norm \
|
| 31 |
+
--quant ternary --weight-quant median --pre-norm \
|
| 32 |
+
--label-smoothing 0.1 \
|
| 33 |
+
--batch 384 --epochs 32 --save-every 4 --lr 2.5e-3
|
| 34 |
+
```
|
| 35 |
+
|
| 36 |
+
- `--quant ternary --weight-quant median` → W1.58A8 QAT: ternary weights
|
| 37 |
+
{−1,0,+1} × per-output-channel **absmedian** scale, int8 activations, STE.
|
| 38 |
+
- `--pre-norm` → SubLN RMSNorm before each ternary linear (stability).
|
| 39 |
+
- boundary blocks stay fp16 (`fp_boundary=1`, the default).
|
| 40 |
+
- `--save-every 4` snapshots every 4 epochs → `slothe_t_25m_ce_ls32_ep{4,8,…,32}`.
|
| 41 |
+
- **No `--teacher`** — teacher-free. Distillation was tried and only matched this.
|
| 42 |
+
- effective batch = 384 × 2 GPUs = 768.
|
| 43 |
+
|
| 44 |
+
## 2. Select the peak epoch (early stopping on held-out)
|
| 45 |
+
|
| 46 |
+
`gate_cels32_snap.sh` gates each snapshot on the held-out sets and prints the
|
| 47 |
+
curve. The model **peaks at epoch 24** and overfits after:
|
| 48 |
+
|
| 49 |
+
```bash
|
| 50 |
+
python3 gate_slothe_ternary.py --model slothe_t_25m_ce_ls32_ep24 \
|
| 51 |
+
--tokenizer tokenizer --table phonetic_table.tsv \
|
| 52 |
+
--testset ../eval/testset.tsv --mspy ../eval/reference_heldout.tsv
|
| 53 |
+
```
|
| 54 |
+
|
| 55 |
+
Expected held-out: **免選字 76 % · homophone-hard 86 % · toneless 77 %**
|
| 56 |
+
(ep32 regresses to ~73 % 免選字 — take **ep24**).
|
| 57 |
+
|
| 58 |
+
## 3. Convert to ternary GGUF
|
| 59 |
+
|
| 60 |
+
Two steps (torch only needed for extraction):
|
| 61 |
+
|
| 62 |
+
```bash
|
| 63 |
+
# a) extract effective ternary weights + fp tensors from the checkpoint (needs torch)
|
| 64 |
+
python3 extract_slothe.py slothe_t_25m_ce_ls32_ep24/slothe.pt \
|
| 65 |
+
-> slothe_tensors.npz + slothe_config.json + roles.json
|
| 66 |
+
|
| 67 |
+
# b) pack GGUF (numpy + gguf-py only): ternary linears -> TQ2_0 (256-padded),
|
| 68 |
+
# fp tensors -> f16, custom "slothe" arch metadata + syllable vocab
|
| 69 |
+
python3 pack_gguf.py -> slothe-t-25m.gguf
|
| 70 |
+
```
|
| 71 |
+
|
| 72 |
+
The ternarization baked into the GGUF is exactly the trainer's inference-time
|
| 73 |
+
quant at `quant_alpha=1.0` (fully annealed): `code = round(clamp(w/scale, −1, 1))`,
|
| 74 |
+
`scale = median(|w|)` per output channel. Because the effective weights are exact
|
| 75 |
+
ternary multiples, requantizing them to TQ2_0 is **loss-free** (verified by
|
| 76 |
+
round-trip: `max|dequant − effective| < 1e-3`). See `NAMES.md` for the
|
| 77 |
+
GGUF-tensor ↔ checkpoint-tensor name map and the 256-padding of in-features.
|
| 78 |
+
|
| 79 |
+
## Environment
|
| 80 |
+
|
| 81 |
+
- PyTorch (CUDA) for training/extraction; `numpy` + `gguf` (gguf-py) for packing.
|
| 82 |
+
- 2× RTX 5090 for the DDP recipe above; a single GPU works with `--nproc_per_node=1`
|
| 83 |
+
(halve the effective batch or double `--batch`).
|
extract_slothe.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Step 1 - extraction (run ON ai-workstation, needs torch).
|
| 3 |
+
|
| 4 |
+
Loads the ternary IME checkpoint and emits plain-numpy artifacts:
|
| 5 |
+
- slothe_tensors.npz : per-tensor fp32 arrays.
|
| 6 |
+
* ternary linears (blocks 1..14, q/o/k/v/w1/w2/w3): store `eff` (effective fp32 = code*scale)
|
| 7 |
+
* everything else: raw fp32
|
| 8 |
+
keys = original checkpoint tensor names, minus the *.quant_alpha scalars (dropped)
|
| 9 |
+
- slothe_config.json : the config dict
|
| 10 |
+
- roles.json : tensor name -> "ternary" | "fp"
|
| 11 |
+
- model.safetensors : RAW fp32 state_dict (all tensors as-is), minus *.quant_alpha (HF-native master)
|
| 12 |
+
"""
|
| 13 |
+
import json, sys
|
| 14 |
+
import numpy as np
|
| 15 |
+
import torch
|
| 16 |
+
from safetensors.torch import save_file
|
| 17 |
+
|
| 18 |
+
CKPT = "/home/luigi/sloth-zhuyin-linux/model/slothe_t_25m_ce_ls32_ep24/slothe.pt"
|
| 19 |
+
OUT_NPZ = "/home/luigi/slothe_tensors.npz"
|
| 20 |
+
OUT_CFG = "/home/luigi/slothe_config.json"
|
| 21 |
+
OUT_ROLES = "/home/luigi/roles.json"
|
| 22 |
+
OUT_ST = "/home/luigi/model.safetensors"
|
| 23 |
+
|
| 24 |
+
FP_BOUNDARY = 1
|
| 25 |
+
DEPTH = 16
|
| 26 |
+
# the 7 linear leaf names inside a block that carry weights
|
| 27 |
+
LINEAR_SUFFIXES = ["attn.q", "attn.o", "attn.k", "attn.v", "ffn.w1", "ffn.w2", "ffn.w3"]
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def is_ternary_linear(name):
|
| 31 |
+
"""True iff `name` is a `blocks.i.<suffix>.weight` on a ternary block (1..DEPTH-2)."""
|
| 32 |
+
if not name.startswith("blocks.") or not name.endswith(".weight"):
|
| 33 |
+
return False
|
| 34 |
+
parts = name.split(".")
|
| 35 |
+
try:
|
| 36 |
+
i = int(parts[1])
|
| 37 |
+
except ValueError:
|
| 38 |
+
return False
|
| 39 |
+
if i < FP_BOUNDARY or i >= DEPTH - FP_BOUNDARY:
|
| 40 |
+
return False # fp island block (0 and 15)
|
| 41 |
+
core = ".".join(parts[2:-1]) # strip 'blocks.i.' prefix and '.weight' suffix
|
| 42 |
+
return core in LINEAR_SUFFIXES
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def ternarize_eff(w):
|
| 46 |
+
"""Effective fp32 weight per verified trainer math (quant_alpha == 1.0)."""
|
| 47 |
+
scale = w.abs().median(dim=1, keepdim=True).values.clamp(min=1e-5) # [out,1] per output channel
|
| 48 |
+
code = (w / scale).round().clamp(-1, 1) # {-1,0,+1}
|
| 49 |
+
eff = code * scale
|
| 50 |
+
return eff, code, scale
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def main():
|
| 54 |
+
ck = torch.load(CKPT, map_location="cpu", weights_only=False)
|
| 55 |
+
sd = ck["model"]
|
| 56 |
+
config = ck["config"]
|
| 57 |
+
|
| 58 |
+
npz = {}
|
| 59 |
+
roles = {}
|
| 60 |
+
st_tensors = {} # raw fp32 master
|
| 61 |
+
sanity_done = False
|
| 62 |
+
|
| 63 |
+
for name, t in sd.items():
|
| 64 |
+
if name.endswith(".quant_alpha"):
|
| 65 |
+
continue # drop the (all-1.0) quant scalars
|
| 66 |
+
t = t.detach().to(torch.float32)
|
| 67 |
+
# raw master (safetensors) keeps every non-quant_alpha tensor as-is
|
| 68 |
+
st_tensors[name] = t.contiguous().clone()
|
| 69 |
+
|
| 70 |
+
if is_ternary_linear(name):
|
| 71 |
+
eff, code, scale = ternarize_eff(t)
|
| 72 |
+
npz[name] = eff.numpy().astype(np.float32)
|
| 73 |
+
roles[name] = "ternary"
|
| 74 |
+
if not sanity_done:
|
| 75 |
+
uniq = set(int(x) for x in torch.unique(code).tolist())
|
| 76 |
+
assert uniq.issubset({-1, 0, 1}), f"code has bad values: {uniq}"
|
| 77 |
+
zero_frac = float((code == 0).float().mean())
|
| 78 |
+
print(f"[sanity] {name}: code.unique()={sorted(uniq)} subset of {{-1,0,1}} OK; "
|
| 79 |
+
f"zero-fraction={zero_frac:.4f} (expected ~0.27)")
|
| 80 |
+
sanity_done = True
|
| 81 |
+
else:
|
| 82 |
+
npz[name] = t.numpy().astype(np.float32)
|
| 83 |
+
roles[name] = "fp"
|
| 84 |
+
|
| 85 |
+
np.savez(OUT_NPZ, **npz)
|
| 86 |
+
with open(OUT_CFG, "w") as f:
|
| 87 |
+
json.dump(config, f, indent=2, ensure_ascii=False)
|
| 88 |
+
with open(OUT_ROLES, "w") as f:
|
| 89 |
+
json.dump(roles, f, indent=2, ensure_ascii=False)
|
| 90 |
+
save_file(st_tensors, OUT_ST)
|
| 91 |
+
|
| 92 |
+
n_tern = sum(1 for v in roles.values() if v == "ternary")
|
| 93 |
+
n_fp = sum(1 for v in roles.values() if v == "fp")
|
| 94 |
+
print(f"[done] stored {len(npz)} tensors: {n_tern} ternary, {n_fp} fp")
|
| 95 |
+
print(f"[done] npz={OUT_NPZ} cfg={OUT_CFG} roles={OUT_ROLES} safetensors={OUT_ST}")
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
if __name__ == "__main__":
|
| 99 |
+
main()
|
gate_slothe_ternary.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Accuracy gate for the ternary SlothLM-E — mirrors reproduce/gate_g2pw.py but
|
| 4 |
+
rebuilds the model from the ternary `config` and evaluates at full ternary
|
| 5 |
+
(quant_alpha=1). Reports the same three numbers so you can compare directly to
|
| 6 |
+
the 12M int8 baseline (免選字 84% / homophone ~84% / toneless 70%).
|
| 7 |
+
|
| 8 |
+
python3 gate_slothe_ternary.py --model slothe_t_20m \
|
| 9 |
+
--tokenizer tokenizer --table phonetic_table.tsv
|
| 10 |
+
"""
|
| 11 |
+
import argparse, json, os, sys
|
| 12 |
+
import torch
|
| 13 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 14 |
+
from train_slothe_ternary import SlothE_T
|
| 15 |
+
|
| 16 |
+
TONES = "ˊˇˋ˙"
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def build(ck):
|
| 20 |
+
c = ck["config"]
|
| 21 |
+
m = SlothE_T(c["n_syl"], c["n_char"], c["dim"], c["depth"], c["heads"], c["kv"],
|
| 22 |
+
c["ffn"], embed_norm=c.get("embed_norm", False),
|
| 23 |
+
weight_bits=c.get("weight_bits", "ternary"),
|
| 24 |
+
weight_quant=c.get("weight_quant", "median"),
|
| 25 |
+
fp_boundary=c.get("fp_boundary", 1),
|
| 26 |
+
pre_norm=c.get("pre_norm", True), act_bits=c.get("act_bits", 8), char_hints=c.get("char_hints", False), tie_hints=c.get("tie_hints", False))
|
| 27 |
+
m.load_state_dict(ck["model"])
|
| 28 |
+
m.set_quant_alpha(1.0) # evaluate the deployed (fully ternary) model
|
| 29 |
+
m.eval()
|
| 30 |
+
return m
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def main():
|
| 34 |
+
ap = argparse.ArgumentParser()
|
| 35 |
+
ap.add_argument("--model", required=True)
|
| 36 |
+
ap.add_argument("--tokenizer", default="tokenizer")
|
| 37 |
+
ap.add_argument("--table", default="phonetic_table.tsv")
|
| 38 |
+
ap.add_argument("--testset", default="eval/testset.tsv")
|
| 39 |
+
ap.add_argument("--mspy", default="eval/reference_mspy.tsv")
|
| 40 |
+
args = ap.parse_args()
|
| 41 |
+
|
| 42 |
+
from transformers import AutoTokenizer
|
| 43 |
+
tok = AutoTokenizer.from_pretrained(args.tokenizer)
|
| 44 |
+
ck = torch.load(os.path.join(args.model, "slothe.pt"), map_location="cpu")
|
| 45 |
+
m = build(ck)
|
| 46 |
+
sv = json.load(open(os.path.join(args.model, "syl_vocab.json"), encoding="utf-8"))
|
| 47 |
+
|
| 48 |
+
tonal = {}
|
| 49 |
+
for line in open(args.table, encoding="utf-8"):
|
| 50 |
+
s, _, r = line.rstrip("\n").partition("\t")
|
| 51 |
+
if r:
|
| 52 |
+
tonal[s] = list(r)
|
| 53 |
+
|
| 54 |
+
def cid(sy):
|
| 55 |
+
ht = any(x in TONES for x in sy)
|
| 56 |
+
ch = tonal.get(sy) if ht else None
|
| 57 |
+
if ch is None:
|
| 58 |
+
b = "".join(x for x in sy if x not in TONES); ch = []
|
| 59 |
+
for k, v in tonal.items():
|
| 60 |
+
if "".join(x for x in k if x not in TONES) == b:
|
| 61 |
+
for x in v:
|
| 62 |
+
if x not in ch:
|
| 63 |
+
ch.append(x)
|
| 64 |
+
return [tok.convert_tokens_to_ids(x) for x in (ch or [sy])
|
| 65 |
+
if tok.convert_tokens_to_ids(x) is not None]
|
| 66 |
+
|
| 67 |
+
def dec(sy, toneless=False):
|
| 68 |
+
if toneless:
|
| 69 |
+
sy = ["".join(x for x in y if x not in TONES) for y in sy]
|
| 70 |
+
sid = torch.tensor([[sv.get(y, 1) for y in sy]])
|
| 71 |
+
am = torch.ones_like(sid, dtype=torch.bool)
|
| 72 |
+
with torch.no_grad():
|
| 73 |
+
lg = m(sid, am)[0].numpy()
|
| 74 |
+
out = []
|
| 75 |
+
for i, s in enumerate(sy):
|
| 76 |
+
ids = cid(s)
|
| 77 |
+
out.append(tok.convert_ids_to_tokens(max(ids, key=lambda t: lg[i, t]))
|
| 78 |
+
if ids else "?")
|
| 79 |
+
return "".join(out)
|
| 80 |
+
|
| 81 |
+
def run(path, name, toneless=False):
|
| 82 |
+
n = ok = 0
|
| 83 |
+
for line in open(path, encoding="utf-8"):
|
| 84 |
+
line = line.rstrip("\n")
|
| 85 |
+
if not line or line.startswith("#"):
|
| 86 |
+
continue
|
| 87 |
+
bopo, exp = line.split("\t")[:2]
|
| 88 |
+
n += 1; ok += dec(bopo.split(), toneless) == exp
|
| 89 |
+
print(f" {name}: {ok}/{n} ({100*ok/n:.0f}%)")
|
| 90 |
+
|
| 91 |
+
npar = sum(p.numel() for p in m.parameters())
|
| 92 |
+
print(f"=== {os.path.basename(args.model)} ({npar/1e6:.0f}M, ternary) ===")
|
| 93 |
+
run(args.testset, "homophone-hard (12M int8: 84%)")
|
| 94 |
+
run(args.mspy, "免選字 230 (12M int8: 84%)")
|
| 95 |
+
run(args.testset, "toneless (12M int8: 70%)", toneless=True)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
if __name__ == "__main__":
|
| 99 |
+
main()
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5c6f7747bc403e360323f546857fa264f8f8279fbd59481f3a4af07da03a335f
|
| 3 |
+
size 98868168
|
pack_gguf.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Step 2 - GGUF packing (run LOCALLY, numpy + gguf-py only).
|
| 3 |
+
|
| 4 |
+
Reads slothe_tensors.npz / roles.json / slothe_config.json / syl_vocab.json and
|
| 5 |
+
writes slothe-t-25m.gguf:
|
| 6 |
+
- fp tensors -> F16
|
| 7 |
+
- ternary linears -> in-features padded to next multiple of 256, quantized to TQ2_0 (lossless on exact-ternary)
|
| 8 |
+
"""
|
| 9 |
+
import json, os, sys
|
| 10 |
+
|
| 11 |
+
GGUF_PY = "/tmp/slothing.cpp/gguf-py"
|
| 12 |
+
sys.path.insert(0, GGUF_PY)
|
| 13 |
+
|
| 14 |
+
import numpy as np
|
| 15 |
+
import gguf
|
| 16 |
+
from gguf import GGUFWriter, GGUFReader, quants
|
| 17 |
+
from gguf.constants import GGMLQuantizationType
|
| 18 |
+
|
| 19 |
+
HF = "/home/luigi/sloth-zhuyin-linux/model/hf_release"
|
| 20 |
+
NPZ = os.path.join(HF, "slothe_tensors.npz")
|
| 21 |
+
ROLES = os.path.join(HF, "roles.json")
|
| 22 |
+
CFG = os.path.join(HF, "slothe_config.json")
|
| 23 |
+
SYL = os.path.join(HF, "syl_vocab.json")
|
| 24 |
+
OUT = os.path.join(HF, "slothe-t-25m.gguf")
|
| 25 |
+
NAMES_MD = os.path.join(HF, "NAMES.md")
|
| 26 |
+
|
| 27 |
+
PAD_TO = 256
|
| 28 |
+
ARCH = "slothe"
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def next_mult(n, m):
|
| 32 |
+
return ((n + m - 1) // m) * m
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def ggml_name(name):
|
| 36 |
+
"""Map original checkpoint tensor name -> ggml tensor name."""
|
| 37 |
+
top = {
|
| 38 |
+
"embed.weight": "token_embd.weight",
|
| 39 |
+
"embed_norm.w": "token_embd_norm.weight",
|
| 40 |
+
"norm.w": "output_norm.weight",
|
| 41 |
+
"head.weight": "output.weight",
|
| 42 |
+
}
|
| 43 |
+
if name in top:
|
| 44 |
+
return top[name]
|
| 45 |
+
assert name.startswith("blocks."), name
|
| 46 |
+
parts = name.split(".")
|
| 47 |
+
i = int(parts[1])
|
| 48 |
+
rest = ".".join(parts[2:])
|
| 49 |
+
per_block = {
|
| 50 |
+
"n1.w": "attn_norm.weight",
|
| 51 |
+
"n2.w": "ffn_norm.weight",
|
| 52 |
+
"attn.qn.w": "attn_q_rmsnorm.weight",
|
| 53 |
+
"attn.kn.w": "attn_k_rmsnorm.weight",
|
| 54 |
+
"attn.q.weight": "attn_q.weight",
|
| 55 |
+
"attn.q.pre.w": "attn_q_subln.weight",
|
| 56 |
+
"attn.o.weight": "attn_output.weight",
|
| 57 |
+
"attn.o.pre.w": "attn_output_subln.weight",
|
| 58 |
+
"attn.k.weight": "attn_k.weight",
|
| 59 |
+
"attn.k.pre.w": "attn_k_subln.weight",
|
| 60 |
+
"attn.v.weight": "attn_v.weight",
|
| 61 |
+
"attn.v.pre.w": "attn_v_subln.weight",
|
| 62 |
+
"ffn.w1.weight": "ffn_gate.weight",
|
| 63 |
+
"ffn.w1.pre.w": "ffn_gate_subln.weight",
|
| 64 |
+
"ffn.w3.weight": "ffn_up.weight",
|
| 65 |
+
"ffn.w3.pre.w": "ffn_up_subln.weight",
|
| 66 |
+
"ffn.w2.weight": "ffn_down.weight",
|
| 67 |
+
"ffn.w2.pre.w": "ffn_down_subln.weight",
|
| 68 |
+
}
|
| 69 |
+
assert rest in per_block, f"unmapped: {name}"
|
| 70 |
+
return f"blk.{i}.{per_block[rest]}"
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def main():
|
| 74 |
+
tensors = dict(np.load(NPZ))
|
| 75 |
+
roles = json.load(open(ROLES))
|
| 76 |
+
cfg = json.load(open(CFG))
|
| 77 |
+
syl = json.load(open(SYL))
|
| 78 |
+
|
| 79 |
+
# syllable vocab in index order (handle dict token->id or list)
|
| 80 |
+
if isinstance(syl, dict):
|
| 81 |
+
syl_tokens = [None] * len(syl)
|
| 82 |
+
for tok, idx in syl.items():
|
| 83 |
+
syl_tokens[int(idx)] = tok
|
| 84 |
+
assert all(t is not None for t in syl_tokens), "gap in syl vocab ids"
|
| 85 |
+
else:
|
| 86 |
+
syl_tokens = list(syl)
|
| 87 |
+
assert len(syl_tokens) == cfg["n_syl"], (len(syl_tokens), cfg["n_syl"])
|
| 88 |
+
|
| 89 |
+
writer = GGUFWriter(OUT, ARCH)
|
| 90 |
+
writer.add_architecture()
|
| 91 |
+
writer.add_name("slothe-t-25m-zhuyin")
|
| 92 |
+
writer.add_uint32("slothe.context_length", 512)
|
| 93 |
+
writer.add_uint32("slothe.embedding_length", cfg["dim"]) # 352
|
| 94 |
+
writer.add_uint32("slothe.block_count", cfg["depth"]) # 16
|
| 95 |
+
writer.add_uint32("slothe.feed_forward_length", cfg["ffn"]) # 960
|
| 96 |
+
writer.add_uint32("slothe.attention.head_count", cfg["heads"]) # 8
|
| 97 |
+
writer.add_uint32("slothe.attention.head_count_kv", cfg["kv"]) # 2
|
| 98 |
+
writer.add_uint32("slothe.attention.key_length", 44)
|
| 99 |
+
writer.add_uint32("slothe.attention.value_length", 44)
|
| 100 |
+
writer.add_float32("slothe.attention.layer_norm_rms_epsilon", 1e-6)
|
| 101 |
+
writer.add_uint32("slothe.fp_boundary", cfg["fp_boundary"]) # 1
|
| 102 |
+
writer.add_uint32("slothe.vocab_syl", cfg["n_syl"]) # 1539
|
| 103 |
+
writer.add_uint32("slothe.vocab_char", cfg["n_char"]) # 8342
|
| 104 |
+
writer.add_uint32("slothe.ternary_pad_to", PAD_TO) # 256
|
| 105 |
+
writer.add_array("slothe.syl_tokens", syl_tokens)
|
| 106 |
+
|
| 107 |
+
# deterministic order: embed/norm/head first, then blocks
|
| 108 |
+
order = sorted(tensors.keys(), key=lambda n: (not n.startswith("embed"),
|
| 109 |
+
not n.endswith("head.weight") and n in ("norm.w",),
|
| 110 |
+
n))
|
| 111 |
+
# simpler explicit order for readability of the reader table
|
| 112 |
+
def sort_key(n):
|
| 113 |
+
pri = {"embed.weight": 0, "embed_norm.w": 1, "norm.w": 2, "head.weight": 3}
|
| 114 |
+
if n in pri:
|
| 115 |
+
return (0, pri[n], "")
|
| 116 |
+
parts = n.split(".")
|
| 117 |
+
return (1, int(parts[1]), n)
|
| 118 |
+
order = sorted(tensors.keys(), key=sort_key)
|
| 119 |
+
|
| 120 |
+
names_rows = [] # (ggml_name, orig_name, role, orig_shape, stored_dtype, padded_in)
|
| 121 |
+
packed_cache = {} # ggml_name -> (packed, eff_padded, orig_shape) for validation
|
| 122 |
+
for name in order:
|
| 123 |
+
arr = np.asarray(tensors[name], dtype=np.float32)
|
| 124 |
+
gname = ggml_name(name)
|
| 125 |
+
role = roles[name]
|
| 126 |
+
orig_shape = tuple(int(x) for x in arr.shape)
|
| 127 |
+
if role == "fp":
|
| 128 |
+
writer.add_tensor(gname, arr.astype(np.float16))
|
| 129 |
+
names_rows.append((gname, name, role, orig_shape, "F16", ""))
|
| 130 |
+
else: # ternary linear -> pad in-features -> TQ2_0
|
| 131 |
+
out_f, in_f = arr.shape
|
| 132 |
+
pad_in = next_mult(in_f, PAD_TO)
|
| 133 |
+
padded = np.zeros((out_f, pad_in), dtype=np.float32)
|
| 134 |
+
padded[:, :in_f] = arr
|
| 135 |
+
packed = quants.quantize(padded, GGMLQuantizationType.TQ2_0)
|
| 136 |
+
writer.add_tensor(gname, packed, raw_dtype=GGMLQuantizationType.TQ2_0)
|
| 137 |
+
names_rows.append((gname, name, role, orig_shape, "TQ2_0", pad_in))
|
| 138 |
+
packed_cache[gname] = (packed, padded, orig_shape, in_f)
|
| 139 |
+
|
| 140 |
+
# ---- VALIDATION 1: round-trip on 2 sample ternary tensors (pre-write) ----
|
| 141 |
+
print("== VALIDATION 1: TQ2_0 round-trip on exact-ternary weights ==")
|
| 142 |
+
samples = [g for g in ("blk.1.attn_q.weight", "blk.7.ffn_down.weight") if g in packed_cache]
|
| 143 |
+
for g in samples:
|
| 144 |
+
packed, padded, orig_shape, in_f = packed_cache[g]
|
| 145 |
+
deq = quants.dequantize(packed, GGMLQuantizationType.TQ2_0).reshape(padded.shape)
|
| 146 |
+
deq_stripped = deq[:, :in_f]
|
| 147 |
+
eff = padded[:, :in_f]
|
| 148 |
+
maxerr = float(np.abs(deq_stripped - eff).max())
|
| 149 |
+
print(f" {g}: orig={orig_shape} padded={padded.shape} maxerr={maxerr:.3e}")
|
| 150 |
+
if maxerr >= 1e-3:
|
| 151 |
+
print(f" FATAL: {g} round-trip maxerr {maxerr} >= 1e-3 -- lossy TQ2_0, aborting")
|
| 152 |
+
sys.exit(1)
|
| 153 |
+
print(" round-trip OK (lossless).")
|
| 154 |
+
|
| 155 |
+
writer.write_header_to_file()
|
| 156 |
+
writer.write_kv_data_to_file()
|
| 157 |
+
writer.write_tensors_to_file()
|
| 158 |
+
writer.close()
|
| 159 |
+
|
| 160 |
+
# ---- NAMES.md ----
|
| 161 |
+
with open(NAMES_MD, "w") as f:
|
| 162 |
+
f.write("# slothe GGUF tensor name map\n\n")
|
| 163 |
+
f.write(f"Architecture: `{ARCH}` | ternary in-features padded to multiple of {PAD_TO} "
|
| 164 |
+
f"(recorded as `slothe.ternary_pad_to`).\n\n")
|
| 165 |
+
f.write("Ternary linears (blocks 1..14 q/o/k/v/w1/w2/w3) are stored TQ2_0 with the "
|
| 166 |
+
"in-features zero-padded to the next multiple of 256 (352->512, 960->1024). "
|
| 167 |
+
"Strip padding using the original in-features listed below. All other tensors are F16.\n\n")
|
| 168 |
+
f.write("| ggml name | original name | role | original shape | stored dtype | padded in-features |\n")
|
| 169 |
+
f.write("|---|---|---|---|---|---|\n")
|
| 170 |
+
for gname, oname, role, oshape, dt, pin in names_rows:
|
| 171 |
+
f.write(f"| `{gname}` | `{oname}` | {role} | {oshape} | {dt} | {pin} |\n")
|
| 172 |
+
|
| 173 |
+
# ---- VALIDATION 2: reload GGUF and print reader table ----
|
| 174 |
+
print("\n== VALIDATION 2: GGUFReader reload ==")
|
| 175 |
+
r = GGUFReader(OUT)
|
| 176 |
+
arch_field = r.fields.get("general.architecture")
|
| 177 |
+
arch_val = None
|
| 178 |
+
if arch_field is not None:
|
| 179 |
+
arch_val = bytes(arch_field.parts[arch_field.data[0]]).decode("utf-8")
|
| 180 |
+
print(f" architecture: {arch_val}")
|
| 181 |
+
print(f" tensor count: {len(r.tensors)}")
|
| 182 |
+
print(f" {'name':40s} {'shape':20s} {'ggml_type'}")
|
| 183 |
+
for t in r.tensors[:12]:
|
| 184 |
+
shp = "x".join(str(int(d)) for d in t.shape)
|
| 185 |
+
print(f" {t.name:40s} {shp:20s} {t.tensor_type.name}")
|
| 186 |
+
|
| 187 |
+
# ---- VALIDATION 3: file size ----
|
| 188 |
+
sz = os.path.getsize(OUT)
|
| 189 |
+
print(f"\n== VALIDATION 3: file size ==")
|
| 190 |
+
print(f" {OUT}: {sz} bytes ({sz/1024/1024:.2f} MiB)")
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
if __name__ == "__main__":
|
| 194 |
+
main()
|
run_ce_ls_32.sh
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
cd ~/sloth-zhuyin-linux/model
|
| 3 |
+
echo "=== DIRECT CE ternary 25M, label-smoothing 0.1, teacher-free, 32ep (train-longer), DDP, snap/4 START ==="; date
|
| 4 |
+
python3 -m torch.distributed.run --nproc_per_node=2 train_slothe_ternary.py \
|
| 5 |
+
--data train_e_g2pw.bin --vocab syl_vocab.json --tokenizer tokenizer --out slothe_t_25m_ce_ls32 \
|
| 6 |
+
--dim 352 --depth 16 --heads 8 --kv-heads 2 --ffn 960 --embed-norm \
|
| 7 |
+
--quant ternary --weight-quant median --pre-norm \
|
| 8 |
+
--label-smoothing 0.1 \
|
| 9 |
+
--batch 384 --epochs 32 --save-every 4 --lr 2.5e-3
|
| 10 |
+
echo "CE_LS32_DONE"; date
|
run_ternary_honest.sh
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
cd ~/sloth-zhuyin-linux/model
|
| 3 |
+
echo "=== HONEST ternary-distilled 25M (working distillation), 16ep, GPU0, snap/4 START ==="; date
|
| 4 |
+
CUDA_VISIBLE_DEVICES=0 python3 train_slothe_ternary.py \
|
| 5 |
+
--data train_e_g2pw.bin --vocab syl_vocab.json --tokenizer tokenizer --out slothe_t_25m_honest \
|
| 6 |
+
--dim 352 --depth 16 --heads 8 --kv-heads 2 --ffn 960 --embed-norm \
|
| 7 |
+
--quant ternary --weight-quant median --pre-norm \
|
| 8 |
+
--teacher slothe_32m_g2pw --distill-alpha 0.7 --distill-temp 2.0 --anneal-frac 0.15 \
|
| 9 |
+
--batch 384 --epochs 16 --save-every 4 --lr 2.5e-3
|
| 10 |
+
echo "=== GATE slothe_t_25m_honest (held-out) ==="
|
| 11 |
+
CUDA_VISIBLE_DEVICES='' python3 gate_slothe_ternary.py --model slothe_t_25m_honest --tokenizer tokenizer --table phonetic_table.tsv --testset ../eval/testset.tsv --mspy ../eval/reference_heldout.tsv
|
| 12 |
+
echo "TERN_HONEST_DONE"; date
|
slothe-t-25m.gguf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4e9ede423e4a52b60bad0f7f3cb175821d6910a2c2e0784ad1e3fe7c399a0fd2
|
| 3 |
+
size 18881056
|
slothe.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c581e9d587a99f864f4e65b5c45e08455b496261088a3df2d4d18d6473055911
|
| 3 |
+
size 98971047
|
slothe_config.json
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"n_syl": 1539,
|
| 3 |
+
"n_char": 8342,
|
| 4 |
+
"dim": 352,
|
| 5 |
+
"depth": 16,
|
| 6 |
+
"heads": 8,
|
| 7 |
+
"kv": 2,
|
| 8 |
+
"ffn": 960,
|
| 9 |
+
"embed_norm": true,
|
| 10 |
+
"weight_bits": "ternary",
|
| 11 |
+
"weight_quant": "median",
|
| 12 |
+
"fp_boundary": 1,
|
| 13 |
+
"pre_norm": true,
|
| 14 |
+
"act_bits": 8,
|
| 15 |
+
"char_hints": false,
|
| 16 |
+
"tie_hints": false
|
| 17 |
+
}
|
syl2legal.npz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f09481e94193da9f5fef15ca7fa45c2e596a19fbae691f1c0d3bbd57dd9be17a
|
| 3 |
+
size 62853
|
syl_vocab.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{"<pad>": 0, "<unk>": 1, "<en>": 2, "ㄅㄚ": 3, "ㄅㄚˇ": 4, "ㄅㄚˊ": 5, "ㄅㄚˋ": 6, "ㄅㄚ˙": 7, "ㄅㄛ": 8, "ㄅㄛˇ": 9, "ㄅㄛˊ": 10, "ㄅㄛˋ": 11, "ㄅㄛ˙": 12, "ㄅㄞ": 13, "ㄅㄞˇ": 14, "ㄅㄞˊ": 15, "ㄅㄞˋ": 16, "ㄅㄟ": 17, "ㄅㄟˇ": 18, "ㄅㄟˋ": 19, "ㄅㄟ˙": 20, "ㄅㄠ": 21, "ㄅㄠˇ": 22, "ㄅㄠˊ": 23, "ㄅㄠˋ": 24, "ㄅㄢ": 25, "ㄅㄢˇ": 26, "ㄅㄢˋ": 27, "ㄅㄢ˙": 28, "ㄅㄣ": 29, "ㄅㄣˇ": 30, "ㄅㄣˋ": 31, "ㄅㄤ": 32, "ㄅㄤˇ": 33, "ㄅㄤˋ": 34, "ㄅㄥ": 35, "ㄅㄥˇ": 36, "ㄅㄥˊ": 37, "ㄅㄥˋ": 38, "ㄅㄧ": 39, "ㄅㄧˇ": 40, "ㄅㄧˊ": 41, "ㄅㄧˋ": 42, "ㄅㄧㄝ": 43, "ㄅㄧㄝˇ": 44, "ㄅㄧㄝˊ": 45, "ㄅㄧㄝˋ": 46, "ㄅㄧㄠ": 47, "ㄅㄧㄠˇ": 48, "ㄅㄧㄠˋ": 49, "ㄅㄧㄢ": 50, "ㄅㄧㄢˇ": 51, "ㄅㄧㄢˋ": 52, "ㄅㄧㄢ˙": 53, "ㄅㄧㄣ": 54, "ㄅㄧㄣˇ": 55, "ㄅㄧㄣˋ": 56, "ㄅㄧㄥ": 57, "ㄅㄧㄥˇ": 58, "ㄅㄧㄥˋ": 59, "ㄅㄨ": 60, "ㄅㄨˇ": 61, "ㄅㄨˊ": 62, "ㄅㄨˋ": 63, "ㄅㄨ˙": 64, "ㄆㄚ": 65, "ㄆㄚˊ": 66, "ㄆㄚˋ": 67, "ㄆㄚ˙": 68, "ㄆㄛ": 69, "ㄆㄛˇ": 70, "ㄆㄛˊ": 71, "ㄆㄛˋ": 72, "ㄆㄛ˙": 73, "ㄆㄞ": 74, "ㄆㄞˇ": 75, "ㄆㄞˊ": 76, "ㄆㄞˋ": 77, "ㄆㄟ": 78, "ㄆㄟˇ": 79, "ㄆㄟˊ": 80, "ㄆㄟˋ": 81, "ㄆㄠ": 82, "ㄆㄠˇ": 83, "ㄆㄠˊ": 84, "ㄆㄠˋ": 85, "ㄆㄡ": 86, "ㄆㄡˇ": 87, "ㄆㄡˊ": 88, "ㄆㄡˋ": 89, "ㄆㄢ": 90, "ㄆㄢˇ": 91, "ㄆㄢˊ": 92, "ㄆㄢˋ": 93, "ㄆㄣ": 94, "ㄆㄣˇ": 95, "ㄆㄣˊ": 96, "ㄆㄣˋ": 97, "ㄆㄤ": 98, "ㄆㄤˇ": 99, "ㄆㄤˊ": 100, "ㄆㄤˋ": 101, "ㄆㄥ": 102, "ㄆㄥˇ": 103, "ㄆㄥˊ": 104, "ㄆㄥˋ": 105, "ㄆㄧ": 106, "ㄆㄧˇ": 107, "ㄆㄧˊ": 108, "ㄆㄧˋ": 109, "ㄆㄧㄝ": 110, "ㄆㄧㄝˇ": 111, "ㄆㄧㄝˋ": 112, "ㄆㄧㄠ": 113, "ㄆㄧㄠˇ": 114, "ㄆㄧㄠˊ": 115, "ㄆㄧㄠˋ": 116, "ㄆㄧㄢ": 117, "ㄆㄧㄢˇ": 118, "ㄆㄧㄢˊ": 119, "ㄆㄧㄢˋ": 120, "ㄆㄧㄣ": 121, "ㄆㄧㄣˇ": 122, "ㄆㄧㄣˊ": 123, "ㄆㄧㄣˋ": 124, "ㄆㄧㄥ": 125, "ㄆㄧㄥˇ": 126, "ㄆㄧㄥˊ": 127, "ㄆㄧㄥˋ": 128, "ㄆㄨ": 129, "ㄆㄨˇ": 130, "ㄆㄨˊ": 131, "ㄆㄨˋ": 132, "ㄇㄚ": 133, "ㄇㄚˇ": 134, "ㄇㄚˊ": 135, "ㄇㄚˋ": 136, "ㄇㄚ˙": 137, "ㄇㄛ": 138, "ㄇㄛˇ": 139, "ㄇㄛˊ": 140, "ㄇㄛˋ": 141, "ㄇㄜ˙": 142, "ㄇㄞˇ": 143, "ㄇㄞˊ": 144, "ㄇㄞˋ": 145, "ㄇㄟˇ": 146, "ㄇㄟˊ": 147, "ㄇㄟˋ": 148, "ㄇㄟ˙": 149, "ㄇㄠ": 150, "ㄇㄠˇ": 151, "ㄇㄠˊ": 152, "ㄇㄠˋ": 153, "ㄇㄡ": 154, "ㄇㄡˇ": 155, "ㄇㄡˊ": 156, "ㄇㄡˋ": 157, "ㄇㄢ": 158, "ㄇㄢˇ": 159, "ㄇㄢˊ": 160, "ㄇㄢˋ": 161, "ㄇㄣ": 162, "ㄇㄣˇ": 163, "ㄇㄣˊ": 164, "ㄇㄣˋ": 165, "ㄇㄣ˙": 166, "ㄇㄤˇ": 167, "ㄇㄤˊ": 168, "ㄇㄥ": 169, "ㄇㄥˇ": 170, "ㄇㄥˊ": 171, "ㄇㄥˋ": 172, "ㄇㄧ": 173, "ㄇㄧˇ": 174, "ㄇㄧˊ": 175, "ㄇㄧˋ": 176, "ㄇㄧㄝ": 177, "ㄇㄧㄝˋ": 178, "ㄇㄧㄝ˙": 179, "ㄇㄧㄠ": 180, "ㄇㄧㄠˇ": 181, "ㄇㄧㄠˊ": 182, "ㄇㄧㄠˋ": 183, "ㄇㄧㄡ": 184, "ㄇㄧㄡˋ": 185, "ㄇㄧㄢˇ": 186, "ㄇㄧㄢˊ": 187, "ㄇㄧㄢˋ": 188, "ㄇㄧㄣˇ": 189, "ㄇㄧㄣˊ": 190, "ㄇㄧㄥˇ": 191, "ㄇㄧㄥˊ": 192, "ㄇㄧㄥˋ": 193, "ㄇㄨ": 194, "ㄇㄨˇ": 195, "ㄇㄨˊ": 196, "ㄇㄨˋ": 197, "ㄈㄚ": 198, "ㄈㄚˇ": 199, "ㄈㄚˊ": 200, "ㄈㄚˋ": 201, "ㄈㄛˊ": 202, "ㄈㄟ": 203, "ㄈㄟˇ": 204, "ㄈㄟˊ": 205, "ㄈㄟˋ": 206, "ㄈㄡ": 207, "ㄈㄡˇ": 208, "ㄈㄡˊ": 209, "ㄈㄡˋ": 210, "ㄈㄢ": 211, "ㄈㄢˇ": 212, "ㄈㄢˊ": 213, "ㄈㄢˋ": 214, "ㄈㄣ": 215, "ㄈㄣˇ": 216, "ㄈㄣˊ": 217, "ㄈㄣˋ": 218, "ㄈㄤ": 219, "ㄈㄤˇ": 220, "ㄈㄤˊ": 221, "ㄈㄤˋ": 222, "ㄈㄤ˙": 223, "ㄈㄥ": 224, "ㄈㄥˇ": 225, "ㄈㄥˊ": 226, "ㄈㄥˋ": 227, "ㄈㄧㄠˋ": 228, "ㄈㄨ": 229, "ㄈㄨˇ": 230, "ㄈㄨˊ": 231, "ㄈㄨˋ": 232, "ㄉㄚ": 233, "ㄉㄚˇ": 234, "ㄉㄚˊ": 235, "ㄉㄚˋ": 236, "ㄉㄚ˙": 237, "ㄉㄜˊ": 238, "ㄉㄜ˙": 239, "ㄉㄞ": 240, "ㄉㄞˇ": 241, "ㄉㄞˋ": 242, "ㄉㄟˇ": 243, "ㄉㄠ": 244, "ㄉㄠˇ": 245, "ㄉㄠˊ": 246, "ㄉㄠˋ": 247, "ㄉㄡ": 248, "ㄉㄡˇ": 249, "ㄉㄡˋ": 250, "ㄉㄢ": 251, "ㄉㄢˇ": 252, "ㄉㄢˋ": 253, "ㄉㄣˋ": 254, "ㄉㄤ": 255, "ㄉㄤˇ": 256, "ㄉㄤˋ": 257, "ㄉㄤ˙": 258, "ㄉㄥ": 259, "ㄉㄥˇ": 260, "ㄉㄥˋ": 261, "ㄉㄧ": 262, "ㄉㄧˇ": 263, "ㄉㄧˊ": 264, "ㄉㄧˋ": 265, "ㄉㄧ˙": 266, "ㄉㄧㄚˇ": 267, "ㄉㄧㄝ": 268, "ㄉㄧㄝˇ": 269, "ㄉㄧㄝˊ": 270, "ㄉㄧㄝˋ": 271, "ㄉㄧㄝ˙": 272, "ㄉㄧㄠ": 273, "ㄉㄧㄠˇ": 274, "ㄉㄧㄠˋ": 275, "ㄉㄧㄡ": 276, "ㄉㄧㄢ": 277, "ㄉㄧㄢˇ": 278, "ㄉㄧㄢˋ": 279, "ㄉㄧㄥ": 280, "ㄉㄧㄥˇ": 281, "ㄉㄧㄥˋ": 282, "ㄉㄨ": 283, "ㄉㄨˇ": 284, "ㄉㄨˊ": 285, "ㄉㄨˋ": 286, "ㄉㄨㄛ": 287, "ㄉㄨㄛˇ": 288, "ㄉㄨㄛˊ": 289, "ㄉㄨㄛˋ": 290, "ㄉㄨㄛ˙": 291, "ㄉㄨㄟ": 292, "ㄉㄨㄟˇ": 293, "ㄉㄨㄟˋ": 294, "ㄉㄨㄢ": 295, "ㄉㄨㄢˇ": 296, "ㄉㄨㄢˋ": 297, "ㄉㄨㄣ": 298, "ㄉㄨㄣˇ": 299, "ㄉㄨㄣˋ": 300, "ㄉㄨㄥ": 301, "ㄉㄨㄥˇ": 302, "ㄉㄨㄥˋ": 303, "ㄊㄚ": 304, "ㄊㄚˇ": 305, "ㄊㄚˋ": 306, "ㄊㄚ˙": 307, "ㄊㄜˋ": 308, "ㄊㄞ": 309, "ㄊㄞˇ": 310, "ㄊㄞˊ": 311, "ㄊㄞˋ": 312, "ㄊㄟ": 313, "ㄊㄠ": 314, "ㄊㄠˇ": 315, "ㄊㄠˊ": 316, "ㄊㄠˋ": 317, "ㄊㄠ˙": 318, "ㄊㄡ": 319, "ㄊㄡˇ": 320, "ㄊㄡˊ": 321, "ㄊㄡˋ": 322, "ㄊㄡ˙": 323, "ㄊㄢ": 324, "ㄊㄢˇ": 325, "ㄊㄢˊ": 326, "ㄊㄢˋ": 327, "ㄊㄤ": 328, "ㄊㄤˇ": 329, "ㄊㄤˊ": 330, "ㄊㄤˋ": 331, "ㄊㄥ": 332, "ㄊㄥˊ": 333, "ㄊㄥˋ": 334, "ㄊㄧ": 335, "ㄊㄧˇ": 336, "ㄊㄧˊ": 337, "ㄊㄧˋ": 338, "ㄊㄧㄝ": 339, "ㄊㄧㄝˇ": 340, "ㄊㄧㄝˋ": 341, "ㄊㄧㄠ": 342, "ㄊㄧㄠˇ": 343, "ㄊㄧㄠˊ": 344, "ㄊㄧㄠˋ": 345, "ㄊㄧㄠ˙": 346, "ㄊㄧㄢ": 347, "ㄊㄧㄢˇ": 348, "ㄊㄧㄢˊ": 349, "ㄊㄧㄢˋ": 350, "ㄊㄧㄥ": 351, "ㄊㄧㄥˇ": 352, "ㄊㄧㄥˊ": 353, "ㄊㄧㄥˋ": 354, "ㄊㄨ": 355, "ㄊㄨˇ": 356, "ㄊㄨˊ": 357, "ㄊㄨˋ": 358, "ㄊㄨㄛ": 359, "ㄊㄨㄛˇ": 360, "ㄊㄨㄛˊ": 361, "ㄊㄨㄛˋ": 362, "ㄊㄨㄟ": 363, "ㄊㄨㄟˇ": 364, "ㄊㄨㄟˊ": 365, "ㄊㄨㄟˋ": 366, "ㄊㄨㄢ": 367, "ㄊㄨㄢˇ": 368, "ㄊㄨㄢˊ": 369, "ㄊㄨㄢˋ": 370, "ㄊㄨㄣ": 371, "ㄊㄨㄣˇ": 372, "ㄊㄨㄣˊ": 373, "ㄊㄨㄣˋ": 374, "ㄊㄨㄥ": 375, "ㄊㄨㄥˇ": 376, "ㄊㄨㄥˊ": 377, "ㄊㄨㄥˋ": 378, "ㄋ": 379, "ㄋ˙": 380, "ㄋㄍ": 381, "ㄋㄍˇ": 382, "ㄋㄍˊ": 383, "ㄋㄍˋ": 384, "ㄋㄍ˙": 385, "ㄋㄚ": 386, "ㄋㄚˇ": 387, "ㄋㄚˊ": 388, "ㄋㄚˋ": 389, "ㄋㄚ˙": 390, "ㄋㄜ": 391, "ㄋㄜˊ": 392, "ㄋㄜˋ": 393, "ㄋㄜ˙": 394, "ㄋㄞˇ": 395, "ㄋㄞˊ": 396, "ㄋㄞˋ": 397, "ㄋㄟˇ": 398, "ㄋㄟˋ": 399, "ㄋㄠ": 400, "ㄋㄠˇ": 401, "ㄋㄠˊ": 402, "ㄋㄠˋ": 403, "ㄋㄡˇ": 404, "ㄋㄡˊ": 405, "ㄋㄡˋ": 406, "ㄋㄢ": 407, "ㄋㄢˇ": 408, "ㄋㄢˊ": 409, "ㄋㄢˋ": 410, "ㄋㄣˋ": 411, "ㄋㄤ": 412, "ㄋㄤˇ": 413, "ㄋㄤˊ": 414, "ㄋㄤˋ": 415, "ㄋㄤ˙": 416, "ㄋㄥˇ": 417, "ㄋㄥˊ": 418, "ㄋㄥˋ": 419, "ㄋㄧ": 420, "ㄋㄧˇ": 421, "ㄋㄧˊ": 422, "ㄋㄧˋ": 423, "ㄋㄧㄝ": 424, "ㄋㄧㄝˊ": 425, "ㄋㄧㄝˋ": 426, "ㄋㄧㄠˇ": 427, "ㄋㄧㄠˋ": 428, "ㄋㄧㄡ": 429, "ㄋㄧㄡˇ": 430, "ㄋㄧㄡˊ": 431, "ㄋㄧㄡˋ": 432, "ㄋㄧㄢ": 433, "ㄋㄧㄢˇ": 434, "ㄋㄧㄢˊ": 435, "ㄋㄧㄢˋ": 436, "ㄋㄧㄣˇ": 437, "ㄋㄧㄣˊ": 438, "ㄋㄧㄤˊ": 439, "ㄋㄧㄤˋ": 440, "ㄋㄧㄥˇ": 441, "ㄋㄧㄥˊ": 442, "ㄋㄧㄥˋ": 443, "ㄋㄨˇ": 444, "ㄋㄨˊ": 445, "ㄋㄨˋ": 446, "ㄋㄨㄛˇ": 447, "ㄋㄨㄛˊ": 448, "ㄋㄨㄛˋ": 449, "ㄋㄨㄢˇ": 450, "ㄋㄨㄢˊ": 451, "ㄋㄨㄢˋ": 452, "ㄋㄨㄥˇ": 453, "ㄋㄨㄥˊ": 454, "ㄋㄨㄥˋ": 455, "ㄋㄨㄥ˙": 456, "ㄋㄩˇ": 457, "ㄋㄩˋ": 458, "ㄋㄩㄝˋ": 459, "ㄌㄚ": 460, "ㄌㄚˇ": 461, "ㄌㄚˊ": 462, "ㄌㄚˋ": 463, "ㄌㄚ˙": 464, "ㄌㄛ˙": 465, "ㄌㄜ": 466, "ㄌㄜˋ": 467, "ㄌㄜ˙": 468, "ㄌㄞˇ": 469, "ㄌㄞˊ": 470, "ㄌㄞˋ": 471, "ㄌㄟ": 472, "ㄌㄟˇ": 473, "ㄌㄟˊ": 474, "ㄌㄟˋ": 475, "ㄌㄟ˙": 476, "ㄌㄠ": 477, "ㄌㄠˇ": 478, "ㄌㄠˊ": 479, "ㄌㄠˋ": 480, "ㄌㄡ": 481, "ㄌㄡˇ": 482, "ㄌㄡˊ": 483, "ㄌㄡˋ": 484, "ㄌㄡ˙": 485, "ㄌㄢˇ": 486, "ㄌㄢˊ": 487, "ㄌㄢˋ": 488, "ㄌㄤ": 489, "ㄌㄤˇ": 490, "ㄌㄤˊ": 491, "ㄌㄤˋ": 492, "ㄌㄤ˙": 493, "ㄌㄥ": 494, "ㄌㄥˇ": 495, "ㄌㄥˊ": 496, "ㄌㄥˋ": 497, "ㄌㄧ": 498, "ㄌㄧˇ": 499, "ㄌㄧˊ": 500, "ㄌㄧˋ": 501, "ㄌㄧ˙": 502, "ㄌㄧㄚˇ": 503, "ㄌㄧㄝ": 504, "ㄌㄧㄝˇ": 505, "ㄌㄧㄝˊ": 506, "ㄌㄧㄝˋ": 507, "ㄌㄧㄝ˙": 508, "ㄌㄧㄠ": 509, "ㄌㄧㄠˇ": 510, "ㄌㄧㄠˊ": 511, "ㄌㄧㄠˋ": 512, "ㄌㄧㄡ": 513, "ㄌㄧㄡˇ": 514, "ㄌㄧㄡˊ": 515, "ㄌㄧㄡˋ": 516, "ㄌㄧㄡ˙": 517, "ㄌㄧㄢˇ": 518, "ㄌㄧㄢˊ": 519, "ㄌㄧㄢˋ": 520, "ㄌㄧㄣ": 521, "ㄌㄧㄣˇ": 522, "ㄌㄧㄣˊ": 523, "ㄌㄧㄣˋ": 524, "ㄌㄧㄤˇ": 525, "ㄌㄧㄤˊ": 526, "ㄌㄧㄤˋ": 527, "ㄌㄧㄥ": 528, "ㄌㄧㄥˇ": 529, "ㄌㄧㄥˊ": 530, "ㄌㄧㄥˋ": 531, "ㄌㄨ": 532, "ㄌㄨˇ": 533, "ㄌㄨˊ": 534, "ㄌㄨˋ": 535, "ㄌㄨ˙": 536, "ㄌㄨㄛ": 537, "ㄌㄨㄛˇ": 538, "ㄌㄨㄛˊ": 539, "ㄌㄨㄛˋ": 540, "ㄌㄨㄛ˙": 541, "ㄌㄨㄢˇ": 542, "ㄌㄨㄢˊ": 543, "ㄌㄨㄢˋ": 544, "ㄌㄨㄣ": 545, "ㄌㄨㄣˇ": 546, "ㄌㄨㄣˊ": 547, "ㄌㄨㄣˋ": 548, "ㄌㄨㄥ": 549, "ㄌㄨㄥˇ": 550, "ㄌㄨㄥˊ": 551, "ㄌㄨㄥˋ": 552, "ㄌㄩˇ": 553, "ㄌㄩˊ": 554, "ㄌㄩˋ": 555, "ㄌㄩㄝˇ": 556, "ㄌㄩㄝˋ": 557, "ㄍㄚ": 558, "ㄍㄚˇ": 559, "ㄍㄚˊ": 560, "ㄍㄚˋ": 561, "ㄍㄚ˙": 562, "ㄍㄜ": 563, "ㄍㄜˇ": 564, "ㄍㄜˊ": 565, "ㄍㄜˋ": 566, "ㄍㄜ˙": 567, "ㄍㄞ": 568, "ㄍㄞˇ": 569, "ㄍㄞˋ": 570, "ㄍㄟˇ": 571, "ㄍㄠ": 572, "ㄍㄠˇ": 573, "ㄍㄠˋ": 574, "ㄍㄡ": 575, "ㄍㄡˇ": 576, "ㄍㄡˋ": 577, "ㄍㄢ": 578, "ㄍㄢˇ": 579, "ㄍㄢˋ": 580, "ㄍㄣ": 581, "ㄍㄣˇ": 582, "ㄍㄣˊ": 583, "ㄍㄣˋ": 584, "ㄍㄤ": 585, "ㄍㄤˇ": 586, "ㄍㄤˋ": 587, "ㄍ���": 588, "ㄍㄥˇ": 589, "ㄍㄥˋ": 590, "ㄍㄨ": 591, "ㄍㄨˇ": 592, "ㄍㄨˊ": 593, "ㄍㄨˋ": 594, "ㄍㄨ˙": 595, "ㄍㄨㄚ": 596, "ㄍㄨㄚˇ": 597, "ㄍㄨㄚˋ": 598, "ㄍㄨㄛ": 599, "ㄍㄨㄛˇ": 600, "ㄍㄨㄛˊ": 601, "ㄍㄨㄛˋ": 602, "ㄍㄨㄛ˙": 603, "ㄍㄨㄞ": 604, "ㄍㄨㄞˇ": 605, "ㄍㄨㄞˋ": 606, "ㄍㄨㄟ": 607, "ㄍㄨㄟˇ": 608, "ㄍㄨㄟˋ": 609, "ㄍㄨㄢ": 610, "ㄍㄨㄢˇ": 611, "ㄍㄨㄢˋ": 612, "ㄍㄨㄣˇ": 613, "ㄍㄨㄣˋ": 614, "ㄍㄨㄤ": 615, "ㄍㄨㄤˇ": 616, "ㄍㄨㄤˋ": 617, "ㄍㄨㄥ": 618, "ㄍㄨㄥˇ": 619, "ㄍㄨㄥˋ": 620, "ㄎㄚ": 621, "ㄎㄚˇ": 622, "ㄎㄚˋ": 623, "ㄎㄜ": 624, "ㄎㄜˇ": 625, "ㄎㄜˊ": 626, "ㄎㄜˋ": 627, "ㄎㄜ˙": 628, "ㄎㄞ": 629, "ㄎㄞˇ": 630, "ㄎㄞˋ": 631, "ㄎㄟ": 632, "ㄎㄠ": 633, "ㄎㄠˇ": 634, "ㄎㄠˋ": 635, "ㄎㄡ": 636, "ㄎㄡˇ": 637, "ㄎㄡˋ": 638, "ㄎㄢ": 639, "ㄎㄢˇ": 640, "ㄎㄢˋ": 641, "ㄎㄣˇ": 642, "ㄎㄣˋ": 643, "ㄎㄤ": 644, "ㄎㄤˇ": 645, "ㄎㄤˊ": 646, "ㄎㄤˋ": 647, "ㄎㄥ": 648, "ㄎㄥˇ": 649, "ㄎㄨ": 650, "ㄎㄨˇ": 651, "ㄎㄨˋ": 652, "ㄎㄨㄚ": 653, "ㄎㄨㄚˇ": 654, "ㄎㄨㄚˋ": 655, "ㄎㄨㄛˇ": 656, "ㄎㄨㄛˋ": 657, "ㄎㄨㄞ": 658, "ㄎㄨㄞˇ": 659, "ㄎㄨㄞˋ": 660, "ㄎㄨㄟ": 661, "ㄎㄨㄟˇ": 662, "ㄎㄨㄟˊ": 663, "ㄎㄨㄟˋ": 664, "ㄎㄨㄢ": 665, "ㄎㄨㄢˇ": 666, "ㄎㄨㄣ": 667, "ㄎㄨㄣˇ": 668, "ㄎㄨㄣˋ": 669, "ㄎㄨㄤ": 670, "ㄎㄨㄤˇ": 671, "ㄎㄨㄤˊ": 672, "ㄎㄨㄤˋ": 673, "ㄎㄨㄥ": 674, "ㄎㄨㄥˇ": 675, "ㄎㄨㄥˋ": 676, "ㄏㄇ": 677, "ㄏㄇ˙": 678, "ㄏㄋㄍ": 679, "ㄏㄋㄍ˙": 680, "ㄏㄚ": 681, "ㄏㄚˇ": 682, "ㄏㄚˊ": 683, "ㄏㄚˋ": 684, "ㄏㄜ": 685, "ㄏㄜˊ": 686, "ㄏㄜˋ": 687, "ㄏㄞ": 688, "ㄏㄞˇ": 689, "ㄏㄞˊ": 690, "ㄏㄞˋ": 691, "ㄏㄟ": 692, "ㄏㄟˇ": 693, "ㄏㄠ": 694, "ㄏㄠˇ": 695, "ㄏㄠˊ": 696, "ㄏㄠˋ": 697, "ㄏㄡ": 698, "ㄏㄡˇ": 699, "ㄏㄡˊ": 700, "ㄏㄡˋ": 701, "ㄏㄡ˙": 702, "ㄏㄢ": 703, "ㄏㄢˇ": 704, "ㄏㄢˊ": 705, "ㄏㄢˋ": 706, "ㄏㄣ": 707, "ㄏㄣˇ": 708, "ㄏㄣˊ": 709, "ㄏㄣˋ": 710, "ㄏㄤ": 711, "ㄏㄤˇ": 712, "ㄏㄤˊ": 713, "ㄏㄤˋ": 714, "ㄏㄥ": 715, "ㄏㄥˊ": 716, "ㄏㄥˋ": 717, "ㄏㄥ˙": 718, "ㄏㄨ": 719, "ㄏㄨˇ": 720, "ㄏㄨˊ": 721, "ㄏㄨˋ": 722, "ㄏㄨㄚ": 723, "ㄏㄨㄚˊ": 724, "ㄏㄨㄚˋ": 725, "ㄏㄨㄛ": 726, "ㄏㄨㄛˇ": 727, "ㄏㄨㄛˊ": 728, "ㄏㄨㄛˋ": 729, "ㄏㄨㄛ˙": 730, "ㄏㄨㄞˊ": 731, "ㄏㄨㄞˋ": 732, "ㄏㄨㄞ˙": 733, "ㄏㄨㄟ": 734, "ㄏㄨㄟˇ": 735, "ㄏㄨㄟˊ": 736, "ㄏㄨㄟˋ": 737, "ㄏㄨㄢ": 738, "ㄏㄨㄢˇ": 739, "ㄏㄨㄢˊ": 740, "ㄏㄨㄢˋ": 741, "ㄏㄨㄣ": 742, "ㄏㄨㄣˇ": 743, "ㄏㄨㄣˊ": 744, "ㄏㄨㄣˋ": 745, "ㄏㄨㄤ": 746, "ㄏㄨㄤˇ": 747, "ㄏㄨㄤˊ": 748, "ㄏㄨㄤˋ": 749, "ㄏㄨㄤ˙": 750, "ㄏㄨㄥ": 751, "ㄏㄨㄥˇ": 752, "ㄏㄨㄥˊ": 753, "ㄏㄨㄥˋ": 754, "ㄐㄧ": 755, "ㄐㄧˇ": 756, "ㄐㄧˊ": 757, "ㄐㄧˋ": 758, "ㄐㄧㄚ": 759, "ㄐㄧㄚˇ": 760, "ㄐㄧㄚˊ": 761, "ㄐㄧㄚˋ": 762, "ㄐㄧㄚ˙": 763, "ㄐㄧㄝ": 764, "ㄐㄧㄝˇ": 765, "ㄐㄧㄝˊ": 766, "ㄐㄧㄝˋ": 767, "ㄐㄧㄝ˙": 768, "ㄐㄧㄠ": 769, "ㄐㄧㄠˇ": 770, "ㄐㄧㄠˊ": 771, "ㄐㄧㄠˋ": 772, "ㄐㄧㄡ": 773, "ㄐㄧㄡˇ": 774, "ㄐㄧㄡˋ": 775, "ㄐㄧㄡ˙": 776, "ㄐㄧㄢ": 777, "ㄐㄧㄢˇ": 778, "ㄐㄧㄢˋ": 779, "ㄐㄧㄣ": 780, "ㄐㄧㄣˇ": 781, "ㄐㄧㄣˋ": 782, "ㄐㄧㄤ": 783, "ㄐㄧㄤˇ": 784, "ㄐㄧㄤˋ": 785, "ㄐㄧㄥ": 786, "ㄐㄧㄥˇ": 787, "ㄐㄧㄥˋ": 788, "ㄐㄩ": 789, "ㄐㄩˇ": 790, "ㄐㄩˊ": 791, "ㄐㄩˋ": 792, "ㄐㄩㄝ": 793, "ㄐㄩㄝˇ": 794, "ㄐㄩㄝˊ": 795, "ㄐㄩㄝˋ": 796, "ㄐㄩㄢ": 797, "ㄐㄩㄢˇ": 798, "ㄐㄩㄢˋ": 799, "ㄐㄩㄣ": 800, "ㄐㄩㄣˇ": 801, "ㄐㄩㄣˋ": 802, "ㄐㄩㄥ": 803, "ㄐㄩㄥˇ": 804, "ㄐㄩㄥˋ": 805, "ㄑㄧ": 806, "ㄑㄧˇ": 807, "ㄑㄧˊ": 808, "ㄑㄧˋ": 809, "ㄑㄧㄚ": 810, "ㄑㄧㄚˇ": 811, "ㄑㄧㄚˋ": 812, "ㄑㄧㄝ": 813, "ㄑㄧㄝˇ": 814, "ㄑㄧㄝˊ": 815, "ㄑㄧㄝˋ": 816, "ㄑㄧㄠ": 817, "ㄑㄧㄠˇ": 818, "ㄑㄧㄠˊ": 819, "ㄑㄧㄠˋ": 820, "ㄑㄧㄡ": 821, "ㄑㄧㄡˇ": 822, "ㄑㄧㄡˊ": 823, "ㄑㄧㄡˋ": 824, "ㄑㄧㄢ": 825, "ㄑㄧㄢˇ": 826, "ㄑㄧㄢˊ": 827, "ㄑㄧㄢˋ": 828, "ㄑㄧㄣ": 829, "ㄑㄧㄣˇ": 830, "ㄑㄧㄣˊ": 831, "ㄑㄧㄣˋ": 832, "ㄑㄧㄤ": 833, "ㄑㄧㄤˇ": 834, "ㄑㄧㄤˊ": 835, "ㄑㄧㄤˋ": 836, "ㄑㄧㄥ": 837, "ㄑㄧㄥˇ": 838, "ㄑㄧㄥˊ": 839, "ㄑㄧㄥˋ": 840, "ㄑㄩ": 841, "ㄑㄩˇ": 842, "ㄑㄩˊ": 843, "ㄑㄩˋ": 844, "ㄑㄩ˙": 845, "ㄑㄩㄝ": 846, "ㄑㄩㄝˊ": 847, "ㄑㄩㄝˋ": 848, "ㄑㄩㄢ": 849, "ㄑㄩㄢˇ": 850, "ㄑㄩㄢˊ": 851, "ㄑㄩㄢˋ": 852, "ㄑㄩㄣ": 853, "ㄑㄩㄣˇ": 854, "ㄑㄩㄣˊ": 855, "ㄑㄩㄥ": 856, "ㄑㄩㄥˇ": 857, "ㄑㄩㄥˊ": 858, "ㄒㄧ": 859, "ㄒㄧˇ": 860, "ㄒㄧˊ": 861, "ㄒㄧˋ": 862, "ㄒㄧ˙": 863, "ㄒㄧㄚ": 864, "ㄒㄧㄚˇ": 865, "ㄒㄧㄚˊ": 866, "ㄒㄧㄚˋ": 867, "ㄒㄧㄝ": 868, "ㄒㄧㄝˇ": 869, "ㄒㄧㄝˊ": 870, "ㄒㄧㄝˋ": 871, "ㄒㄧㄝ˙": 872, "ㄒㄧㄠ": 873, "ㄒㄧㄠˇ": 874, "ㄒㄧㄠˊ": 875, "ㄒㄧㄠˋ": 876, "ㄒㄧㄡ": 877, "ㄒㄧㄡˇ": 878, "ㄒㄧㄡˊ": 879, "ㄒㄧㄡˋ": 880, "ㄒㄧㄢ": 881, "ㄒㄧㄢˇ": 882, "ㄒㄧㄢˊ": 883, "ㄒㄧㄢˋ": 884, "ㄒㄧㄣ": 885, "ㄒㄧㄣˇ": 886, "ㄒㄧㄣˊ": 887, "ㄒㄧㄣˋ": 888, "ㄒㄧㄤ": 889, "ㄒㄧㄤˇ": 890, "ㄒㄧㄤˊ": 891, "ㄒㄧㄤˋ": 892, "ㄒㄧㄥ": 893, "ㄒㄧㄥˇ": 894, "ㄒㄧㄥˊ": 895, "ㄒㄧㄥˋ": 896, "ㄒㄩ": 897, "ㄒㄩˇ": 898, "ㄒㄩˊ": 899, "ㄒㄩˋ": 900, "ㄒㄩ˙": 901, "ㄒㄩㄝ": 902, "ㄒㄩㄝˇ": 903, "ㄒㄩㄝˊ": 904, "ㄒㄩㄝˋ": 905, "ㄒㄩㄢ": 906, "ㄒㄩㄢˇ": 907, "ㄒㄩㄢˊ": 908, "ㄒㄩㄢˋ": 909, "ㄒㄩㄣ": 910, "ㄒㄩㄣˊ": 911, "ㄒㄩㄣˋ": 912, "ㄒㄩㄥ": 913, "ㄒㄩㄥˇ": 914, "ㄒㄩㄥˊ": 915, "ㄒㄩㄥˋ": 916, "ㄓ": 917, "ㄓˇ": 918, "ㄓˊ": 919, "ㄓˋ": 920, "ㄓㄚ": 921, "ㄓㄚˇ": 922, "ㄓㄚˊ": 923, "ㄓㄚˋ": 924, "ㄓㄚ˙": 925, "ㄓㄜ": 926, "ㄓㄜˇ": 927, "ㄓㄜˊ": 928, "ㄓㄜˋ": 929, "ㄓㄜ˙": 930, "ㄓㄞ": 931, "ㄓㄞˇ": 932, "ㄓㄞˊ": 933, "ㄓㄞˋ": 934, "ㄓㄟˋ": 935, "ㄓㄠ": 936, "ㄓㄠˇ": 937, "ㄓㄠˊ": 938, "ㄓㄠˋ": 939, "ㄓㄡ": 940, "ㄓㄡˇ": 941, "ㄓㄡˊ": 942, "ㄓㄡˋ": 943, "ㄓㄢ": 944, "ㄓㄢˇ": 945, "ㄓㄢˋ": 946, "ㄓㄣ": 947, "ㄓㄣˇ": 948, "ㄓㄣˋ": 949, "ㄓㄤ": 950, "ㄓㄤˇ": 951, "ㄓㄤˋ": 952, "ㄓㄥ": 953, "ㄓㄥˇ": 954, "ㄓㄥˋ": 955, "ㄓㄨ": 956, "ㄓㄨˇ": 957, "ㄓㄨˊ": 958, "ㄓㄨˋ": 959, "ㄓㄨㄚ": 960, "ㄓㄨㄚˇ": 961, "ㄓㄨㄛ": 962, "ㄓㄨㄛˊ": 963, "ㄓㄨㄛˋ": 964, "ㄓㄨㄞ": 965, "ㄓㄨㄞˇ": 966, "ㄓㄨㄞˋ": 967, "ㄓㄨㄟ": 968, "ㄓㄨㄟˇ": 969, "ㄓㄨㄟˋ": 970, "ㄓㄨㄢ": 971, "ㄓㄨㄢˇ": 972, "ㄓㄨㄢˋ": 973, "ㄓㄨㄣ": 974, "ㄓㄨㄣˇ": 975, "ㄓㄨㄣˋ": 976, "ㄓㄨㄤ": 977, "ㄓㄨㄤˇ": 978, "ㄓㄨㄤˋ": 979, "ㄓㄨㄥ": 980, "ㄓㄨㄥˇ": 981, "ㄓㄨㄥˋ": 982, "ㄔ": 983, "ㄔˇ": 984, "ㄔˊ": 985, "ㄔˋ": 986, "ㄔ˙": 987, "ㄔㄚ": 988, "ㄔㄚˇ": 989, "ㄔㄚˊ": 990, "ㄔㄚˋ": 991, "ㄔㄜ": 992, "ㄔㄜˇ": 993, "ㄔㄜˋ": 994, "ㄔㄞ": 995, "ㄔㄞˇ": 996, "ㄔㄞˊ": 997, "ㄔㄞˋ": 998, "ㄔㄠ": 999, "ㄔㄠˇ": 1000, "ㄔㄠˊ": 1001, "ㄔㄠˋ": 1002, "ㄔㄡ": 1003, "ㄔㄡˇ": 1004, "ㄔㄡˊ": 1005, "ㄔㄡˋ": 1006, "ㄔㄢ": 1007, "ㄔㄢˇ": 1008, "ㄔㄢˊ": 1009, "ㄔㄢˋ": 1010, "ㄔㄣ": 1011, "ㄔㄣˇ": 1012, "ㄔㄣˊ": 1013, "ㄔㄣˋ": 1014, "ㄔㄣ˙": 1015, "ㄔㄤ": 1016, "ㄔㄤˇ": 1017, "ㄔㄤˊ": 1018, "ㄔㄤˋ": 1019, "ㄔㄥ": 1020, "ㄔㄥˇ": 1021, "ㄔㄥˊ": 1022, "ㄔㄥˋ": 1023, "ㄔㄨ": 1024, "ㄔㄨˇ": 1025, "ㄔㄨˊ": 1026, "ㄔㄨˋ": 1027, "ㄔㄨ˙": 1028, "ㄔㄨㄚ": 1029, "ㄔㄨㄚˇ": 1030, "ㄔㄨㄛ": 1031, "ㄔㄨㄛˋ": 1032, "ㄔㄨㄞ": 1033, "ㄔㄨㄞˇ": 1034, "ㄔㄨㄞˊ": 1035, "ㄔㄨㄞˋ": 1036, "ㄔㄨㄟ": 1037, "ㄔㄨㄟˊ": 1038, "ㄔㄨㄟˋ": 1039, "ㄔㄨㄢ": 1040, "ㄔㄨㄢˇ": 1041, "ㄔㄨㄢˊ": 1042, "ㄔㄨㄢˋ": 1043, "ㄔㄨㄣ": 1044, "ㄔㄨㄣˇ": 1045, "ㄔㄨㄣˊ": 1046, "ㄔㄨㄤ": 1047, "ㄔㄨㄤˇ": 1048, "ㄔㄨㄤˊ": 1049, "ㄔㄨㄤˋ": 1050, "ㄔㄨㄥ": 1051, "ㄔㄨㄥˇ": 1052, "ㄔㄨㄥˊ": 1053, "ㄔㄨㄥˋ": 1054, "ㄕ": 1055, "ㄕˇ": 1056, "ㄕˊ": 1057, "ㄕˋ": 1058, "ㄕ˙": 1059, "ㄕㄚ": 1060, "ㄕㄚˇ": 1061, "ㄕㄚˊ": 1062, "ㄕㄚˋ": 1063, "ㄕㄚ˙": 1064, "ㄕㄜ": 1065, "ㄕㄜˇ": 1066, "ㄕㄜˊ": 1067, "ㄕㄜˋ": 1068, "ㄕㄞ": 1069, "ㄕㄞˇ": 1070, "ㄕㄞˋ": 1071, "ㄕㄟˊ": 1072, "ㄕㄠ": 1073, "ㄕㄠˇ": 1074, "ㄕㄠˊ": 1075, "ㄕㄠˋ": 1076, "ㄕㄡ": 1077, "ㄕㄡˇ": 1078, "ㄕㄡˊ": 1079, "ㄕㄡˋ": 1080, "ㄕㄢ": 1081, "ㄕㄢˇ": 1082, "ㄕㄢˋ": 1083, "ㄕㄣ": 1084, "ㄕㄣˇ": 1085, "ㄕㄣˊ": 1086, "ㄕㄣˋ": 1087, "ㄕㄣ˙": 1088, "ㄕㄤ": 1089, "ㄕㄤˇ": 1090, "ㄕㄤˋ": 1091, "ㄕㄤ˙": 1092, "ㄕㄥ": 1093, "ㄕㄥˇ": 1094, "ㄕㄥˊ": 1095, "ㄕㄥˋ": 1096, "ㄕㄨ": 1097, "ㄕㄨˇ": 1098, "ㄕㄨˊ": 1099, "ㄕㄨˋ": 1100, "ㄕㄨ˙": 1101, "ㄕㄨㄚ": 1102, "ㄕㄨㄚˇ": 1103, "ㄕㄨㄚˋ": 1104, "ㄕㄨㄛ": 1105, "ㄕㄨㄛˊ": 1106, "ㄕㄨㄛˋ": 1107, "ㄕㄨㄞ": 1108, "ㄕㄨㄞˇ": 1109, "ㄕㄨㄞˋ": 1110, "ㄕㄨㄟˇ": 1111, "ㄕㄨㄟˊ": 1112, "ㄕㄨㄟˋ": 1113, "ㄕㄨㄢ": 1114, "ㄕㄨㄢˋ": 1115, "ㄕㄨㄣˇ": 1116, "ㄕㄨㄣˋ": 1117, "ㄕㄨㄤ": 1118, "ㄕㄨㄤˇ": 1119, "ㄕㄨㄤˋ": 1120, "ㄖ": 1121, "ㄖˋ": 1122, "ㄖㄜˇ": 1123, "ㄖㄜˊ": 1124, "ㄖㄜˋ": 1125, "ㄖㄠˇ": 1126, "ㄖㄠˊ": 1127, "ㄖㄠˋ": 1128, "ㄖㄡˇ": 1129, "ㄖㄡˊ": 1130, "ㄖㄡˋ": 1131, "ㄖㄢˇ": 1132, "ㄖㄢˊ": 1133, "ㄖㄣˇ": 1134, "ㄖㄣˊ": 1135, "ㄖㄣˋ": 1136, "ㄖㄤ": 1137, "ㄖㄤˇ": 1138, "ㄖㄤˊ": 1139, "ㄖㄤˋ": 1140, "ㄖㄥ": 1141, "ㄖㄥˇ": 1142, "ㄖㄥˊ": 1143, "ㄖㄥˋ": 1144, "ㄖㄨˇ": 1145, "ㄖㄨˊ": 1146, "ㄖㄨˋ": 1147, "ㄖㄨㄚˊ": 1148, "ㄖㄨㄛˊ": 1149, "ㄖㄨㄛˋ": 1150, "ㄖㄨㄟˇ": 1151, "ㄖㄨㄟˊ": 1152, "ㄖㄨㄟˋ": 1153, "ㄖㄨㄢˇ": 1154, "ㄖㄨㄢˊ": 1155, "ㄖㄨㄢˋ": 1156, "ㄖㄨㄣ": 1157, "ㄖㄨㄣˊ": 1158, "ㄖㄨㄣˋ": 1159, "ㄖㄨㄥ": 1160, "ㄖㄨㄥˇ": 1161, "ㄖㄨㄥˊ": 1162, "ㄖㄨㄥˋ": 1163, "ㄗ": 1164, "ㄗˇ": 1165, "ㄗˊ": 1166, "ㄗˋ": 1167, "ㄗ˙": 1168, "ㄗㄚ": 1169, "ㄗㄚˇ": 1170, "ㄗㄚˊ": 1171, "ㄗㄚˋ": 1172, "ㄗㄜˇ": 1173, "ㄗㄜˊ": 1174, "ㄗㄜˋ": 1175, "ㄗㄜ˙": 1176, "ㄗㄞ": 1177, "ㄗㄞˇ": 1178, "ㄗㄞˋ": 1179, "ㄗㄞ˙": 1180, "ㄗㄟˊ": 1181, "ㄗㄠ": 1182, "ㄗㄠˇ": 1183, "ㄗㄠˊ": 1184, "ㄗㄠˋ": 1185, "ㄗㄡ": 1186, "ㄗㄡˇ": 1187, "ㄗㄡˋ": 1188, "ㄗㄢ": 1189, "ㄗㄢˇ": 1190, "ㄗㄢˊ": 1191, "ㄗㄢˋ": 1192, "ㄗㄢ˙": 1193, "ㄗㄣˇ": 1194, "ㄗㄣˋ": 1195, "ㄗㄤ": 1196, "ㄗㄤˇ": 1197, "ㄗㄤˋ": 1198, "ㄗㄥ": 1199, "ㄗㄥˇ": 1200, "ㄗㄥˋ": 1201, "ㄗㄨ": 1202, "ㄗㄨˇ": 1203, "ㄗㄨˊ": 1204, "ㄗㄨˋ": 1205, "ㄗㄨㄛ": 1206, "ㄗㄨㄛˇ": 1207, "ㄗㄨㄛˊ": 1208, "ㄗㄨㄛˋ": 1209, "ㄗㄨㄟ": 1210, "ㄗㄨㄟˇ": 1211, "ㄗㄨㄟˋ": 1212, "ㄗㄨㄢ": 1213, "ㄗㄨㄢˇ": 1214, "ㄗㄨㄢˋ": 1215, "ㄗㄨㄣ": 1216, "ㄗㄨㄣˇ": 1217, "ㄗㄨㄣˊ": 1218, "ㄗㄨㄣˋ": 1219, "ㄗㄨㄥ": 1220, "ㄗㄨㄥˇ": 1221, "ㄗㄨㄥˋ": 1222, "ㄘ": 1223, "ㄘˇ": 1224, "ㄘˊ": 1225, "ㄘˋ": 1226, "ㄘㄚ": 1227, "ㄘㄚˇ": 1228, "ㄘㄚˋ": 1229, "ㄘㄜˋ": 1230, "ㄘㄞ": 1231, "ㄘㄞˇ": 1232, "ㄘㄞˊ": 1233, "ㄘㄞˋ": 1234, "ㄘㄠ": 1235, "ㄘㄠˇ": 1236, "ㄘㄠˊ": 1237, "ㄘㄠˋ": 1238, "ㄘㄡ": 1239, "ㄘㄡˇ": 1240, "ㄘㄡˋ": 1241, "ㄘㄢ": 1242, "ㄘㄢˇ": 1243, "ㄘㄢˊ": 1244, "ㄘㄢˋ": 1245, "ㄘㄣ": 1246, "ㄘㄣˊ": 1247, "ㄘㄤ": 1248, "ㄘㄤˇ": 1249, "ㄘㄤˊ": 1250, "ㄘㄤˋ": 1251, "ㄘㄥ": 1252, "ㄘㄥˇ": 1253, "ㄘㄥˊ": 1254, "ㄘㄥˋ": 1255, "ㄘㄨ": 1256, "ㄘㄨˇ": 1257, "ㄘㄨˊ": 1258, "ㄘㄨˋ": 1259, "ㄘㄨㄛ": 1260, "ㄘㄨㄛˇ": 1261, "ㄘㄨㄛˊ": 1262, "ㄘㄨㄛˋ": 1263, "ㄘㄨㄟ": 1264, "ㄘㄨㄟˇ": 1265, "ㄘㄨㄟˊ": 1266, "ㄘㄨㄟˋ": 1267, "ㄘㄨㄢ": 1268, "ㄘㄨㄢˊ": 1269, "ㄘㄨㄢˋ": 1270, "ㄘㄨㄣ": 1271, "ㄘㄨㄣˇ": 1272, "ㄘㄨㄣˊ": 1273, "ㄘㄨㄣˋ": 1274, "ㄘㄨㄥ": 1275, "ㄘㄨㄥˇ": 1276, "ㄘㄨㄥˊ": 1277, "ㄘㄨㄥˋ": 1278, "ㄙ": 1279, "ㄙˇ": 1280, "ㄙˋ": 1281, "ㄙ˙": 1282, "ㄙㄚ": 1283, "ㄙㄚˇ": 1284, "ㄙㄚˋ": 1285, "ㄙㄚ˙": 1286, "ㄙㄜˋ": 1287, "ㄙㄞ": 1288, "ㄙㄞˋ": 1289, "ㄙㄠ": 1290, "ㄙㄠˇ": 1291, "ㄙㄠˋ": 1292, "ㄙㄡ": 1293, "ㄙㄡˇ": 1294, "ㄙㄡˋ": 1295, "ㄙㄢ": 1296, "ㄙㄢˇ": 1297, "ㄙㄢˋ": 1298, "ㄙㄣ": 1299, "ㄙㄣˇ": 1300, "ㄙㄤ": 1301, "ㄙㄤˇ": 1302, "ㄙㄤˋ": 1303, "ㄙㄥ": 1304, "ㄙㄨ": 1305, "ㄙㄨˊ": 1306, "ㄙㄨˋ": 1307, "ㄙㄨㄛ": 1308, "ㄙㄨㄛˇ": 1309, "ㄙㄨㄛˊ": 1310, "ㄙㄨㄛˋ": 1311, "ㄙㄨㄛ˙": 1312, "ㄙㄨㄟ": 1313, "ㄙㄨㄟˇ": 1314, "ㄙㄨㄟˊ": 1315, "ㄙㄨㄟˋ": 1316, "ㄙㄨㄢ": 1317, "ㄙㄨㄢˇ": 1318, "ㄙㄨㄢˋ": 1319, "ㄙㄨㄣ": 1320, "ㄙㄨㄣˇ": 1321, "ㄙㄨㄣˋ": 1322, "ㄙㄨㄥ": 1323, "ㄙㄨㄥˇ": 1324, "ㄙㄨㄥˊ": 1325, "ㄙㄨㄥˋ": 1326, "ㄚ": 1327, "ㄚˇ": 1328, "ㄚˊ": 1329, "ㄚˋ": 1330, "ㄚ˙": 1331, "ㄛ": 1332, "ㄛˇ": 1333, "ㄛˊ": 1334, "ㄛˋ": 1335, "ㄛ˙": 1336, "ㄜ": 1337, "ㄜˇ": 1338, "ㄜˊ": 1339, "ㄜˋ": 1340, "ㄜ˙": 1341, "ㄝ": 1342, "ㄝˇ": 1343, "ㄝˊ": 1344, "ㄝˋ": 1345, "ㄞ": 1346, "ㄞˇ": 1347, "ㄞˊ": 1348, "ㄞˋ": 1349, "ㄟ": 1350, "ㄟˇ": 1351, "ㄟˊ": 1352, "ㄟˋ": 1353, "ㄠ": 1354, "ㄠˇ": 1355, "ㄠˊ": 1356, "ㄠˋ": 1357, "ㄡ": 1358, "ㄡˇ": 1359, "ㄡˊ": 1360, "ㄡˋ": 1361, "ㄡ˙": 1362, "ㄢ": 1363, "ㄢˇ": 1364, "ㄢˊ": 1365, "ㄢˋ": 1366, "ㄣ": 1367, "ㄣˇ": 1368, "ㄣˊ": 1369, "ㄣˋ": 1370, "ㄤ": 1371, "ㄤˇ": 1372, "ㄤˊ": 1373, "ㄤˋ": 1374, "ㄥ": 1375, "ㄦ": 1376, "ㄦˇ": 1377, "ㄦˊ": 1378, "ㄦˋ": 1379, "ㄦ˙": 1380, "ㄧ": 1381, "ㄧˇ": 1382, "ㄧˊ": 1383, "ㄧˋ": 1384, "ㄧㄚ": 1385, "ㄧㄚˇ": 1386, "ㄧㄚˊ": 1387, "ㄧㄚˋ": 1388, "ㄧㄚ˙": 1389, "ㄧㄛ": 1390, "ㄧㄛ˙": 1391, "ㄧㄝ": 1392, "ㄧㄝˇ": 1393, "ㄧㄝˊ": 1394, "ㄧㄝˋ": 1395, "ㄧㄝ˙": 1396, "ㄧㄠ": 1397, "ㄧㄠˇ": 1398, "ㄧㄠˊ": 1399, "ㄧㄠˋ": 1400, "ㄧㄡ": 1401, "ㄧㄡˇ": 1402, "ㄧㄡˊ": 1403, "ㄧㄡˋ": 1404, "ㄧㄡ˙": 1405, "ㄧㄢ": 1406, "ㄧㄢˇ": 1407, "ㄧㄢˊ": 1408, "ㄧㄢˋ": 1409, "ㄧㄣ": 1410, "ㄧㄣˇ": 1411, "ㄧㄣˊ": 1412, "ㄧㄣˋ": 1413, "ㄧㄣ˙": 1414, "ㄧㄤ": 1415, "ㄧㄤˇ": 1416, "ㄧㄤˊ": 1417, "ㄧㄤˋ": 1418, "ㄧㄤ˙": 1419, "ㄧㄥ": 1420, "ㄧㄥˇ": 1421, "ㄧㄥˊ": 1422, "ㄧㄥˋ": 1423, "ㄨ": 1424, "ㄨˇ": 1425, "ㄨˊ": 1426, "ㄨˋ": 1427, "ㄨㄚ": 1428, "ㄨㄚˇ": 1429, "ㄨㄚˊ": 1430, "ㄨㄚˋ": 1431, "ㄨㄚ˙": 1432, "ㄨㄛ": 1433, "ㄨㄛˇ": 1434, "ㄨㄛˋ": 1435, "ㄨㄞ": 1436, "ㄨㄞˇ": 1437, "ㄨㄞˋ": 1438, "ㄨㄟ": 1439, "ㄨㄟˇ": 1440, "ㄨㄟˊ": 1441, "ㄨㄟˋ": 1442, "ㄨㄢ": 1443, "ㄨㄢˇ": 1444, "ㄨㄢˊ": 1445, "ㄨㄢˋ": 1446, "ㄨㄣ": 1447, "ㄨㄣˇ": 1448, "ㄨㄣˊ": 1449, "ㄨㄣˋ": 1450, "ㄨㄤ": 1451, "ㄨㄤˇ": 1452, "ㄨㄤˊ": 1453, "ㄨㄤˋ": 1454, "ㄨㄥ": 1455, "ㄨㄥˇ": 1456, "ㄨㄥˋ": 1457, "ㄩ": 1458, "ㄩˇ": 1459, "ㄩˊ": 1460, "ㄩˋ": 1461, "ㄩㄝ": 1462, "ㄩㄝˇ": 1463, "ㄩㄝˋ": 1464, "ㄩㄢ": 1465, "ㄩㄢˇ": 1466, "ㄩㄢˊ": 1467, "ㄩㄢˋ": 1468, "ㄩㄣ": 1469, "ㄩㄣˇ": 1470, "ㄩㄣˊ": 1471, "ㄩㄣˋ": 1472, "ㄩㄥ": 1473, "ㄩㄥˇ": 1474, "ㄩㄥˊ": 1475, "ㄩㄥˋ": 1476, "ㄇㄜ": 1477, "ㄇㄞ": 1478, "ㄇㄟ": 1479, "ㄇㄤ": 1480, "ㄇㄧㄢ": 1481, "ㄇㄧㄣ": 1482, "ㄇㄧㄥ": 1483, "ㄈㄛ": 1484, "ㄈㄧㄠ": 1485, "ㄉㄜ": 1486, "ㄉㄟ": 1487, "ㄉㄣ": 1488, "ㄉㄧㄚ": 1489, "ㄊㄜ": 1490, "ㄋㄞ": 1491, "ㄋㄟ": 1492, "ㄋㄡ": 1493, "ㄋㄣ": 1494, "ㄋㄥ": 1495, "ㄋㄧㄠ": 1496, "ㄋㄧㄣ": 1497, "ㄋㄧㄤ": 1498, "ㄋㄧㄥ": 1499, "ㄋㄨ": 1500, "ㄋㄨㄛ": 1501, "ㄋㄨㄢ": 1502, "ㄋㄨㄥ": 1503, "ㄋㄩ": 1504, "ㄋㄩㄝ": 1505, "ㄌㄛ": 1506, "ㄌㄞ": 1507, "ㄌㄢ": 1508, "ㄌㄧㄚ": 1509, "ㄌㄧㄢ": 1510, "ㄌㄧㄤ": 1511, "ㄌㄨㄢ": 1512, "ㄌㄩ": 1513, "ㄌㄩㄝ": 1514, "ㄍㄟ": 1515, "ㄍㄨㄣ": 1516, "ㄎㄣ": 1517, "ㄎㄨㄛ": 1518, "ㄏㄨㄞ": 1519, "ㄓㄟ": 1520, "ㄕㄟ": 1521, "ㄕㄨㄟ": 1522, "ㄕㄨㄣ": 1523, "ㄖㄜ": 1524, "ㄖㄠ": 1525, "ㄖㄡ": 1526, "ㄖㄢ": 1527, "ㄖㄣ": 1528, "ㄖㄨ": 1529, "ㄖㄨㄚ": 1530, "ㄖㄨㄛ": 1531, "ㄖㄨㄟ": 1532, "ㄖㄨㄢ": 1533, "ㄗㄜ": 1534, "ㄗㄟ": 1535, "ㄗㄣ": 1536, "ㄘㄜ": 1537, "ㄙㄜ": 1538}
|
train_slothe_ternary.py
ADDED
|
@@ -0,0 +1,585 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Ternary (1.58-bit) SlothLM-E trainer — a drop-in replacement for
|
| 4 |
+
`reproduce/train_slothlm_e.py` that adds:
|
| 5 |
+
|
| 6 |
+
* BitLinear (ternary {-1,0,+1} weights, absMEDIAN scale per output channel,
|
| 7 |
+
int8 activations, straight-through estimator) in place of nn.Linear inside
|
| 8 |
+
every transformer block.
|
| 9 |
+
* Mixed-precision islands: the syllable embedding, the char head, and the
|
| 10 |
+
first/last `--fp-boundary` blocks stay full precision (int8 at deploy).
|
| 11 |
+
* Progressive fp->ternary annealing (`--anneal-frac`).
|
| 12 |
+
* "Extra RMSNorm before every linear" (SubLN) for ternary stability.
|
| 13 |
+
* Knowledge distillation from a full-precision SlothLM-E teacher (`--teacher`).
|
| 14 |
+
|
| 15 |
+
Module names MIRROR reproduce/train_slothlm_e.py exactly, so:
|
| 16 |
+
- a full-precision teacher checkpoint saved by the original script loads into
|
| 17 |
+
an all-fp SlothE_T for distillation, and
|
| 18 |
+
- the saved ternary checkpoint carries a `config` dict that the companion
|
| 19 |
+
gate/export scripts use to rebuild the model.
|
| 20 |
+
|
| 21 |
+
Design rationale + citations: see docs/GUIDE_TERNARY_SLOTHLM_E.md in this repo.
|
| 22 |
+
|
| 23 |
+
Example (on the RTX 5090):
|
| 24 |
+
python3 train_slothe_ternary.py \
|
| 25 |
+
--data train_e_g2pw.bin --vocab syl_vocab.json --tokenizer tokenizer \
|
| 26 |
+
--out slothe_t_20m \
|
| 27 |
+
--dim 320 --depth 16 --heads 8 --kv-heads 2 --ffn 880 --embed-norm \
|
| 28 |
+
--teacher slothe_32m --distill-alpha 0.7 --distill-temp 2.0 \
|
| 29 |
+
--anneal-frac 0.15 --weight-quant median --pre-norm \
|
| 30 |
+
--batch 384 --epochs 8 --lr 2.5e-3
|
| 31 |
+
"""
|
| 32 |
+
import argparse
|
| 33 |
+
import json
|
| 34 |
+
import math
|
| 35 |
+
import os
|
| 36 |
+
|
| 37 |
+
import numpy as np
|
| 38 |
+
import torch
|
| 39 |
+
import torch.distributed as dist
|
| 40 |
+
import torch.nn as nn
|
| 41 |
+
import torch.nn.functional as F
|
| 42 |
+
from torch.utils.data import DataLoader, Dataset
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
# ======================================================================
|
| 46 |
+
# Quantization primitives (ternary weights + int8 activations, STE)
|
| 47 |
+
# ======================================================================
|
| 48 |
+
|
| 49 |
+
def activation_quant(x, eps=1e-5):
|
| 50 |
+
"""Per-token int8 (absmax) fake-quant; returns a dequantized tensor."""
|
| 51 |
+
scale = 127.0 / x.abs().amax(dim=-1, keepdim=True).clamp_(min=eps)
|
| 52 |
+
return (x * scale).round().clamp_(-128, 127) / scale
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def weight_quant_ternary(w, mode="median", eps=1e-5):
|
| 56 |
+
"""Ternarize to {-1,0,+1} * scale, per OUTPUT channel.
|
| 57 |
+
|
| 58 |
+
mode="median" uses the absmedian scale (BitNet b1.58 Reloaded, arXiv:2407.09527
|
| 59 |
+
— more robust than absmean for SMALL models). mode="mean" is vanilla b1.58.
|
| 60 |
+
"""
|
| 61 |
+
if mode == "median":
|
| 62 |
+
scale = w.abs().median(dim=1, keepdim=True).values.clamp_(min=eps)
|
| 63 |
+
else:
|
| 64 |
+
scale = w.abs().mean(dim=1, keepdim=True).clamp_(min=eps)
|
| 65 |
+
return (w / scale).round().clamp_(-1, 1) * scale
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def _ste(real, quant, alpha):
|
| 69 |
+
"""Straight-through estimator with anneal blend (alpha in [0,1])."""
|
| 70 |
+
blended = real + alpha * (quant - real)
|
| 71 |
+
return real + (blended - real).detach()
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
class RMSNorm(nn.Module):
|
| 75 |
+
# param name `.w` matches reproduce/train_slothlm_e.py so teacher weights load
|
| 76 |
+
def __init__(self, d, eps=1e-6):
|
| 77 |
+
super().__init__()
|
| 78 |
+
self.w = nn.Parameter(torch.ones(d))
|
| 79 |
+
self.eps = eps
|
| 80 |
+
|
| 81 |
+
def forward(self, x):
|
| 82 |
+
x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
|
| 83 |
+
return x * self.w
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class BitLinear(nn.Linear):
|
| 87 |
+
"""nn.Linear drop-in. weight_bits='ternary' -> W1.58A8 QAT; 'fp' -> plain fp.
|
| 88 |
+
|
| 89 |
+
pre_norm=True inserts an RMSNorm before the projection (the "extra RMSNorm"
|
| 90 |
+
stabilizer, arXiv:2505.08823). For 'fp' islands we keep it off so the state
|
| 91 |
+
dict matches the original SlothE exactly (teacher loading).
|
| 92 |
+
"""
|
| 93 |
+
|
| 94 |
+
def __init__(self, in_f, out_f, bias=False, weight_bits="ternary",
|
| 95 |
+
act_bits=8, weight_quant="median", pre_norm=True):
|
| 96 |
+
super().__init__(in_f, out_f, bias=bias)
|
| 97 |
+
self.weight_bits = weight_bits
|
| 98 |
+
self.act_bits = act_bits
|
| 99 |
+
self.wq_mode = weight_quant
|
| 100 |
+
self.pre = RMSNorm(in_f) if (pre_norm and weight_bits == "ternary") else None
|
| 101 |
+
self.register_buffer("quant_alpha", torch.tensor(1.0), persistent=True)
|
| 102 |
+
|
| 103 |
+
def set_quant_alpha(self, a):
|
| 104 |
+
self.quant_alpha.fill_(float(a))
|
| 105 |
+
|
| 106 |
+
def forward(self, x):
|
| 107 |
+
if self.pre is not None:
|
| 108 |
+
x = self.pre(x)
|
| 109 |
+
if self.weight_bits == "fp":
|
| 110 |
+
return F.linear(x, self.weight, self.bias)
|
| 111 |
+
a = self.quant_alpha # tensor keeps torch.compile stable across anneal
|
| 112 |
+
if self.act_bits:
|
| 113 |
+
x = _ste(x, activation_quant(x), a)
|
| 114 |
+
w = _ste(self.weight, weight_quant_ternary(self.weight, self.wq_mode), a)
|
| 115 |
+
return F.linear(x, w, self.bias)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
# ======================================================================
|
| 119 |
+
# Model — mirrors reproduce/train_slothlm_e.py, Linear -> BitLinear
|
| 120 |
+
# ======================================================================
|
| 121 |
+
|
| 122 |
+
def rope(x, pos, dim):
|
| 123 |
+
half = dim // 2
|
| 124 |
+
freq = 1.0 / (10000 ** (torch.arange(0, half, device=x.device) / half))
|
| 125 |
+
ang = pos[:, None].float() * freq[None, :]
|
| 126 |
+
cos = torch.cat([ang.cos(), ang.cos()], -1)[None, None]
|
| 127 |
+
sin = torch.cat([ang.sin(), ang.sin()], -1)[None, None]
|
| 128 |
+
x1, x2 = x[..., :half], x[..., half:]
|
| 129 |
+
rot = torch.cat([-x2, x1], -1)
|
| 130 |
+
return x * cos + rot * sin
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _lin(in_f, out_f, fp, wq, pre_norm):
|
| 134 |
+
return BitLinear(in_f, out_f, bias=False,
|
| 135 |
+
weight_bits="fp" if fp else "ternary",
|
| 136 |
+
weight_quant=wq, pre_norm=pre_norm)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
class Attn(nn.Module):
|
| 140 |
+
def __init__(self, dim, heads, kv, fp, wq, pre_norm):
|
| 141 |
+
super().__init__()
|
| 142 |
+
self.h, self.kv, self.dh = heads, kv, dim // heads
|
| 143 |
+
self.q = _lin(dim, heads * self.dh, fp, wq, pre_norm)
|
| 144 |
+
self.k = _lin(dim, kv * self.dh, fp, wq, pre_norm)
|
| 145 |
+
self.v = _lin(dim, kv * self.dh, fp, wq, pre_norm)
|
| 146 |
+
self.o = _lin(heads * self.dh, dim, fp, wq, pre_norm)
|
| 147 |
+
self.qn = RMSNorm(self.dh)
|
| 148 |
+
self.kn = RMSNorm(self.dh)
|
| 149 |
+
|
| 150 |
+
def forward(self, x, pos, amask):
|
| 151 |
+
B, T, _ = x.shape
|
| 152 |
+
q = self.q(x).view(B, T, self.h, self.dh).transpose(1, 2)
|
| 153 |
+
k = self.k(x).view(B, T, self.kv, self.dh).transpose(1, 2)
|
| 154 |
+
v = self.v(x).view(B, T, self.kv, self.dh).transpose(1, 2)
|
| 155 |
+
q, k = self.qn(q), self.kn(k)
|
| 156 |
+
q, k = rope(q, pos, self.dh), rope(k, pos, self.dh)
|
| 157 |
+
rep = self.h // self.kv
|
| 158 |
+
k = k.repeat_interleave(rep, 1)
|
| 159 |
+
v = v.repeat_interleave(rep, 1)
|
| 160 |
+
o = F.scaled_dot_product_attention(q, k, v, attn_mask=amask[:, None, None, :])
|
| 161 |
+
o = o.transpose(1, 2).reshape(B, T, -1)
|
| 162 |
+
return self.o(o)
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
class SwiGLU(nn.Module):
|
| 166 |
+
def __init__(self, dim, hidden, fp, wq, pre_norm):
|
| 167 |
+
super().__init__()
|
| 168 |
+
self.w1 = _lin(dim, hidden, fp, wq, pre_norm)
|
| 169 |
+
self.w3 = _lin(dim, hidden, fp, wq, pre_norm)
|
| 170 |
+
self.w2 = _lin(hidden, dim, fp, wq, pre_norm)
|
| 171 |
+
|
| 172 |
+
def forward(self, x):
|
| 173 |
+
return self.w2(F.silu(self.w1(x)) * self.w3(x))
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
class Block(nn.Module):
|
| 177 |
+
def __init__(self, dim, heads, kv, ffn, fp, wq, pre_norm):
|
| 178 |
+
super().__init__()
|
| 179 |
+
self.n1 = RMSNorm(dim)
|
| 180 |
+
self.attn = Attn(dim, heads, kv, fp, wq, pre_norm)
|
| 181 |
+
self.n2 = RMSNorm(dim)
|
| 182 |
+
self.ffn = SwiGLU(dim, ffn, fp, wq, pre_norm)
|
| 183 |
+
|
| 184 |
+
def forward(self, x, pos, amask):
|
| 185 |
+
x = x + self.attn(self.n1(x), pos, amask)
|
| 186 |
+
x = x + self.ffn(self.n2(x))
|
| 187 |
+
return x
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
class SlothE_T(nn.Module):
|
| 191 |
+
"""Ternary SlothLM-E. embed + head + first/last `fp_boundary` blocks stay fp."""
|
| 192 |
+
|
| 193 |
+
def __init__(self, n_syl, n_char, dim=320, depth=16, heads=8, kv=2, ffn=880,
|
| 194 |
+
embed_norm=True, weight_bits="ternary", weight_quant="median",
|
| 195 |
+
fp_boundary=1, pre_norm=True, act_bits=8, char_hints=False, tie_hints=False):
|
| 196 |
+
super().__init__()
|
| 197 |
+
self.embed = nn.Embedding(n_syl, dim) # fp island
|
| 198 |
+
self.hint_tied = char_hints and tie_hints
|
| 199 |
+
self.hint_embed = nn.Embedding(n_char + 1, dim, padding_idx=0) if (char_hints and not tie_hints) else None
|
| 200 |
+
self.hint_none = nn.Parameter(torch.zeros(dim)) if self.hint_tied else None
|
| 201 |
+
self.embed_norm = RMSNorm(dim) if embed_norm else None
|
| 202 |
+
blocks = []
|
| 203 |
+
for i in range(depth):
|
| 204 |
+
is_fp = (weight_bits == "fp") or i < fp_boundary or i >= depth - fp_boundary
|
| 205 |
+
blocks.append(Block(dim, heads, kv, ffn, is_fp, weight_quant, pre_norm))
|
| 206 |
+
self.blocks = nn.ModuleList(blocks)
|
| 207 |
+
self.norm = RMSNorm(dim)
|
| 208 |
+
self.head = nn.Linear(dim, n_char, bias=False) # fp island (int8 at deploy)
|
| 209 |
+
self.apply(self._init)
|
| 210 |
+
|
| 211 |
+
def _init(self, m):
|
| 212 |
+
if isinstance(m, nn.Linear):
|
| 213 |
+
nn.init.normal_(m.weight, std=0.02)
|
| 214 |
+
elif isinstance(m, nn.Embedding):
|
| 215 |
+
nn.init.normal_(m.weight, std=0.02)
|
| 216 |
+
|
| 217 |
+
def set_quant_alpha(self, a):
|
| 218 |
+
for m in self.modules():
|
| 219 |
+
if isinstance(m, BitLinear):
|
| 220 |
+
m.set_quant_alpha(a)
|
| 221 |
+
|
| 222 |
+
def forward(self, syl, amask, hints=None):
|
| 223 |
+
pos = torch.arange(syl.shape[1], device=syl.device)
|
| 224 |
+
x = self.embed(syl)
|
| 225 |
+
if hints is not None and self.hint_tied:
|
| 226 |
+
has = (hints > 0).unsqueeze(-1).to(x.dtype)
|
| 227 |
+
he = F.embedding((hints - 1).clamp(min=0), self.head.weight)
|
| 228 |
+
x = x + he * has + self.hint_none * (1.0 - has)
|
| 229 |
+
elif self.hint_embed is not None and hints is not None:
|
| 230 |
+
x = x + self.hint_embed(hints)
|
| 231 |
+
if self.embed_norm is not None:
|
| 232 |
+
x = self.embed_norm(x)
|
| 233 |
+
for b in self.blocks:
|
| 234 |
+
x = b(x, pos, amask)
|
| 235 |
+
return self.head(self.norm(x))
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
# ======================================================================
|
| 239 |
+
# Data (reuses the AlignedBin format from the original pipeline)
|
| 240 |
+
# ======================================================================
|
| 241 |
+
|
| 242 |
+
class AlignedBin(Dataset):
|
| 243 |
+
CTX_MAX = 12
|
| 244 |
+
def __init__(self, path, ctx_p=0.0, typo=None, typo_p=0.0, teacher_path=None):
|
| 245 |
+
self.data = np.fromfile(path, dtype=np.uint16)
|
| 246 |
+
self.tdata = np.fromfile(teacher_path, dtype=np.uint16) if teacher_path else None
|
| 247 |
+
self.ctx_p, self.typo, self.typo_p = ctx_p, typo, typo_p
|
| 248 |
+
self.idx = []
|
| 249 |
+
i, d = 0, self.data
|
| 250 |
+
while i < len(d):
|
| 251 |
+
n = int(d[i]); self.idx.append((i + 1, n)); i += 1 + 2 * n
|
| 252 |
+
print(f"{len(self.idx)} aligned pairs")
|
| 253 |
+
def __len__(self): return len(self.idx)
|
| 254 |
+
def __getitem__(self, k):
|
| 255 |
+
s, n = self.idx[k]
|
| 256 |
+
syl = self.data[s:s + n].astype(np.int64)
|
| 257 |
+
tsyl = (self.tdata[s:s + n].astype(np.int64)
|
| 258 |
+
if self.tdata is not None else syl.copy())
|
| 259 |
+
chr_ = self.data[s + n:s + 2 * n].astype(np.int64)
|
| 260 |
+
chr_[chr_ == 65535] = -100
|
| 261 |
+
if self.typo is not None and self.typo_p > 0 and np.random.rand() < self.typo_p:
|
| 262 |
+
zh = np.nonzero(chr_ >= 0)[0]
|
| 263 |
+
if len(zh):
|
| 264 |
+
i = int(np.random.choice(zh)); nb = self.typo.get(int(syl[i]))
|
| 265 |
+
if nb: syl[i] = int(nb[np.random.randint(len(nb))])
|
| 266 |
+
forced = np.zeros(n, dtype=np.int64)
|
| 267 |
+
if self.ctx_p > 0 and k >= 2 and np.random.rand() < self.ctx_p:
|
| 268 |
+
ps, pn = self.idx[k - 2]
|
| 269 |
+
pch = self.data[ps + pn:ps + 2 * pn].astype(np.int64)
|
| 270 |
+
pch = pch[pch != 65535][-self.CTX_MAX:]
|
| 271 |
+
if len(pch):
|
| 272 |
+
L = len(pch)
|
| 273 |
+
syl = np.concatenate([np.zeros(L, dtype=np.int64), syl])
|
| 274 |
+
tsyl = np.concatenate([np.zeros(L, dtype=np.int64), tsyl])
|
| 275 |
+
chr_ = np.concatenate([np.full(L, -100, dtype=np.int64), chr_])
|
| 276 |
+
forced = np.concatenate([pch + 1, forced])
|
| 277 |
+
return (torch.from_numpy(syl), torch.from_numpy(tsyl),
|
| 278 |
+
torch.from_numpy(chr_), torch.from_numpy(forced))
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def collate(batch, pad=0):
|
| 282 |
+
n = max(len(s) for s, _, _, _ in batch); B = len(batch)
|
| 283 |
+
syl = torch.zeros(B, n, dtype=torch.long)
|
| 284 |
+
tsyl = torch.zeros(B, n, dtype=torch.long)
|
| 285 |
+
chr_ = torch.full((B, n), -100, dtype=torch.long)
|
| 286 |
+
mask = torch.zeros(B, n, dtype=torch.bool)
|
| 287 |
+
forced = torch.zeros(B, n, dtype=torch.long)
|
| 288 |
+
for i, (sq, tq, c, f) in enumerate(batch):
|
| 289 |
+
syl[i, :len(sq)] = sq; tsyl[i, :len(tq)] = tq
|
| 290 |
+
chr_[i, :len(c)] = c; mask[i, :len(sq)] = True; forced[i, :len(f)] = f
|
| 291 |
+
return syl, tsyl, chr_, mask, forced
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
def masked_ce(logits, target, syl, legal_mask, eps):
|
| 295 |
+
"""CE + label-smoothing restricted to the per-position legal char set (mirrors
|
| 296 |
+
serve-time masking): the ternary net optimizes only the ~1-50 legal chars eval
|
| 297 |
+
scores on, not all 8342. Positions whose gold char is not legal (train/serve
|
| 298 |
+
skew) are dropped so the loss never goes -inf. logits (B,T,V); target (B,T,-100
|
| 299 |
+
pad); syl (B,T) input syllable ids; legal_mask (n_syl,V) bool."""
|
| 300 |
+
lm = legal_mask[syl]
|
| 301 |
+
logits = logits.masked_fill(~lm, float("-inf"))
|
| 302 |
+
logp = F.log_softmax(logits, dim=-1)
|
| 303 |
+
tgt = target.clamp(min=0)
|
| 304 |
+
gold_legal = lm.gather(-1, tgt.unsqueeze(-1)).squeeze(-1)
|
| 305 |
+
valid = (target != -100) & gold_legal
|
| 306 |
+
ce_hard = -logp.gather(-1, tgt.unsqueeze(-1)).squeeze(-1)
|
| 307 |
+
nlegal = lm.sum(-1).clamp(min=1)
|
| 308 |
+
ls_term = -(logp.masked_fill(~lm, 0.0)).sum(-1) / nlegal
|
| 309 |
+
loss = (1.0 - eps) * ce_hard + eps * ls_term
|
| 310 |
+
return loss[valid].mean()
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def load_teacher(path, dev):
|
| 314 |
+
"""Load a frozen all-fp SlothE_T teacher (KD). Mirrors gate build()."""
|
| 315 |
+
ck = torch.load(os.path.join(path, "slothe.pt"), map_location="cpu")
|
| 316 |
+
c = ck["config"]
|
| 317 |
+
m = SlothE_T(c["n_syl"], c["n_char"], c["dim"], c["depth"], c["heads"],
|
| 318 |
+
c["kv"], c["ffn"], embed_norm=c.get("embed_norm", False),
|
| 319 |
+
weight_bits=c.get("weight_bits") or "fp",
|
| 320 |
+
weight_quant=c.get("weight_quant", "median"),
|
| 321 |
+
fp_boundary=c.get("fp_boundary", 1),
|
| 322 |
+
pre_norm=c.get("pre_norm", True), act_bits=c.get("act_bits", 8),
|
| 323 |
+
char_hints=c.get("char_hints", False),
|
| 324 |
+
tie_hints=c.get("tie_hints", False))
|
| 325 |
+
m.load_state_dict(ck["model"], strict=False) # teacher predates quant_alpha buffers (unused for fp)
|
| 326 |
+
m.set_quant_alpha(1.0)
|
| 327 |
+
m.eval().to(dev)
|
| 328 |
+
for prm in m.parameters():
|
| 329 |
+
prm.requires_grad_(False)
|
| 330 |
+
return m
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def distill_loss(logits, t_logits, target, alpha, temp):
|
| 334 |
+
"""(1-alpha)*CE + alpha*KL(student||teacher) at temperature `temp`."""
|
| 335 |
+
V = logits.shape[-1]
|
| 336 |
+
flat = target.reshape(-1)
|
| 337 |
+
ce = F.cross_entropy(logits.reshape(-1, V), flat, ignore_index=-100)
|
| 338 |
+
keep = flat != -100
|
| 339 |
+
s = logits.reshape(-1, V)[keep]
|
| 340 |
+
t = t_logits.reshape(-1, V)[keep]
|
| 341 |
+
kl = F.kl_div(F.log_softmax(s / temp, -1), F.softmax(t / temp, -1),
|
| 342 |
+
reduction="batchmean") * (temp * temp)
|
| 343 |
+
return (1.0 - alpha) * ce + alpha * kl, ce.detach(), kl.detach()
|
| 344 |
+
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
def _dachen_key_adjacency():
|
| 348 |
+
"""Bopomofo symbol -> set of symbols on physically 8-adjacent keys, per the
|
| 349 |
+
TAAI-2024 keyboard 8-adjacency error model (Li/Yeh/Chang, 大城市 = 284t/6g4):
|
| 350 |
+
a real mis-key lands on one of the 8 neighbours in the QWERTY 3x3 block
|
| 351 |
+
('h' -> t/y/u/g/j/b/n/m). Used to make typo-noise reflect finger slips
|
| 352 |
+
instead of arbitrary edit-distance-1 substitutions."""
|
| 353 |
+
rows = ["1234567890-=", "qwertyuiop[]", "asdfghjkl;'", "zxcvbnm,./"]
|
| 354 |
+
pos = {k: (r, c) for r, row in enumerate(rows) for c, k in enumerate(row)}
|
| 355 |
+
key_nb = {}
|
| 356 |
+
for k, (r, c) in pos.items():
|
| 357 |
+
key_nb[k] = {k2 for k2, (r2, c2) in pos.items()
|
| 358 |
+
if abs(r2 - r) <= 1 and abs(c2 - c) <= 1 and k2 != k}
|
| 359 |
+
DACHEN = {"1": "\u3105", "q": "\u3106", "a": "\u3107", "z": "\u3108",
|
| 360 |
+
"2": "\u3109", "w": "\u310a", "s": "\u310b", "x": "\u310c",
|
| 361 |
+
"e": "\u310d", "d": "\u310e", "c": "\u310f", "r": "\u3110",
|
| 362 |
+
"f": "\u3111", "v": "\u3112", "5": "\u3113", "t": "\u3114",
|
| 363 |
+
"g": "\u3115", "b": "\u3116", "y": "\u3117", "h": "\u3118",
|
| 364 |
+
"n": "\u3119", "u": "\u3127", "j": "\u3128", "m": "\u3129",
|
| 365 |
+
"8": "\u311a", "i": "\u311b", "k": "\u311c", ",": "\u311d",
|
| 366 |
+
"9": "\u311e", "o": "\u311f", "l": "\u3120", ".": "\u3121",
|
| 367 |
+
"0": "\u3122", "p": "\u3123", ";": "\u3124", "/": "\u3125",
|
| 368 |
+
"-": "\u3126"}
|
| 369 |
+
sym_key = {v: k for k, v in DACHEN.items()}
|
| 370 |
+
return {sym: {DACHEN[k2] for k2 in key_nb.get(k, ()) if k2 in DACHEN}
|
| 371 |
+
for sym, k in sym_key.items()}
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
def build_typo_neighbors(syl_vocab):
|
| 375 |
+
TONES = set("ˊˇˋ˙"); by_key = {}
|
| 376 |
+
sym_adj = _dachen_key_adjacency() # TAAI 8-adjacency slip model
|
| 377 |
+
for sy, i in syl_vocab.items():
|
| 378 |
+
if sy.startswith("<"): continue
|
| 379 |
+
tone = "".join(c for c in sy if c in TONES); base = tuple(c for c in sy if c not in TONES)
|
| 380 |
+
by_key.setdefault((tone, len(base)), []).append((base, i))
|
| 381 |
+
out = {}
|
| 382 |
+
for sy, i in syl_vocab.items():
|
| 383 |
+
if sy.startswith("<"): continue
|
| 384 |
+
tone = "".join(c for c in sy if c in TONES); base = tuple(c for c in sy if c not in TONES); nb = []
|
| 385 |
+
for dl in (-1, 0, 1):
|
| 386 |
+
for other, j in by_key.get((tone, len(base) + dl), []):
|
| 387 |
+
if j == i: continue
|
| 388 |
+
a, b = base, other
|
| 389 |
+
if len(a) == len(b):
|
| 390 |
+
diffs = [(x, y) for x, y in zip(a, b) if x != y]
|
| 391 |
+
if len(diffs) == 1 and diffs[0][1] in sym_adj.get(diffs[0][0], ()):
|
| 392 |
+
nb.append(j) # only physically-adjacent-key slips
|
| 393 |
+
else:
|
| 394 |
+
s1, s2 = (a, b) if len(a) < len(b) else (b, a)
|
| 395 |
+
for pp in range(len(s2)):
|
| 396 |
+
if s2[:pp] + s2[pp+1:] == s1: nb.append(j); break
|
| 397 |
+
if nb: out[i] = nb
|
| 398 |
+
return out
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
def main():
|
| 402 |
+
ap = argparse.ArgumentParser()
|
| 403 |
+
ap.add_argument("--data", required=True)
|
| 404 |
+
ap.add_argument("--vocab", default="syl_vocab.json")
|
| 405 |
+
ap.add_argument("--tokenizer", default="tokenizer")
|
| 406 |
+
ap.add_argument("--out", default="slothe_t")
|
| 407 |
+
# architecture (recommended ~20M ternary: dim320 depth16 heads8 kv2 ffn880)
|
| 408 |
+
ap.add_argument("--dim", type=int, default=320)
|
| 409 |
+
ap.add_argument("--depth", type=int, default=16)
|
| 410 |
+
ap.add_argument("--heads", type=int, default=8)
|
| 411 |
+
ap.add_argument("--kv-heads", type=int, default=2)
|
| 412 |
+
ap.add_argument("--ffn", type=int, default=880)
|
| 413 |
+
ap.add_argument("--embed-norm", action="store_true")
|
| 414 |
+
# quantization
|
| 415 |
+
ap.add_argument("--quant", choices=["ternary", "fp"], default="ternary")
|
| 416 |
+
ap.add_argument("--weight-quant", choices=["median", "mean"], default="median")
|
| 417 |
+
ap.add_argument("--fp-boundary", type=int, default=1)
|
| 418 |
+
ap.add_argument("--pre-norm", action="store_true", default=True)
|
| 419 |
+
ap.add_argument("--no-pre-norm", dest="pre_norm", action="store_false")
|
| 420 |
+
ap.add_argument("--act-bits", type=int, default=8)
|
| 421 |
+
ap.add_argument("--anneal-frac", type=float, default=0.15,
|
| 422 |
+
help="ramp quant_alpha 0->1 over this fraction of steps")
|
| 423 |
+
# distillation
|
| 424 |
+
ap.add_argument("--teacher", default="", help="dir with a fp teacher slothe.pt")
|
| 425 |
+
ap.add_argument("--teacher-data", default="", help="exact-syllable .bin for cross-input distillation (teacher input)")
|
| 426 |
+
ap.add_argument("--distill-alpha", type=float, default=0.7)
|
| 427 |
+
ap.add_argument("--distill-temp", type=float, default=2.0)
|
| 428 |
+
# optimization (retuned for small ternary: higher LR ok, see 2407.09527)
|
| 429 |
+
ap.add_argument("--batch", type=int, default=384)
|
| 430 |
+
ap.add_argument("--epochs", type=float, default=8.0)
|
| 431 |
+
ap.add_argument("--lr", type=float, default=2.5e-3)
|
| 432 |
+
ap.add_argument("--weight-decay", type=float, default=0.1)
|
| 433 |
+
ap.add_argument("--steps", type=int, default=0)
|
| 434 |
+
ap.add_argument("--resume", default="")
|
| 435 |
+
ap.add_argument("--char-hints", action="store_true")
|
| 436 |
+
ap.add_argument("--tie-hints", action="store_true")
|
| 437 |
+
ap.add_argument("--typo-noise", type=float, default=0.0)
|
| 438 |
+
ap.add_argument("--compile", action="store_true", help="torch.compile the model (compile BEFORE DDP wrap)")
|
| 439 |
+
ap.add_argument("--compile-mode", default="reduce-overhead", help="reduce-overhead | cudagraphs | default")
|
| 440 |
+
ap.add_argument("--legal-mask", default="", help="npz (n_syl,V) legal-char mask -> mask CE + LS to the legal set")
|
| 441 |
+
ap.add_argument("--label-smoothing", type=float, default=0.0, help="teacher-free soft targets for direct (pure-CE) training")
|
| 442 |
+
ap.add_argument("--context", type=float, default=0.0)
|
| 443 |
+
ap.add_argument("--save-every", type=int, default=0, help="snapshot {out}_ep{N} every N epochs (for accuracy-vs-epoch curve)")
|
| 444 |
+
args = ap.parse_args()
|
| 445 |
+
|
| 446 |
+
from transformers import AutoTokenizer
|
| 447 |
+
tok = AutoTokenizer.from_pretrained(args.tokenizer)
|
| 448 |
+
syl_vocab = json.load(open(args.vocab, encoding="utf-8"))
|
| 449 |
+
ddp = int(os.environ.get("WORLD_SIZE", 1)) > 1
|
| 450 |
+
if ddp:
|
| 451 |
+
dist.init_process_group("nccl")
|
| 452 |
+
local_rank = int(os.environ["LOCAL_RANK"]); rank = dist.get_rank(); world = dist.get_world_size()
|
| 453 |
+
torch.cuda.set_device(local_rank); dev = f"cuda:{local_rank}"
|
| 454 |
+
else:
|
| 455 |
+
rank, local_rank, world = 0, 0, 1
|
| 456 |
+
dev = "cuda" if torch.cuda.is_available() else "cpu"
|
| 457 |
+
is_main = (rank == 0)
|
| 458 |
+
if dev.startswith("cuda"):
|
| 459 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
| 460 |
+
torch.backends.cudnn.allow_tf32 = True
|
| 461 |
+
torch.backends.cudnn.benchmark = True
|
| 462 |
+
|
| 463 |
+
typo_map = build_typo_neighbors(syl_vocab) if args.typo_noise > 0 else None
|
| 464 |
+
ds = AlignedBin(args.data, ctx_p=args.context, typo=typo_map, typo_p=args.typo_noise,
|
| 465 |
+
teacher_path=(args.teacher_data or None))
|
| 466 |
+
sampler = torch.utils.data.DistributedSampler(ds, shuffle=True) if ddp else None
|
| 467 |
+
dl = DataLoader(ds, batch_size=args.batch, shuffle=(sampler is None), sampler=sampler,
|
| 468 |
+
num_workers=8, collate_fn=collate, pin_memory=True, drop_last=True)
|
| 469 |
+
model = SlothE_T(len(syl_vocab), len(tok), args.dim, args.depth, args.heads,
|
| 470 |
+
args.kv_heads, args.ffn, embed_norm=args.embed_norm,
|
| 471 |
+
weight_bits=args.quant, weight_quant=args.weight_quant,
|
| 472 |
+
fp_boundary=args.fp_boundary, pre_norm=args.pre_norm,
|
| 473 |
+
act_bits=args.act_bits, char_hints=args.char_hints, tie_hints=args.tie_hints).to(dev)
|
| 474 |
+
if args.resume:
|
| 475 |
+
model.load_state_dict(torch.load(args.resume, map_location=dev)["model"])
|
| 476 |
+
if int(os.environ.get("RANK","0"))==0: print("resumed from "+args.resume, flush=True)
|
| 477 |
+
np_ = sum(p.numel() for p in model.parameters())
|
| 478 |
+
print(f"SlothLM-E-T {np_/1e6:.1f}M params | quant={args.quant} "
|
| 479 |
+
f"wq={args.weight_quant} fp_boundary={args.fp_boundary} pre_norm={args.pre_norm}")
|
| 480 |
+
|
| 481 |
+
teacher = load_teacher(args.teacher, dev) if args.teacher else None
|
| 482 |
+
legal_mask = (torch.from_numpy(np.load(args.legal_mask)["mask"]).to(dev)
|
| 483 |
+
if args.legal_mask else None)
|
| 484 |
+
raw_model = model
|
| 485 |
+
if args.compile:
|
| 486 |
+
model = (torch.compile(model, backend="cudagraphs")
|
| 487 |
+
if args.compile_mode == "cudagraphs"
|
| 488 |
+
else torch.compile(model, mode=args.compile_mode))
|
| 489 |
+
if ddp:
|
| 490 |
+
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[local_rank], broadcast_buffers=False)
|
| 491 |
+
|
| 492 |
+
opt = torch.optim.AdamW(model.parameters(), lr=args.lr, betas=(0.9, 0.95),
|
| 493 |
+
weight_decay=args.weight_decay)
|
| 494 |
+
total = args.steps or int(len(dl) * args.epochs)
|
| 495 |
+
sched = torch.optim.lr_scheduler.OneCycleLR(opt, args.lr, total_steps=total,
|
| 496 |
+
pct_start=0.03)
|
| 497 |
+
anneal_steps = max(1, int(args.anneal_frac * total))
|
| 498 |
+
|
| 499 |
+
model.train()
|
| 500 |
+
step = 0
|
| 501 |
+
for ep in range(math.ceil(args.epochs)):
|
| 502 |
+
if ddp: sampler.set_epoch(ep)
|
| 503 |
+
for syl, tsyl, chr_, mask, forced in dl:
|
| 504 |
+
syl, tsyl, chr_, mask, forced = (syl.to(dev), tsyl.to(dev),
|
| 505 |
+
chr_.to(dev), mask.to(dev), forced.to(dev))
|
| 506 |
+
hints = None
|
| 507 |
+
if args.char_hints:
|
| 508 |
+
reveal_p = torch.rand(syl.shape[0], 1, device=dev) * 0.3
|
| 509 |
+
reveal = (torch.rand(syl.shape, device=dev) < reveal_p) & (chr_ >= 0)
|
| 510 |
+
hints = torch.where(reveal, chr_ + 1, torch.zeros_like(chr_))
|
| 511 |
+
hints = torch.where(forced > 0, forced, hints)
|
| 512 |
+
raw_model.set_quant_alpha(1.0 if args.resume else min(1.0, step / anneal_steps))
|
| 513 |
+
with torch.autocast(dev, dtype=torch.bfloat16, enabled=dev == "cuda"):
|
| 514 |
+
logits = model(syl, mask, hints)
|
| 515 |
+
if teacher is not None:
|
| 516 |
+
with torch.no_grad():
|
| 517 |
+
t_logits = teacher(tsyl, mask, hints) # exact syllables
|
| 518 |
+
loss, ce, kl = distill_loss(logits, t_logits, chr_,
|
| 519 |
+
args.distill_alpha, args.distill_temp)
|
| 520 |
+
elif legal_mask is not None:
|
| 521 |
+
loss = masked_ce(logits, chr_, syl, legal_mask, args.label_smoothing)
|
| 522 |
+
ce = kl = loss.detach()
|
| 523 |
+
else:
|
| 524 |
+
loss = F.cross_entropy(logits.reshape(-1, logits.shape[-1]),
|
| 525 |
+
chr_.reshape(-1), ignore_index=-100,
|
| 526 |
+
label_smoothing=args.label_smoothing)
|
| 527 |
+
ce = kl = loss.detach()
|
| 528 |
+
opt.zero_grad(set_to_none=True)
|
| 529 |
+
loss.backward()
|
| 530 |
+
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
| 531 |
+
opt.step(); sched.step(); step += 1
|
| 532 |
+
if step % 50 == 0 and is_main:
|
| 533 |
+
a = min(1.0, step / anneal_steps)
|
| 534 |
+
print(f"step {step}/{total} loss {loss.item():.3f} "
|
| 535 |
+
f"ce {ce.item():.3f} kl {kl.item():.3f} alpha {a:.2f}", flush=True)
|
| 536 |
+
if step >= total:
|
| 537 |
+
break
|
| 538 |
+
if args.save_every and (ep + 1) % args.save_every == 0:
|
| 539 |
+
if is_main:
|
| 540 |
+
_snap = f"{args.out}_ep{ep + 1}"
|
| 541 |
+
os.makedirs(_snap, exist_ok=True)
|
| 542 |
+
torch.save({"model": raw_model.state_dict(),
|
| 543 |
+
"config": {"n_syl": len(syl_vocab), "n_char": len(tok),
|
| 544 |
+
"dim": args.dim, "depth": args.depth,
|
| 545 |
+
"heads": args.heads, "kv": args.kv_heads,
|
| 546 |
+
"ffn": args.ffn, "embed_norm": args.embed_norm,
|
| 547 |
+
"weight_bits": args.quant,
|
| 548 |
+
"weight_quant": args.weight_quant,
|
| 549 |
+
"fp_boundary": args.fp_boundary,
|
| 550 |
+
"pre_norm": args.pre_norm,
|
| 551 |
+
"act_bits": args.act_bits,
|
| 552 |
+
"char_hints": args.char_hints,
|
| 553 |
+
"tie_hints": args.tie_hints}},
|
| 554 |
+
os.path.join(_snap, "slothe.pt"))
|
| 555 |
+
json.dump(syl_vocab, open(os.path.join(_snap, "syl_vocab.json"),
|
| 556 |
+
"w", encoding="utf-8"), ensure_ascii=False)
|
| 557 |
+
print(f"snapshot saved to {_snap}", flush=True)
|
| 558 |
+
if ddp: dist.barrier()
|
| 559 |
+
if step >= total:
|
| 560 |
+
break
|
| 561 |
+
|
| 562 |
+
raw_model.set_quant_alpha(1.0)
|
| 563 |
+
if ddp: dist.barrier()
|
| 564 |
+
if not is_main:
|
| 565 |
+
if ddp: dist.destroy_process_group()
|
| 566 |
+
return
|
| 567 |
+
os.makedirs(args.out, exist_ok=True)
|
| 568 |
+
torch.save({"model": raw_model.state_dict(),
|
| 569 |
+
"config": {"n_syl": len(syl_vocab), "n_char": len(tok),
|
| 570 |
+
"dim": args.dim, "depth": args.depth, "heads": args.heads,
|
| 571 |
+
"kv": args.kv_heads, "ffn": args.ffn,
|
| 572 |
+
"embed_norm": args.embed_norm, "weight_bits": args.quant,
|
| 573 |
+
"weight_quant": args.weight_quant,
|
| 574 |
+
"fp_boundary": args.fp_boundary, "pre_norm": args.pre_norm,
|
| 575 |
+
"act_bits": args.act_bits, "char_hints": args.char_hints,
|
| 576 |
+
"tie_hints": args.tie_hints}},
|
| 577 |
+
os.path.join(args.out, "slothe.pt"))
|
| 578 |
+
_sd = raw_model.state_dict()
|
| 579 |
+
json.dump(syl_vocab, open(os.path.join(args.out, "syl_vocab.json"), "w",
|
| 580 |
+
encoding="utf-8"), ensure_ascii=False)
|
| 581 |
+
print(f"saved to {args.out}")
|
| 582 |
+
|
| 583 |
+
|
| 584 |
+
if __name__ == "__main__":
|
| 585 |
+
main()
|