kgrabko commited on
Commit
d573835
路
verified 路
1 Parent(s): cc734a4

Create JiRackTernaryPyTorch_70b.py

Browse files
prepared_sft_data/JiRackTernaryPyTorch_70b.py ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ self.in_features, self.out_features = in_features, out_features
3
+ self.register_buffer("packed", None)
4
+ self.register_buffer("scale", None)
5
+ self.register_buffer("orig_shape", None)
6
+ def unpack(self):
7
+ if self.packed is None: return None
8
+ p = self.packed.to(torch.int32)
9
+ b = torch.stack([(p >> 6) & 3, (p >> 4) & 3, (p >> 2) & 3, p & 3], dim=1).view(-1)
10
+ shape = self.orig_shape if self.orig_shape is not None else torch.tensor([self.out_features, self.in_features])
11
+ w = (b[:shape.numel()].to(torch.float16) - 1.0)
12
+ return w.view(int(shape[0]), int(shape[1])) * self.scale
13
+ def forward(self, x):
14
+ w = self.unpack()
15
+ if w is None: return F.linear(x, torch.zeros(self.out_features, self.in_features, device=x.device, dtype=x.dtype))
16
+ x_norm = x - x.mean(dim=-1, keepdim=True)
17
+ x_scale = x_norm.abs().max(dim=-1, keepdim=True).values.clamp(min=1e-5)
18
+ return F.linear((x_norm / x_scale).to(w.dtype), w) * (x_scale * 0.67)
19
+
20
+ class RMSNorm(nn.Module):
21
+ def __init__(self, dim, eps=1e-5):
22
+ super().__init__()
23
+ self.eps, self.weight = eps, nn.Parameter(torch.ones(dim))
24
+ def forward(self, x):
25
+ v = x.to(torch.float32).pow(2).mean(-1, keepdim=True)
26
+ return (x.to(torch.float32) * torch.rsqrt(v + self.eps) * self.weight.to(torch.float32)).to(x.dtype)
27
+
28
+ def apply_rotary_emb(x, freqs):
29
+ cos, sin = freqs.cos().unsqueeze(0).unsqueeze(1), freqs.sin().unsqueeze(0).unsqueeze(1)
30
+ x1, x2 = x[..., :x.shape[-1]//2], x[..., x.shape[-1]//2:]
31
+ rh = torch.cat((-x2, x1), dim=-1)
32
+ return (x * torch.cat([cos,cos],-1)) + (rh * torch.cat([sin,sin],-1))
33
+
34
+ class JiRackAttention(nn.Module):
35
+ def __init__(self):
36
+ super().__init__()
37
+ self.q_proj = JiRackBitLinear(8192, 8192)
38
+
39
+ self.o_proj = JiRackBitLinear(8192, 8192)
40
+ def forward(self, x, freqs):
41
+ bsz, q_len, _ = x.shape
42
+ q = self.q_proj(x).view(bsz, q_len, 64, 128).transpose(1, 2)
43
+ k = self.k_proj(x).view(bsz, q_len, 8, 128).transpose(1, 2)
44
+ v = self.v_proj(x).view(bsz, q_len, 8, 128).transpose(1, 2)
45
+ q, k = apply_rotary_emb(q, freqs), apply_rotary_emb(k, freqs)
46
+ k = k.repeat_interleave(8, dim=1)
47
+ v = v.repeat_interleave(8, dim=1)
48
+ attn = F.scaled_dot_product_attention(q, k, v, is_causal=True)
49
+ return self.o_proj(attn.transpose(1, 2).reshape(bsz, q_len, -1))
50
+
51
+ class JiRackDecoderLayer(nn.Module):
52
+ def __init__(self, layer_idx):
53
+ super().__init__()
54
+ self.self_attn = JiRackAttention()
55
+ self.gate_proj = JiRackBitLinear(8192, 28672)
56
+ self.up_proj = JiRackBitLinear(8192, 28672)
57
+ self.down_proj = JiRackBitLinear(28672, 8192)
58
+ self.input_layernorm = RMSNorm(8192)
59
+ self.post_attention_layernorm = RMSNorm(8192)
60
+ def forward(self, x, freqs):
61
+ h = self.self_attn(self.input_layernorm(x), freqs)
62
+ x = x + h * 0.4
63
+ mlp = self.down_proj(F.silu(self.gate_proj(self.post_attention_layernorm(x))) * self.up_proj(self.post_attention_layernorm(x)))
64
+ return x + mlp * 0.4
65
+
66
+ class JiRackTernary70B(PreTrainedModel, GenerationMixin):
67
+ config_class = JiRackTernaryConfig
68
+ def __init__(self, config):
69
+ super().__init__(config)
70
+ self.embed_tokens = nn.Embedding(128256, 8192)
71
+ self.layers = nn.ModuleList([JiRackDecoderLayer(i) for i in range(80)])
72
+ self.norm = RMSNorm(8192)
73
+ self.lm_head = nn.Linear(8192, 128256, bias=False)
74
+ self.register_buffer("inv_freq", 1.0 / (500000.0 ** (torch.arange(0, 128, 2).float() / 128)))
75
+ self.use_gc = False # 袪校效袧袨袡 袩袝袪袝袣袥挟效袗孝袝袥鞋
76
+
77
+ def forward(self, input_ids, **kwargs):
78
+ x = self.embed_tokens(input_ids)
79
+ t = torch.arange(x.shape[1], device=x.device).type_as(self.inv_freq)
80
+ freqs = torch.outer(t, self.inv_freq)
81
+ for layer in self.layers:
82
+ if self.training and self.use_gc:
83
+ x = checkpoint(layer, x, freqs, use_reentrant=False)
84
+ else:
85
+ x = layer(x, freqs)
86
+ return CausalLMOutputWithPast(logits=self.lm_head(self.norm(x)) * 0.8)