Initial upload of JiRackTernary 70B converted weights

JiRackTernaryPyTorch_70b.py

@@ -0,0 +1,168 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from typing import Optional, Tuple
+from transformers import PreTrainedModel, PretrainedConfig, GenerationMixin
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.cache_utils import DynamicCache
+
+# --- КОНСТАНТЫ LLAMA-3 70B ---
+VOCAB_SIZE = 128256
+HIDDEN_SIZE = 8192
+INTERMEDIATE_SIZE = 28672
+NUM_LAYERS = 80
+NUM_HEADS = 64
+NUM_KV_HEADS = 8
+HEAD_DIM = 128
+RMS_NORM_EPS = 1e-5
+ROPE_THETA = 500000.0
+
+class JiRackTernaryConfig(PretrainedConfig):
+    model_type = "bitnet"
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.vocab_size = VOCAB_SIZE
+        self.hidden_size = HIDDEN_SIZE
+        self.intermediate_size = INTERMEDIATE_SIZE
+        self.num_hidden_layers = NUM_LAYERS
+        self.num_attention_heads = NUM_HEADS
+        self.num_key_value_heads = NUM_KV_HEADS
+        self.head_dim = HEAD_DIM
+        self.rms_norm_eps = RMS_NORM_EPS
+
+class JiRackBitLinear(nn.Module):
+    def __init__(self, in_features, out_features, bias=False):
+        super().__init__()
+        self.register_buffer("packed", None)
+        self.register_buffer("scale", None)
+        self.register_buffer("orig_shape", None)
+
+    def unpack(self):
+        if self.packed is None: return None
+        p = self.packed.to(torch.int32)
+        b = torch.stack([(p >> 6) & 3, (p >> 4) & 3, (p >> 2) & 3, p & 3], dim=1).view(-1)
+        num_el = int(self.orig_shape[0] * self.orig_shape[1])
+        w = (b[:num_el].to(torch.float16) - 1.0)
+        return w.view(int(self.orig_shape[0]), int(self.orig_shape[1])) * self.scale
+
+    def forward(self, x):
+        w = self.unpack()
+        if w is None: return F.linear(x, torch.zeros(int(self.orig_shape[0]), int(self.orig_shape[1])).to(x.device))
+        
+        x_norm = x - x.mean(dim=-1, keepdim=True)
+        x_norm = torch.clamp(x_norm, -6.0, 6.0) 
+        x_scale = x_norm.abs().max(dim=-1, keepdim=True).values.clamp(min=1e-5)
+        x_quant = (x_norm / x_scale).to(w.dtype)
+        
+        y = F.linear(x_quant, w)
+        return y * (x_scale * 0.67)
+
+class RMSNorm(nn.Module):
+    def __init__(self, dim, eps=1e-5):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+    def forward(self, x):
+        orig_dtype = x.dtype
+        x32 = x.to(torch.float32)
+        variance = x32.pow(2).mean(-1, keepdim=True)
+        x32 = x32 * torch.rsqrt(variance + self.eps)
+        return (x32 * self.weight.to(torch.float32)).to(orig_dtype)
+
+def rotate_half(x):
+    x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+
+def apply_rotary_emb(x, freqs):
+    cos = freqs.cos().to(x.dtype).unsqueeze(0).unsqueeze(1)
+    sin = freqs.sin().to(x.dtype).unsqueeze(0).unsqueeze(1)
+    cos = torch.cat([cos, cos], dim=-1)
+    sin = torch.cat([sin, sin], dim=-1)
+    return (x * cos) + (rotate_half(x) * sin)
+
+class JiRackAttention(nn.Module):
+    def __init__(self, layer_idx):
+        super().__init__()
+        self.layer_idx = layer_idx
+        self.q_proj = JiRackBitLinear(HIDDEN_SIZE, HIDDEN_SIZE)
+        self.k_proj = JiRackBitLinear(HIDDEN_SIZE, NUM_KV_HEADS * HEAD_DIM)
+        self.v_proj = JiRackBitLinear(HIDDEN_SIZE, NUM_KV_HEADS * HEAD_DIM)
+        self.o_proj = JiRackBitLinear(HIDDEN_SIZE, HIDDEN_SIZE)
+
+    def forward(self, x, mask, past, freqs):
+        bsz, q_len, _ = x.shape
+        q = self.q_proj(x).view(bsz, q_len, NUM_HEADS, HEAD_DIM).transpose(1, 2)
+        k = self.k_proj(x).view(bsz, q_len, NUM_KV_HEADS, HEAD_DIM).transpose(1, 2)
+        v = self.v_proj(x).view(bsz, q_len, NUM_KV_HEADS, HEAD_DIM).transpose(1, 2)
+        q, k = apply_rotary_emb(q, freqs), apply_rotary_emb(k, freqs)
+        if past is not None: k, v = past.update(k, v, self.layer_idx)
+        k_rep = k.repeat_interleave(NUM_HEADS // NUM_KV_HEADS, dim=1)
+        v_rep = v.repeat_interleave(NUM_HEADS // NUM_KV_HEADS, dim=1)
+        attn = F.scaled_dot_product_attention(q, k_rep, v_rep, is_causal=(q_len > 1))
+        return self.o_proj(attn.transpose(1, 2).reshape(bsz, q_len, -1)), (k, v)
+
+class JiRackDecoderLayer(nn.Module):
+    def __init__(self, layer_idx):
+        super().__init__()
+        self.layer_idx = layer_idx
+        self.self_attn = JiRackAttention(layer_idx)
+        self.gate_proj = JiRackBitLinear(HIDDEN_SIZE, INTERMEDIATE_SIZE)
+        self.up_proj = JiRackBitLinear(HIDDEN_SIZE, INTERMEDIATE_SIZE)
+        self.down_proj = JiRackBitLinear(INTERMEDIATE_SIZE, HIDDEN_SIZE)
+        self.input_layernorm = RMSNorm(HIDDEN_SIZE, RMS_NORM_EPS)
+        self.post_attention_layernorm = RMSNorm(HIDDEN_SIZE, RMS_NORM_EPS)
+
+    def forward(self, x, mask, past, freqs):
+        # Первые 20 слоев - полный резонанс
+        if self.layer_idx < 20:
+            res_scale = 1.0
+        else:
+            res_scale = 1.0 / (1.0 + (self.layer_idx - 20) / 40.0)
+        
+        h, kv = self.self_attn(self.input_layernorm(x), mask, past, freqs)
+        x = x + h * (res_scale * 0.4) 
+        
+        nx = self.post_attention_layernorm(x)
+        mlp = self.down_proj(F.silu(self.gate_proj(nx)) * self.up_proj(nx))
+        x = x + mlp * (res_scale * 0.4)
+
+        # Мягкая стабилизация каждые 10 слоев вместо жесткого Clipping
+        if self.layer_idx % 10 == 0:
+            std = x.std(dim=-1, keepdim=True).clamp(min=1e-8)
+            x = x / std * 0.05
+            
+        return x, kv
+
+class JiRackTernary70B(PreTrainedModel, GenerationMixin):
+    config_class = JiRackTernaryConfig
+    def __init__(self, config):
+        super().__init__(config)
+        self.embed_tokens = nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE)
+        self.layers = nn.ModuleList([JiRackDecoderLayer(i) for i in range(NUM_LAYERS)])
+        self.norm = RMSNorm(HIDDEN_SIZE, RMS_NORM_EPS)
+        self.lm_head = nn.Linear(HIDDEN_SIZE, VOCAB_SIZE, bias=False)
+        inv_freq = 1.0 / (ROPE_THETA ** (torch.arange(0, HEAD_DIM, 2).float() / HEAD_DIM))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+    def forward(self, input_ids, attention_mask=None, past_key_values=None, use_cache=None, **kwargs):
+        x = self.embed_tokens(input_ids)
+        x = x / (x.norm(dim=-1, keepdim=True) + 1e-8)
+        
+        past_len = past_key_values.get_seq_length() if past_key_values is not None else 0
+        t = torch.arange(past_len, past_len + x.shape[1], device=x.device).type_as(self.inv_freq)
+        freqs = torch.outer(t, self.inv_freq)
+        
+        if use_cache and past_key_values is None: past_key_values = DynamicCache()
+        for layer in self.layers:
+            x, _ = layer(x, attention_mask, past_key_values, freqs)
+            
+        x = self.norm(x)
+        logits = self.lm_head(x.to(torch.float32))
+
+        # Мягкое центрирование логитов без агрессивного скейлинга
+        logits = logits - logits.mean(dim=-1, keepdim=True)
+        return CausalLMOutputWithPast(logits=logits * 0.8, past_key_values=past_key_values if use_cache else None)
+
+    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, attention_mask=None, **kwargs):
+        if past_key_values: input_ids = input_ids[:, -1:]
+        return {"input_ids": input_ids, "past_key_values": past_key_values, "attention_mask": attention_mask}

JiRack_Final_Inference.py

@@ -0,0 +1,62 @@
+import torch
+import torch.nn.functional as F
+from transformers import AutoTokenizer
+from JiRackTernaryPyTorch_70b import JiRackTernaryConfig, JiRackTernary70B
+from load_packed_70b import load_jirack_70b_packed
+
+# Оптимальные параметры из Grid Search
+MODEL_PATH = "JiRack_BitNet_70B_Packed"
+TEMP = 0.5
+ALPHA = 0.4
+K = 3
+MAX_TOKENS = 100
+
+def generate_optimized():
+    tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-70B")
+    config = JiRackTernaryConfig.from_pretrained(MODEL_PATH)
+    model = load_jirack_70b_packed(MODEL_PATH, config).to("cuda:0").eval()
+
+    prompt = "The solar system consists of the Sun and the objects that orbit it. User: Hello, how are you today? Assistant:"
+    input_ids = tokenizer.encode(prompt, return_tensors="pt").to("cuda:0")
+    
+    print(f"\n🚀 Запуск с оптимизированными параметрами (PPL ~10.49)")
+    print(f"🤖 Ответ: ", end="", flush=True)
+
+    generated = input_ids.clone()
+    
+    for _ in range(MAX_TOKENS):
+        with torch.no_grad():
+            outputs = model(generated)
+            # Применяем найденную температуру 0.5
+            logits = outputs.logits[:, -1, :] / TEMP
+            probs = F.softmax(logits, dim=-1)
+            
+            # Contrastive Выбор среди K=3 кандидатов
+            top_k_probs, top_k_ids = torch.topk(probs, K)
+            
+            best_score = -float('inf')
+            best_token = top_k_ids[:, 0:1]
+            
+            for i in range(K):
+                candidate_id = top_k_ids[:, i:i+1]
+                candidate_prob = top_k_probs[:, i]
+                
+                # Штраф за деградацию (повторы в контексте)
+                rep_penalty = torch.isin(candidate_id, generated[0, -32:]).float().item()
+                score = (1 - ALPHA) * candidate_prob.item() - ALPHA * rep_penalty
+                
+                if score > best_score:
+                    best_score = score
+                    best_token = candidate_id
+
+            generated = torch.cat([generated, best_token], dim=-1)
+            word = tokenizer.decode(best_token[0])
+            print(word, end="", flush=True)
+
+            if best_token.item() == 128256 or (best_token.item() == 13 and _ > 30):
+                break
+
+    print("\n" + "-"*60)
+
+if __name__ == "__main__":
+    generate_optimized()

config.json

@@ -0,0 +1,39 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 8192,
+  "initializer_range": 0.02,
+  "intermediate_size": 28672,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 64,
+  "num_hidden_layers": 80,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "quantization_config": {
+    "bits": 1.58,
+    "packing": "2bit_uint8"
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.6",
+  "use_cache": true,
+  "vocab_size": 128256
+}

contrast_optimizer_70b.py

@@ -0,0 +1,53 @@
+import torch
+import numpy as np
+from transformers import AutoTokenizer
+from JiRackTernaryPyTorch_70b import JiRackTernaryConfig
+from load_packed_70b import load_jirack_70b_packed
+
+def test_contrast_levels():
+    PATH = "JiRack_BitNet_70B_Packed"
+    tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-70B")
+    config = JiRackTernaryConfig.from_pretrained(PATH)
+    model = load_jirack_70b_packed(PATH, config)
+    model.eval().to("cuda:0")
+
+    # Уровни контрастности для теста
+    contrast_scales = [1.0, 2.0, 4.0, 6.0, 8.0, 10.0, 15.0]
+    test_text = "The solar system consists of the Sun and the objects that orbit it."
+    
+    inputs = tokenizer(test_text, return_tensors="pt").to("cuda:0")
+    if "attention_mask" in inputs:
+        inputs["attention_mask"] = inputs["attention_mask"].bool()
+    
+    target_ids = inputs["input_ids"].clone()
+
+    print(f"\n🔍 Поиск оптимальной резкости логитов...")
+    print(f"{'Contrast':<10} | {'Average Loss':<15} | {'Perplexity':<12}")
+    print("-" * 45)
+
+    results = []
+    with torch.no_grad():
+        # Базовый прогон для получения "чистых" логитов
+        outputs = model(inputs["input_ids"])
+        base_logits = outputs.logits
+
+        for scale in contrast_scales:
+            # Применяем масштаб контрастности вручную поверх выхода
+            scaled_logits = (base_logits - base_logits.mean(dim=-1, keepdim=True)) * scale
+            
+            shift_logits = scaled_logits[..., :-1, :].contiguous()
+            shift_labels = target_ids[..., 1:].contiguous()
+
+            loss_fct = torch.nn.CrossEntropyLoss()
+            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+            
+            ppl = torch.exp(loss).item()
+            print(f"{scale:<10.1f} | {loss.item():>15.4f} | {ppl:>12.2f}")
+            results.append((scale, ppl))
+
+    best_scale = min(results, key=lambda x: x[1])[0]
+    print("-" * 45)
+    print(f"🎯 Рекомендованная контрастность: {best_scale}")
+
+if __name__ == "__main__":
+    test_contrast_levels()

create_index.py

@@ -0,0 +1,28 @@
+import os
+import json
+from safetensors import safe_open
+
+MODEL_PATH = "./JiRackTernary_70B_converted"
+files = [f for f in os.listdir(MODEL_PATH) if f.endswith(".safetensors")]
+files.sort()
+
+metadata = {}
+weight_map = {}
+
+print(f"🔍 Сканирую {len(files)} шардов...")
+
+for filename in files:
+    full_path = os.path.join(MODEL_PATH, filename)
+    with safe_open(full_path, framework="pt") as f:
+        for key in f.keys():
+            weight_map[key] = filename
+
+index_data = {
+    "metadata": {"total_size": 0}, # Размер можно не указывать точно
+    "weight_map": weight_map
+}
+
+with open(os.path.join(MODEL_PATH, "model.safetensors.index.json"), "w") as f:
+    json.dump(index_data, f, indent=2)
+
+print("✅ Файл model.safetensors.index.json успешно создан!")

grid_search_contrastive_v2.py

@@ -0,0 +1,76 @@
+import torch
+import torch.nn.functional as F
+from transformers import AutoTokenizer
+from JiRackTernaryPyTorch_70b import JiRackTernaryConfig, JiRackTernary70B
+from load_packed_70b import load_jirack_70b_packed
+
+def evaluate_quality(model, input_ids, alpha, k_val, temp):
+    total_loss = 0
+    context = input_ids.clone()
+    test_len = 12 
+    
+    for _ in range(test_len):
+        with torch.no_grad():
+            outputs = model(context)
+            logits = outputs.logits[:, -1, :] / max(temp, 1e-6)
+            
+            # Contrastive Logic
+            probs = F.softmax(logits, dim=-1)
+            top_k_probs, top_k_ids = torch.topk(probs, k_val)
+            
+            best_score = -float('inf')
+            best_token = top_k_ids[:, 0:1]
+            
+            for i in range(k_val):
+                candidate_id = top_k_ids[:, i:i+1]
+                candidate_prob = top_k_probs[:, i]
+                
+                # Косинусное сходство (аппроксимация через частоту в контексте)
+                rep_count = (context == candidate_id).sum().item()
+                penalty = rep_count / context.size(1)
+                
+                score = (1 - alpha) * candidate_prob.item() - alpha * penalty
+                
+                if score > best_score:
+                    best_score = score
+                    best_token = candidate_id
+            
+            target_prob = probs[0, best_token.item()]
+            total_loss += -torch.log(target_prob + 1e-10)
+            context = torch.cat([context, best_token], dim=-1)
+            
+    return torch.exp(total_loss / test_len).item()
+
+def run_advanced_grid_search():
+    PATH = "JiRack_BitNet_70B_Packed"
+    tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-70B")
+    config = JiRackTernaryConfig.from_pretrained(PATH)
+    model = load_jirack_70b_packed(PATH, config).to("cuda:0").eval()
+
+    # Сетка параметров
+    alphas = [0.4, 0.6]
+    ks = [3, 5]
+    temps = [0.5, 0.7, 0.9]
+    
+    test_phrase = "The solar system is composed of"
+    input_ids = tokenizer.encode(test_phrase, return_tensors="pt").to("cuda:0")
+
+    print(f"\n🚀 Глобальный поиск гиперпараметров (Temperature + Contrastive)")
+    print(f"{'Temp':<6} | {'Alpha':<6} | {'K':<4} | {'PPL Score':<10}")
+    print("-" * 35)
+
+    results = []
+    for t in temps:
+        for a in alphas:
+            for k in ks:
+                ppl = evaluate_quality(model, input_ids, a, k, t)
+                print(f"{t:<6.1f} | {a:<6.1f} | {k:<4} | {ppl:<10.2f}")
+                results.append((t, a, k, ppl))
+
+    best = min(results, key=lambda x: x[3])
+    print("-" * 35)
+    print(f"🎯 ЛУЧШАЯ КОМБИНАЦИЯ: Temp={best[0]}, Alpha={best[1]}, K={best[2]}")
+    print(f"📈 Минимальная перплексия: {best[3]:.2f}")
+
+if __name__ == "__main__":
+    run_advanced_grid_search()

layer_inspector_70b.py

@@ -0,0 +1,82 @@
+import torch
+import gc
+from transformers import AutoTokenizer
+from JiRackTernaryPyTorch_70b import JiRackTernaryConfig
+from load_packed_70b import load_jirack_70b_packed
+
+def inspect_activations():
+    PATH = "JiRack_BitNet_70B_Packed"
+    tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-70B")
+    config = JiRackTernaryConfig.from_pretrained(PATH)
+    
+    print("🏗️ Загрузка упакованной 70B на GPU...")
+    model = load_jirack_70b_packed(PATH, config)
+    model.eval().to("cuda:0")
+
+    # Тот же промпт для консистентности
+    prompt = "The solar system consists of the Sun and the objects that orbit it."
+    inputs = tokenizer(prompt, return_tensors="pt").to("cuda:0")
+    
+    # ИСПРАВЛЕНИЕ: Приведение маски к bool для SDPA
+    if "attention_mask" in inputs:
+        inputs["attention_mask"] = inputs["attention_mask"].bool()
+
+    stats = {}
+
+    def get_hook(name):
+        def hook(model, input, output):
+            # output - это CausalLMOutputWithPast или тензор в зависимости от места
+            # В слоях это обычно тензор (hidden_states)
+            data = output[0] if isinstance(output, tuple) else output
+            data = data.detach().to(torch.float32)
+            
+            stats[name] = {
+                "mean": data.mean().item(),
+                "std": data.std().item(),
+                "l2": torch.norm(data, p=2).item(),
+                "zeros": (data.abs() < 1e-6).float().mean().item(),
+                "has_nan": torch.isnan(data).any().item()
+            }
+        return hook
+
+    # Регистрируем хуки на ключевых точках
+    layers_to_watch = [0, 10, 40, 79]
+    for idx in layers_to_watch:
+        model.layers[idx].register_forward_hook(get_hook(f"Layer_{idx}"))
+
+    print("\n🔍 Запуск диагностического прогона...")
+    with torch.no_grad():
+        try:
+            model(**inputs)
+        except Exception as e:
+            print(f"❌ Ошибка во время прогона: {e}")
+            return
+
+    print("\n📊 Результаты инспекции слоев:")
+    print(f"{'Слой':<10} | {'Среднее':<12} | {'Std':<12} | {'L2-Норма':<12} | {'% Нулей':<10} | {'NaN'}")
+    print("-" * 85)
+    
+    for name in [f"Layer_{i}" for i in layers_to_watch]:
+        if name in stats:
+            s = stats[name]
+            nan_str = "❌ YES" if s['has_nan'] else "✅ NO"
+            print(f"{name:<10} | {s['mean']:>12.6f} | {s['std']:>12.6f} | {s['l2']:>12.1f} | {s['zeros']:>10.1%}| {nan_str}")
+
+    # Анализ здоровья сигнала
+    l0 = stats["Layer_0"]["l2"]
+    l79 = stats["Layer_79"]["l2"]
+    ratio = l79 / l0 if l0 != 0 else 0
+
+    print(f"\n📈 Коэффициент сохранения энергии (L2_L79/L2_L0): {ratio:.4f}")
+    
+    if ratio < 0.2:
+        print("⚠️ Сигнал критически затухает. Модель 'засыпает' к выходу.")
+    elif ratio > 5.0:
+        print("⚠️ Сигнал перегружен. Возможны галлюцинации и 'салат из слов'.")
+    else:
+        print("✅ Динамика в пределах нормы. Тернарный стек стабилен.")
+
+if __name__ == "__main__":
+    torch.cuda.empty_cache()
+    gc.collect()
+    inspect_activations()

load_packed_70b.py

@@ -0,0 +1,78 @@
+import torch
+import os
+import glob
+from safetensors.torch import load_file
+from JiRackTernaryPyTorch_70b import JiRackTernary70B
+
+def load_jirack_70b_packed(path, config):
+    print(f"🏗️ Инициализация структуры 70B в RAM...")
+    model = JiRackTernary70B(config)
+    
+    # Карта модулей BitLinear
+    bit_modules = {name: mod for name, mod in model.named_modules() if hasattr(mod, "packed")}
+
+    shards = sorted(glob.glob(os.path.join(path, "*.safetensors")))
+    print(f"📖 Загрузка весов...")
+    
+    for shard_path in shards:
+        current_shard = load_file(shard_path, device="cpu")
+        print(f" - {os.path.basename(shard_path)}")
+        
+        for k, v in current_shard.items():
+            clean_k = k[6:] if k.startswith("model.") else k
+            
+            suffix = None
+            pure_path = clean_k
+            for s in [".packed", ".scale", ".orig_shape"]:
+                if clean_k.endswith(s):
+                    suffix = s
+                    pure_path = clean_k.replace(".weight" + s, "")
+                    break
+            
+            model_key = pure_path.replace(".mlp.", ".")
+            
+            # 1. Загрузка в BitLinear (наши упакованные слои)
+            if suffix:
+                if model_key in bit_modules:
+                    mod = bit_modules[model_key]
+                    if suffix == ".packed": mod.packed = v.to(torch.uint8)
+                    elif suffix == ".scale": mod.scale = v.to(torch.float16)
+                    elif suffix == ".orig_shape": mod.orig_shape = v.to(torch.int32)
+                continue
+
+            # 2. Загрузка в обычные слои (Embedding, Norm, lm_head)
+            try:
+                # Убираем .weight из ключа, чтобы найти сам параметр
+                is_weight = clean_k.endswith(".weight")
+                attr_path = clean_k[:-7] if is_weight else clean_k
+                
+                parts = attr_path.split(".")
+                obj = model
+                for part in parts[:-1]:
+                    obj = getattr(obj, part)
+                
+                # Находим целевой параметр (например, .weight)
+                target_name = parts[-1]
+                target_attr = getattr(obj, target_name)
+                
+                if is_weight and hasattr(target_attr, 'weight'):
+                    # Если мы нашли слой (например, Embedding), копируем в его .weight
+                    target_attr.weight.data.copy_(v.to(torch.float16))
+                elif isinstance(target_attr, torch.nn.Parameter):
+                    # Если это уже параметр (например, в RMSNorm)
+                    target_attr.data.copy_(v.to(torch.float16))
+                else:
+                    # Резервный вариант: если это просто атрибут
+                    setattr(obj, target_name, torch.nn.Parameter(v.to(torch.float16), requires_grad=False))
+            except Exception:
+                continue
+                
+        del current_shard
+    
+    missing = [name for name, mod in bit_modules.items() if mod.packed is None]
+    if not missing:
+        print("✅ Все 560 тернарных слоев загружены успешно!")
+    else:
+        print(f"❌ Пропущено {len(missing)} слоев.")
+
+    return model

model-00030-of-00030.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed4d0a25ffa885bdbe02a859a4e915a1f7ffa9c5d91bb82e86ca300a88bfe49c
-size 262668546
+oid sha256:18abf0bdcb9575bfdd74cb6987e3101bc20391509dd6e1042afdf13bfc89c14f
+size 2101346400

perplexity_tester_70b.py

@@ -0,0 +1,64 @@
+import torch
+import numpy as np
+from transformers import AutoTokenizer
+from JiRackTernaryPyTorch_70b import JiRackTernaryConfig
+from load_packed_70b import load_jirack_70b_packed
+
+def calculate_perplexity():
+    PATH = "JiRack_BitNet_70B_Packed"
+    tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-70B")
+    config = JiRackTernaryConfig.from_pretrained(PATH)
+    model = load_jirack_70b_packed(PATH, config)
+    model.eval().to("cuda:0")
+
+    # Контрольные фразы (эталонный английский)
+    test_texts = [
+        "The solar system consists of the Sun and the objects that orbit it.",
+        "Artificial intelligence is changing the way we interact with technology.",
+        "Deep learning models require a lot of computational resources to train.",
+        "Hello, how are you today? I am doing well, thank you for asking."
+    ]
+
+    print(f"\n🧪 Тестирование Perplexity (Текущий Gain)...")
+    print("-" * 50)
+
+    total_loss = 0
+    total_tokens = 0
+
+    with torch.no_grad():
+        for text in test_texts:
+            inputs = tokenizer(text, return_tensors="pt").to("cuda:0")
+            if "attention_mask" in inputs:
+                inputs["attention_mask"] = inputs["attention_mask"].bool()
+            
+            input_ids = inputs["input_ids"]
+            target_ids = input_ids.clone()
+
+            outputs = model(input_ids)
+            logits = outputs.logits
+
+            # Сдвигаем логиты и цели для расчета Loss (стандарт для Causal LM)
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = target_ids[..., 1:].contiguous()
+
+            loss_fct = torch.nn.CrossEntropyLoss(reduction="sum")
+            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+            
+            total_loss += loss.item()
+            total_tokens += shift_labels.numel()
+
+    avg_loss = total_loss / total_tokens
+    perplexity = np.exp(avg_loss)
+
+    print(f"📊 Average Loss: {avg_loss:.4f}")
+    print(f"🚀 PERPLEXITY: {perplexity:.2f}")
+    
+    if perplexity < 20:
+        print("✅ Идеально! Модель адекватно воспринимает язык.")
+    elif perplexity < 100:
+        print("⚠️ Терпимо. Модель понимает структуру, но шум мешает.")
+    else:
+        print("❌ Плохо. Сигнал слишком зашумлен или затухает.")
+
+if __name__ == "__main__":
+    calculate_perplexity()

setup_tokenizer.py

@@ -0,0 +1,24 @@
+from transformers import AutoTokenizer
+import os
+
+def setup():
+    # Используем официальный токенизатор Llama-3.1-70B
+    # (Убедитесь, что вы авторизованы в HF: huggingface-cli login)
+    model_id = "meta-llama/Llama-3.1-70B-Instruct" 
+    target_path = "JiRack_BitNet_70B_Packed"
+
+    print(f"📥 Скачивание токенизатора {model_id}...")
+    
+    # Загружаем с флагом fix_mistral_regex, чтобы избежать ворнингов в будущем
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+    if not os.path.exists(target_path):
+        os.makedirs(target_path)
+
+    print(f"💾 Сохранение файлов токенизатора в {target_path}...")
+    tokenizer.save_pretrained(target_path)
+    
+    print("✅ Готово! Теперь папка содержит правильные файлы: tokenizer.json, tokenizer_config.json и др.")
+
+if __name__ == "__main__":
+    setup()

test_70b_inference_6.py

@@ -0,0 +1,96 @@
+import torch
+import torch.nn.functional as F
+from transformers import AutoTokenizer
+from JiRackTernaryPyTorch_70b import JiRackTernaryConfig, JiRackTernary70B
+from load_packed_70b import load_jirack_70b_packed
+
+# --- CONFIGURATION CONSTANTS ---
+MODEL_PATH = "JiRack_BitNet_70B_Packed"
+TOKENIZER_ID = "meta-llama/Meta-Llama-3-70B"
+DEVICE = "cuda:0"
+
+# --- GENERATION HYPERPARAMETERS ---
+MAX_NEW_TOKENS = 100
+TEMPERATURE = 0.75
+MIN_P = 0.08
+TOP_K = 50
+REPETITION_PENALTY = 3.0
+PENALTY_WINDOW = 30
+PUNCTUATION_BIAS = 2.0
+
+# --- TOKEN CONSTANTS (Llama-3) ---
+DOT_TOKEN = 13
+COMMA_TOKEN = 11
+QUEST_TOKEN = 30
+EXCL_TOKEN = 0
+EOS_TOKEN = 128256
+PUNCTUATION_TOKENS = [DOT_TOKEN, COMMA_TOKEN, QUEST_TOKEN, EXCL_TOKEN, EOS_TOKEN]
+
+def apply_filters(logits, min_p=MIN_P, top_k=TOP_K):
+    """Комбинированная фильтрация логитов."""
+    # Защита от NaN
+    logits = torch.where(torch.isnan(logits), torch.full_like(logits, -1e4), logits)
+    
+    # 1. Top-K
+    if top_k > 0:
+        threshold = torch.topk(logits, min(top_k, logits.size(-1)))[0][-1]
+        logits[logits < threshold] = float('-inf')
+    
+    # 2. Min-P
+    probs = F.softmax(logits, dim=-1)
+    limit = probs.max() * min_p
+    logits[probs < limit] = float('-inf')
+    
+    return logits
+
+def generate():
+    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_ID)
+    config = JiRackTernaryConfig.from_pretrained(MODEL_PATH)
+    
+    print(f"🏗️ Загрузка модели из {MODEL_PATH}...")
+    model = load_jirack_70b_packed(MODEL_PATH, config).to(DEVICE).eval()
+
+    prompt = "The solar system consists of the Sun and the objects that orbit it. User: Hello, how are you today? Assistant:"
+    input_ids = tokenizer.encode(prompt, return_tensors="pt").to(DEVICE)
+    generated_ids = input_ids[0].tolist()
+
+    print(f"\n📝 Промпт: {prompt}\n" + "-"*60 + "\n🤖 Ответ: ", end="", flush=True)
+
+    for i in range(MAX_NEW_TOKENS):
+        with torch.no_grad():
+            # Получаем предсказание для всей последовательности
+            outputs = model(torch.tensor([generated_ids]).to(DEVICE))
+            logits = outputs.logits[0, -1, :]
+            
+            # Применяем Repetition Penalty
+            for token_id in set(generated_ids[-PENALTY_WINDOW:]):
+                if logits[token_id] > 0:
+                    logits[token_id] /= REPETITION_PENALTY
+                else:
+                    logits[token_id] *= REPETITION_PENALTY
+            
+            # Применяем Punctuation Bias
+            for p_token in PUNCTUATION_TOKENS:
+                if p_token < logits.size(0):
+                    logits[p_token] += PUNCTUATION_BIAS
+            
+            # Фильтрация
+            filtered = apply_filters(logits)
+            
+            # Сэмплирование
+            probs = F.softmax(filtered / TEMPERATURE, dim=-1)
+            next_token = torch.multinomial(probs, 1).item()
+            
+            generated_ids.append(next_token)
+            print(tokenizer.decode(next_token), end="", flush=True)
+            
+            # Условия остановки
+            if next_token == EOS_TOKEN:
+                break
+            if next_token == DOT_TOKEN and i > 20:
+                break
+                
+    print("\n" + "-" * 60)
+
+if __name__ == "__main__":
+    generate()

test_70b_inference_7.py

@@ -0,0 +1,63 @@
+import torch
+import torch.nn.functional as F
+from transformers import AutoTokenizer
+from JiRackTernaryPyTorch_70b import JiRackTernaryConfig, JiRackTernary70B
+from load_packed_70b import load_jirack_70b_packed
+
+# --- CONFIGURATION ---
+MODEL_PATH = "JiRack_BitNet_70B_Packed"
+DEVICE = "cuda:0"
+ALPHA = 0.6  # Коэффициент штрафа за вырожденность (0.5-0.8)
+K = 5        # Количество кандидатов для сравнения
+
+def generate_contrastive():
+    tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-70B")
+    config = JiRackTernaryConfig.from_pretrained(MODEL_PATH)
+    model = load_jirack_70b_packed(MODEL_PATH, config).to(DEVICE).eval()
+
+    prompt = "The solar system consists of the Sun and the objects that orbit it. User: Hello, how are you today? Assistant:"
+    input_ids = tokenizer.encode(prompt, return_tensors="pt").to(DEVICE)
+    
+    print(f"\n📝 Промпт: {prompt}\n" + "-"*60 + "\n🤖 Ответ: ", end="", flush=True)
+
+    for _ in range(80):
+        with torch.no_grad():
+            # 1. Получаем логиты для текущего шага
+            outputs = model(input_ids)
+            last_hidden = outputs.logits[:, -1, :] # Используем логиты как прокси для семантики
+            
+            # 2. Выбираем Top-K кандидатов
+            top_k_probs, top_k_ids = torch.topk(F.softmax(last_hidden, dim=-1), K)
+            
+            best_score = -float('inf')
+            best_token = None
+
+            # 3. Contrastive Ranking
+            # Ищем токен, который вероятен, но не является "пустым повтором"
+            for i in range(K):
+                candidate_id = top_k_ids[:, i:i+1]
+                candidate_prob = top_k_probs[:, i]
+                
+                # Вычисляем максимальное сходство с предыдущим контекстом (Cosine Similarity прокси)
+                # В упрощенной версии ищем повторы в последних 32 токенах
+                context = input_ids[0, -32:]
+                repetition_penalty = torch.isin(candidate_id, context).float().item()
+                
+                # Формула: Score = (1-Alpha) * Probability - Alpha * Max_Similarity
+                score = (1 - ALPHA) * candidate_prob.item() - ALPHA * repetition_penalty
+                
+                if score > best_score:
+                    best_score = score
+                    best_token = candidate_id
+
+            input_ids = torch.cat([input_ids, best_token], dim=-1)
+            word = tokenizer.decode(best_token[0])
+            print(word, end="", flush=True)
+
+            if best_token.item() == 128256 or (best_token.item() == 13 and _ > 20):
+                break
+
+    print("\n" + "-" * 60)
+
+if __name__ == "__main__":
+    generate_contrastive()

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c5cf44023714fb39b05e71e425f8d7b92805ff73f7988b083b8c87f0bf87393
+size 17209961

tokenizer_config.json

@@ -0,0 +1,2062 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

train_recovery_70b_v3.py

@@ -0,0 +1,82 @@
+import torch
+import torch.nn as nn
+from torch.optim import AdamW 
+from transformers import get_linear_schedule_with_warmup
+from JiRackTernaryPyTorch_70b import JiRackTernaryConfig, JiRackTernary70B
+from load_packed_70b import load_jirack_70b_packed
+import numpy as np
+from huggingface_hub import hf_hub_download
+
+# --- HYPERPARAMETERS ---
+MODEL_PATH = "JiRack_BitNet_70B_Packed"
+REPO_ID = "kgrabko/wikipedia"
+FILENAME = "wikipedia_llama3_tokens.bin"
+GRADIENT_ACCUMULATION_STEPS = 8  
+LR = 3e-5 
+MAX_STEPS = 1000
+MAX_SEQ_LEN = 512 
+
+def train_recovery():
+    # 1. Скачивание файла (если его еще нет локально)
+    print("⏳ Проверка/Загрузка бинарного файла данных...")
+    bin_path = hf_hub_download(repo_id=REPO_ID, filename=FILENAME, repo_type="dataset")
+
+    # 2. Загрузка данных через Memory Mapping
+    # Предполагаем, что токены сохранены как uint16 или uint32
+    # Llama-3 имеет словарь > 128k, значит это uint32 (4 байта на токен)
+    data = np.memmap(bin_path, dtype=np.uint32, mode='r')
+    num_samples = len(data) // MAX_SEQ_LEN
+    print(f"📊 Найдено {num_samples} последовательностей по {MAX_SEQ_LEN} токенов.")
+
+    config = JiRackTernaryConfig.from_pretrained(MODEL_PATH)
+    print("🏗️ Загрузка модели...")
+    model = load_jirack_70b_packed(MODEL_PATH, config)
+    model.to("cuda:0")
+
+    # Размораживаем семантические слои
+    for name, param in model.named_parameters():
+        param.requires_grad = any(x in name for x in ["embed_tokens", "lm_head", "norm"])
+
+    optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=LR)
+    scheduler = get_linear_schedule_with_warmup(optimizer, 50, MAX_STEPS)
+    criterion = nn.CrossEntropyLoss()
+
+    model.train()
+    print("🚀 Старт калибровки...")
+
+    accumulated_loss = 0
+    optimizer.zero_grad()
+
+    for step in range(MAX_STEPS * GRADIENT_ACCUMULATION_STEPS):
+        # Случайная выборка блока из данных
+        idx = np.random.randint(0, num_samples)
+        chunk = data[idx * MAX_SEQ_LEN : (idx + 1) * MAX_SEQ_LEN]
+        input_ids = torch.from_numpy(chunk.astype(np.int64)).unsqueeze(0).to("cuda:0")
+
+        outputs = model(input_ids)
+        logits = outputs.logits[:, :-1, :].contiguous()
+        labels = input_ids[:, 1:].contiguous()
+        
+        loss = criterion(logits.view(-1, config.vocab_size), labels.view(-1))
+        (loss / GRADIENT_ACCUMULATION_STEPS).backward()
+        
+        accumulated_loss += loss.item()
+
+        if (step + 1) % GRADIENT_ACCUMULATION_STEPS == 0:
+            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+            optimizer.step()
+            scheduler.step()
+            optimizer.zero_grad()
+            
+            curr_step = (step + 1) // GRADIENT_ACCUMULATION_STEPS
+            print(f"Step {curr_step}/{MAX_STEPS} | Loss: {accumulated_loss:.4f}")
+            accumulated_loss = 0
+
+            if curr_step % 250 == 0:
+                torch.save(model.state_dict(), f"recovered_70b_step_{curr_step}.pt")
+
+    torch.save(model.state_dict(), "recovered_semantic_70b_final.pt")
+    print("✅ Готово!")
+
+if __name__ == "__main__":
+    train_recovery()