Upload folder using huggingface_hub

SlimOrca/SlimOrca.py

@@ -0,0 +1,61 @@
+import json
+import os
+from datasets import load_dataset
+from tqdm import tqdm
+
+def process_and_save_dataset(output_path="slimorca_llama3_formatted.jsonl"):
+    # 1. Загрузка датасета
+    print("📥 Загрузка SlimOrca из Hugging Face (это может занять время)...")
+    try:
+        # Используем streaming=True, если интернет медленный, 
+        # но для конвертации лучше выкачать целиком.
+        dataset = load_dataset("Open-Orca/SlimOrca", split="train")
+    except Exception as e:
+        print(f"❌ Ошибка при загрузке: {e}")
+        return
+
+    print(f"✅ Загружено {len(dataset)} диалогов. Начинаю конвертацию...")
+
+    # 2. Форматирование под Llama 3
+    # Формат: <|begin_of_text|><|start_header_id|>role<|end_header_id|>\n\ncontent<|eot_id|>
+    
+    with open(output_path, "w", encoding="utf-8") as f:
+        for entry in tqdm(dataset, desc="Конвертация"):
+            # Начинаем с BOS (Beginning of String)
+            formatted_text = "<|begin_of_text|>"
+            
+            for msg in entry["conversations"]:
+                # SlimOrca использует 'from': 'human'/'gpt'/'system'
+                role_map = {
+                    "human": "user",
+                    "gpt": "assistant",
+                    "system": "system"
+                }
+                
+                raw_role = msg["from"]
+                role = role_map.get(raw_role, "user")
+                content = msg["value"].strip()
+
+                # Сборка блока сообщения
+                block = (
+                    f"<|start_header_id|>{role}<|end_header_id|>\n\n"
+                    f"{content}<|eot_id|>"
+                )
+                formatted_text += block
+
+            # Сохраняем как JSONL объект (совместимо с HuggingFace Trainer и Axolotl)
+            json_line = json.dumps({"text": formatted_text}, ensure_ascii=False)
+            f.write(json_line + "\n")
+
+    print(f"\n✨ Готово! Файл сохранен здесь: {os.path.abspath(output_path)}")
+    print("💡 Теперь вы можете использовать этот файл для SFT (Supervised Fine-Tuning).")
+
+if __name__ == "__main__":
+    # Проверка наличия библиотек
+    try:
+        import datasets
+    except ImportError:
+        print("🚀 Установка необходимых библиотек...")
+        os.system("pip install datasets tqdm")
+    
+    process_and_save_dataset()

SlimOrca/SlimOrcaSFTPtData.py

@@ -0,0 +1,90 @@
+import json
+import torch
+import os
+from transformers import AutoTokenizer
+from tqdm import tqdm
+
+def tokenize_to_chunks(
+    input_file="slimorca_llama3_formatted.jsonl",
+    model_id="meta-llama/Llama-3.2-3B-Instruct",
+    chunk_size=5000, # Увеличил размер чанка, так как данные стали "легче" без паддинга
+    max_length=8192,
+    output_prefix="jirack_sft_packed",
+    start_chunk_idx=0
+):
+    print(f"📥 Загрузка токенизатора: {model_id}")
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+    # В Llama 3.x используем eos_token как pad_token, если он не задан
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+
+    # Токены заголовков для поиска границ ответа
+    assistant_header_ids = tokenizer.encode("<|start_header_id|>assistant<|end_header_id|>\n\n", add_special_tokens=False)
+    eot_id = tokenizer.convert_tokens_to_ids("<|eot_id|>")
+
+    input_ids_buffer = []
+    labels_buffer = []
+    chunk_idx = start_chunk_idx
+
+    def save_current_chunk(ids, labels, idx):
+        filename = f"{output_prefix}_{idx}.pt"
+        # Сохраняем как список тензоров (без stack, так как длины разные)
+        torch.save({
+            "input_ids": ids,
+            "labels": labels
+        }, filename)
+        print(f"💾 Чанк {idx} сохранен: {filename} ({len(ids)} примеров)")
+
+    print(f"🔄 Обработка файла: {input_file}")
+
+    # Примерное кол-во строк для прогресс-бара
+    total_lines = 517982 
+
+    with open(input_file, 'r', encoding='utf-8') as f:
+        for line in tqdm(f, total=total_lines, desc="Tokenizing"):
+            try:
+                data = json.loads(line)
+                text = data['text']
+
+                # Токенизируем весь текст сразу
+                full_ids = tokenizer.encode(text, add_special_tokens=False)
+                
+                # Создаем маску labels (-100 по умолчанию — игнорируем при расчете Loss)
+                labels = [-100] * len(full_ids)
+
+                # Ищем блоки ассистента для обучения
+                header_len = len(assistant_header_ids)
+                for i in range(len(full_ids) - header_len):
+                    if full_ids[i:i+header_len] == assistant_header_ids:
+                        start_res = i + header_len
+                        for j in range(start_res, len(full_ids)):
+                            labels[j] = full_ids[j] # Обучаем на этих токенах
+                            if full_ids[j] == eot_id:
+                                break
+                
+                # Обрезаем, если текст слишком длинный, но НЕ дополняем (No Padding!)
+                if len(full_ids) > max_length:
+                    full_ids = full_ids[:max_length]
+                    labels = labels[:max_length]
+
+                # Добавляем в буфер только если есть что учить
+                if any(l != -100 for l in labels):
+                    input_ids_buffer.append(torch.tensor(full_ids, dtype=torch.int64))
+                    labels_buffer.append(torch.tensor(labels, dtype=torch.int64))
+
+                if len(input_ids_buffer) == chunk_size:
+                    save_current_chunk(input_ids_buffer, labels_buffer, chunk_idx)
+                    chunk_idx += 1
+                    input_ids_buffer = []
+                    labels_buffer = []
+
+            except Exception as e:
+                continue
+
+    # Финальный чанк
+    if input_ids_buffer:
+        save_current_chunk(input_ids_buffer, labels_buffer, chunk_idx)
+
+if __name__ == "__main__":
+    tokenize_to_chunks()

SlimOrca/SlimOrcaSFTPtData_old.py

@@ -0,0 +1,91 @@
+import json
+import torch
+import os
+from transformers import AutoTokenizer
+from tqdm import tqdm
+
+def tokenize_to_chunks(
+    input_file="slimorca_llama3_formatted.jsonl",
+    model_id="meta-llama/Llama-3.2-3B-Instruct",
+    chunk_size=11000, 
+    max_length=2048,
+    output_prefix="jirack_sft_sft_data",
+    start_chunk_idx=10
+):
+    print(f"📥 Загрузка токенизатора: {model_id}")
+    # Убедитесь, что вы авторизованы в HF: huggingface-cli login
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+    
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+
+    # Получаем ID для поиска границ ответа ассистента
+    assistant_header_ids = tokenizer.encode("<|start_header_id|>assistant<|end_header_id|>\n\n", add_special_tokens=False)
+    eot_id = tokenizer.convert_tokens_to_ids("<|eot_id|>")
+
+    input_ids_buffer = []
+    labels_buffer = []
+    chunk_idx = start_chunk_idx
+
+    def save_current_chunk(ids, labels, idx):
+        filename = f"{output_prefix}_{idx}.pt"
+        # Явное приведение к int64 для соответствия вашему размеру 5.9 ГБ
+        torch.save({
+            "input_ids": torch.stack(ids).to(torch.int64),
+            "labels": torch.stack(labels).to(torch.int64)
+        }, filename)
+        print(f"💾 Чанк {idx} сохранен: {filename} ({len(ids)} строк)")
+
+    print(f"🔄 Обработка файла: {input_file}")
+    
+    # Считаем строки для tqdm, если файл уже на диске
+    total_lines = 517982 
+    
+    with open(input_file, 'r', encoding='utf-8') as f:
+        for line in tqdm(f, total=total_lines, desc="Tokenizing"):
+            try:
+                data = json.loads(line)
+                text = data['text']
+                
+                full_ids = tokenizer.encode(text, add_special_tokens=False)
+                labels = [-100] * len(full_ids)
+                
+                # Маскируем всё, кроме ответов ассистента
+                header_len = len(assistant_header_ids)
+                for i in range(len(full_ids) - header_len):
+                    if full_ids[i:i+header_len] == assistant_header_ids:
+                        start_res = i + header_len
+                        for j in range(start_res, len(full_ids)):
+                            labels[j] = full_ids[j]
+                            if full_ids[j] == eot_id:
+                                break
+                
+                # Truncation
+                if len(full_ids) > max_length:
+                    full_ids = full_ids[:max_length]
+                    labels = labels[:max_length]
+                
+                # Padding
+                pad_len = max_length - len(full_ids)
+                if pad_len > 0:
+                    full_ids += [tokenizer.pad_token_id] * pad_len
+                    labels += [-100] * pad_len
+
+                input_ids_buffer.append(torch.tensor(full_ids))
+                labels_buffer.append(torch.tensor(labels))
+
+                if len(input_ids_buffer) == chunk_size:
+                    save_current_chunk(input_ids_buffer, labels_buffer, chunk_idx)
+                    chunk_idx += 1
+                    input_ids_buffer = []
+                    labels_buffer = []
+
+            except Exception as e:
+                continue
+
+    # Сохраняем финальный чанк
+    if input_ids_buffer:
+        save_current_chunk(input_ids_buffer, labels_buffer, chunk_idx)
+
+if __name__ == "__main__":
+    tokenize_to_chunks()

SlimOrca/jirack_sft_packed_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1760e9f78b63a58e479d22a1fd0a4385550fd041a3c0fe75ed35cb887defdf35
+size 34559823

SlimOrca/jirack_sft_packed_1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fba40b344c81aaa83c4d2a5c02c8756be69885186dbd956824077f16c17b2bec
+size 34450831

SlimOrca/jirack_sft_packed_10.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e517f60bbf91818766561051efeebd4deb75dc2100ba31ed3b2695cd11a0f42f
+size 33467301

SlimOrca/jirack_sft_packed_100.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ef79b2790a6840b62a4b50733a527be02307a275e8aa77129b1865c254aaa52
+size 34934779

SlimOrca/jirack_sft_packed_101.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54dd03cab138f730aee1c71826c6df9cc6a46dbda277d2c9b2462d139ce28e4e
+size 34869499

SlimOrca/jirack_sft_packed_102.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86eb49157a80ec3b923665af689d1a25ad0b4e8e027ba21a8ce5d3fc8f6e297c
+size 34659963

SlimOrca/jirack_sft_packed_103.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2529dbcbc452011483bebbd19c9fbae3bf44bce24a935926535e6765c2172d87
+size 21007559

SlimOrca/jirack_sft_packed_11.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34b259fb86e95964c99e6bb47c9772274fefbe77162f1f79527d6fcb205a5512
+size 34219685

SlimOrca/jirack_sft_packed_12.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16202c6f3b23e702160235646624319b11b84080fd6982a50fcd0778721370b3
+size 33696805

SlimOrca/jirack_sft_packed_13.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bc405e76c87435d0d000ad068298ce3eff1c0273005c96f39e50ee32415abbd
+size 34527205

SlimOrca/jirack_sft_packed_14.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:777e15e0e59d9ad13c79f389a4352b94c6a9326f53511a757a0e6c4a393ad309
+size 34396453

SlimOrca/jirack_sft_packed_15.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:282b51f7b5158974b788e70579c3370dce4629a4cb85384b4f3c3543d5678ead
+size 33908133

SlimOrca/jirack_sft_packed_16.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3849b2e6fcfc83921f4382786dc9297e5875e5bc765469e26c88eee8d182f634
+size 33999269

SlimOrca/jirack_sft_packed_17.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:840f9b1e0bc2dde0996745f87fcdf652d85e8427326d0691222701d981d962ba
+size 33956965

SlimOrca/jirack_sft_packed_18.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:894081c39782024eb083526d9cf3228ea7f3e2a90cc319c74cec9ef7818320a3
+size 33643941

SlimOrca/jirack_sft_packed_19.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dbf7c705325c03d98f188e6ef4c6211f244636564bddb3b0afccb984287b2d1
+size 34039077

SlimOrca/jirack_sft_packed_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8c0676e9c0c15f3d8c13797c0c3df79167bed984941e561dc6a9b92398afbc4
+size 34428687

SlimOrca/jirack_sft_packed_20.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66b2a95277b4b7ba603a892d644cbc6027d040bf6d4e53845e3349bc51f4b035
+size 34481957

SlimOrca/jirack_sft_packed_21.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec1e60c2962b9866aa329f8c017020377647b414a68001082914ede12c25851c
+size 34069093

SlimOrca/jirack_sft_packed_22.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f665e399e7b4cc253e2775ca315edf2ec8cd179f61602d9ca15180df3be35ffa
+size 34640229

SlimOrca/jirack_sft_packed_23.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18a334163c51ff766253aa687c3b03a0d45b9eb84d9b4213f16fa1ede30a85e6
+size 33966117

SlimOrca/jirack_sft_packed_24.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38ac9d6f2ea13fedea1d7bcbcbcbc7fce9549faba4050deccc12ef8f28746202
+size 33783205

SlimOrca/jirack_sft_packed_25.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8416efdfe0e0f8b0aca80742c3ad3690cc946c8815889db8cb3d41c789f45b3
+size 33865701

SlimOrca/jirack_sft_packed_26.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cd0e77dbab8b2a3b959ecf5e5c17644972036b57e2f4b4be88b71e34935cdd6
+size 34137445

SlimOrca/jirack_sft_packed_27.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e9d91ed03a1b0b4927d0b21b9d8090fce546376d7c97996204f2ab3fcb7a6b1
+size 33831845

SlimOrca/jirack_sft_packed_28.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edfc47d64c197798fa066104780d52b79c09907082428724c90abede6636d7bc
+size 33174117

SlimOrca/jirack_sft_packed_29.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e28374a7bb8eb7abe01206e1b97f0ff66a1dfc0ce717a60d58dc2875416393ee
+size 34141605

SlimOrca/jirack_sft_packed_3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24dc76829bc8be947319d4719993f0a40a19e9a3da6772846bc591ffabb50bad
+size 33995215

SlimOrca/jirack_sft_packed_30.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf249fdc1666843217e4ad120bc1f6dfb424df12b2670c799547373d550afe13
+size 33769765

SlimOrca/jirack_sft_packed_31.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:209d1501f895707959869aebee63ff604c43aa06946113b51e01bf744c45dd67
+size 34400421

SlimOrca/jirack_sft_packed_32.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48a816f17ffa1078850f3b037ab4f7247ab17b8b551bef5a8a4a4572b54a5edd
+size 33672485

SlimOrca/jirack_sft_packed_33.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88073a2b1a7e9d1fcb04274a4fcda6f2ef6153ea8e059a69f48428b478e34694
+size 34558053

SlimOrca/jirack_sft_packed_34.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f70862e9875ead2854dc3e0dd90f9244f2b92fe959631785b1f1f75273f89e3f
+size 33598629

SlimOrca/jirack_sft_packed_35.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:deba10021b55e39784d845890e15f9337bb812794374ce54b193ff8915972219
+size 33816997

SlimOrca/jirack_sft_packed_36.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ad607bc3a3ad84088f5253dc2a814412746a810043f9c10582226ba8a117dc9
+size 33884901

SlimOrca/jirack_sft_packed_37.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f093bf4f917b72a9ba00fba047a2bc73e1097f63d456f3bfed30b1e34ecb1e02
+size 34127333

SlimOrca/jirack_sft_packed_38.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e56fad06c174316873863593f857b3354b41cb8d240df237cad0ef5ac0e5f253
+size 33192741

SlimOrca/jirack_sft_packed_39.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:348ca781bf4b823de9c87929255767eb3568d0f4657ad7d339eb12b24f3a5fbd
+size 33145637

SlimOrca/jirack_sft_packed_4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97cd19091a8eba560f4bfae4d9d89a8ffd3330c7fc926a0565e3101d8e55b3d5
+size 34189071

SlimOrca/jirack_sft_packed_40.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0aca45d7597c3a81aa854b95981d1a1cef920b5061a7a74b8f72a1fac07726f7
+size 34063077

SlimOrca/jirack_sft_packed_41.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40f60722f494acd7a027df3e802782d6feb82e0d7d216eda8ce1f5e2456f02d7
+size 33934501

SlimOrca/jirack_sft_packed_42.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:217986951ab011aacec05b49ba214be3efa494588070aa06e86f1f813c6c59e1
+size 33815269

SlimOrca/jirack_sft_packed_43.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:baba4ec2da23cef8cffcc079ecb410152c7ebc8e6a7c60b3f7fa5762e283e262
+size 33326949

SlimOrca/jirack_sft_packed_44.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc15dc6194c83948ac5bb1e5d2d3cb5bada5912cdeb9f957be1ae4c2e4d9d62f
+size 33656293

SlimOrca/jirack_sft_packed_45.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51832d39b8b7ed5bfa2eb18ac2dd3164d9247ee6ab142c425707852a7ea28599
+size 33168869

SlimOrca/jirack_sft_packed_46.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc8ee0a729fbf13bf34530714321598bce5e46ebd70e701c2cff320e71d3a2d1
+size 33945381

SlimOrca/jirack_sft_packed_47.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af7188b241bce3524c7fed2cc384e5c4158651f5672990eb36364edf23150116
+size 34026149