Upload setup_and_train_300m.sh with huggingface_hub

setup_and_train_300m.sh

@@ -63,37 +63,48 @@ else
     exit 1
 fi
 
-python -c "
+DATA_DIR_ENV="$DATA_DIR" python - <<'PYEOF'
 from pathlib import Path
 import json
-data_dir = Path('$DATA_DIR')
-index_path = data_dir / 'index.json'
+import os
+
+data_dir = Path(os.environ["DATA_DIR_ENV"])
+index_path = data_dir / "index.json"
 if not index_path.exists():
-    candidates = sorted(data_dir.glob('**/index.json'))
+    candidates = sorted(data_dir.glob("**/index.json"))
     if candidates:
-        print('ERROR: index.json still not at dataset root after fetch.')
-        print('Found candidates:')
+        print("ERROR: index.json still not at dataset root after fetch.")
+        print("Found candidates:")
         for candidate in candidates[:10]:
-            print(f'  - {candidate}')
+            print(f"  - {candidate}")
     else:
-        print(f'ERROR: No index.json found under {data_dir}')
+        print(f"ERROR: No index.json found under {data_dir}")
     raise SystemExit(1)
-with open(index_path, encoding='utf-8') as f:
+
+with open(index_path, encoding="utf-8") as f:
     idx = json.load(f)
-total = sum(c.get('dim', 0) for c in idx.get('chunks', []))
-print(f'  Dataset ready: {len(idx.get("chunks", []))} chunks, {total:,} tokens at {index_path}')
-"
+
+chunks = idx.get("chunks", [])
+total = sum(c.get("dim", 0) for c in chunks)
+print(f"  Dataset ready: {len(chunks)} chunks, {total:,} tokens at {index_path}")
+PYEOF
 
 # ── 3. System probe ──────────────────────────────────────────────────────────
 echo ""
 echo "[3/5] System probe..."
-python -c "
-import torch, psutil, os
+python - <<'PYEOF'
+import os
+import psutil
+import torch
+
 props = torch.cuda.get_device_properties(0) if torch.cuda.is_available() else None
-print(f'  GPU  : {props.name if props else \"CPU only\"} ({props.total_memory/1024**3:.1f} GB)' if props else '  GPU: None')
-print(f'  RAM  : {psutil.virtual_memory().total/1024**3:.1f} GB')
-print(f'  CPUs : {os.cpu_count()}')
-"
+if props:
+    print(f"  GPU  : {props.name} ({props.total_memory / 1024**3:.1f} GB)")
+else:
+    print("  GPU: None")
+print(f"  RAM  : {psutil.virtual_memory().total / 1024**3:.1f} GB")
+print(f"  CPUs : {os.cpu_count()}")
+PYEOF
 
 # ── 4. Pretrain (with validation every 100 steps) ─────────────────────────────
 echo ""