Add tokenizer_config.json

tokenizer_config.json

@@ -0,0 +1,28 @@
+{
+  "tokenizer_class": "NeuralQuantumOllamaTokenizer",
+  "auto_map": {
+    "AutoTokenizer": "tokenization_ollama.NeuralQuantumOllamaTokenizer"
+  },
+  "vocab_size": 50257,
+  "model_max_length": 2048,
+  "padding_side": "right",
+  "truncation_side": "right",
+  "special_tokens": {
+    "bos_token": "<|endoftext|>",
+    "eos_token": "<|endoftext|>",
+    "unk_token": "<|endoftext|>",
+    "pad_token": "<|endoftext|>",
+    "quantum_token": "<|quantum|>",
+    "classical_token": "<|classical|>",
+    "system_token": "<|system|>",
+    "user_token": "<|user|>",
+    "assistant_token": "<|assistant|>"
+  },
+  "clean_up_tokenization_spaces": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "quantum_enhanced": true,
+  "ollama_optimized": true
+}