Upload tokenizer

README.md

@@ -1,12 +1,12 @@
 ---
+base_model: tiiuae/Falcon3-3B-Instruct
 library_name: transformers
+license: other
+license_name: falcon-llm-license
+license_link: https://falconllm.tii.ae/falcon-terms-and-conditions.html
 tags:
 - bitnet
 - falcon3
-base_model: tiiuae/Falcon3-3B-Instruct
-license: other 
-license_name: falcon-llm-license 
-license_link: https://falconllm.tii.ae/falcon-terms-and-conditions.html
 ---
 
 

special_tokens_map.json

@@ -24,22 +24,15 @@
     ">>PASSWORD<<",
     ">>KEY<<"
   ],
-  "bos_token": {
-    "content": "<|startoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
   "eos_token": {
-    "content": "<|im_end|>",
+    "content": "<|endoftext|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": {
-    "content": "<|endoftext|>",
+    "content": "<|pad|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -1139,7 +1139,7 @@
     },
     {
       "id": 126,
-      "content": "<|im_start|>",
+      "content": ">>UNUSED_0<<",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -1148,7 +1148,7 @@
     },
     {
       "id": 127,
-      "content": "<|im_end|>",
+      "content": ">>UNUSED_1<<",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -18212,7 +18212,7 @@
     },
     {
       "id": 2023,
-      "content": ">>UNUSED_1897<<",
+      "content": "<|pad|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -18383,8 +18383,8 @@
       ">>POS_97<<": 123,
       ">>POS_98<<": 124,
       ">>POS_99<<": 125,
-      "<|im_start|>": 126,
-      "<|im_end|>": 127,
+      ">>UNUSED_0<<": 126,
+      ">>UNUSED_1<<": 127,
       ">>UNUSED_2<<": 128,
       ">>UNUSED_3<<": 129,
       ">>UNUSED_4<<": 130,
@@ -20280,7 +20280,7 @@
       ">>UNUSED_1894<<": 2020,
       ">>UNUSED_1895<<": 2021,
       ">>UNUSED_1896<<": 2022,
-      ">>UNUSED_1897<<": 2023,
+      "<|pad|>": 2023,
       "!": 2024,
       "\"": 2025,
       "#": 2026,

tokenizer_config.json

@@ -1010,7 +1010,7 @@
       "special": true
     },
     "126": {
-      "content": "<|im_start|>",
+      "content": ">>UNUSED_0<<",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -1018,7 +1018,7 @@
       "special": true
     },
     "127": {
-      "content": "<|im_end|>",
+      "content": ">>UNUSED_1<<",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -16186,7 +16186,7 @@
       "special": true
     },
     "2023": {
-      "content": ">>UNUSED_1897<<",
+      "content": "<|pad|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -16219,11 +16219,15 @@
     ">>PASSWORD<<",
     ">>KEY<<"
   ],
-  "bos_token": "<|startoftext|>",
-  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "chat_template": "{% if tools %}{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>\n' + message['content'] + '\nYou are an expert in composing functions. You are given a question and a set of possible functions. \nBased on the question, you will need to make one or more function/tool calls to achieve the purpose. \nIf none of the functions can be used, point it out and refuse to answer. \nIf the given question lacks the parameters required by the function, also point it out.\n\n You have access to the following tools:\n<tools>'  + tools|tojson + '</tools>\n\nThe output MUST strictly adhere to the following format, and NO other text MUST be included.\nThe example format is as follows. Please make sure the parameter type is correct. If no function call is needed, please make the tool calls an empty list [].\n<tool_call>[\n{\"name\": \"function_name1\", \"arguments\": {\"argument1\": \"value1\", \"argument2\": \"value2\"}},\n... (more tool calls as required)\n]</tool_call>' }}{% elif message['role'] == 'user' %}{{ '<|user|>\n' + message['content'] + '\n' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>\n' + message['content'] + eos_token + '\n' }}{% else %}{{ '<|assistant|>\n' + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>\n' }}{% endif %}{% endfor %}{% else %}{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>\n' + message['content'] + '\n' }}{% elif message['role'] == 'user' %}{{ '<|user|>\n' + message['content'] + '\n' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>\n' + message['content'] + eos_token + '\n' }}{% else %}{{ '<|assistant|>\n' + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>\n' }}{% endif %}{% endfor %}{% endif %}",
   "clean_up_tokenization_spaces": true,
-  "eos_token": "<|im_end|>",
-  "model_max_length": 8192,
-  "pad_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 32768,
+  "pad_token": "<|pad|>",
   "tokenizer_class": "PreTrainedTokenizerFast"
 }