Download usage.py from Tifinagh/Tifinagh-Unigram-Tokenizer-32K: direct link, hf CLI and curl.
- Browser
- Download file 692 Bytes
-
https://huggingface.co/Tifinagh/Tifinagh-Unigram-Tokenizer-32K/resolve/main/usage.py
- Command line
-
hf download hf://Tifinagh/Tifinagh-Unigram-Tokenizer-32K/usage.py
-
curl -L -o usage.py https://huggingface.co/Tifinagh/Tifinagh-Unigram-Tokenizer-32K/resolve/main/usage.py
692 Bytes
| import sentencepiece as spm | |
| from huggingface_hub import hf_hub_download | |
| # Download the tokenizer model directly from Hugging Face | |
| model_path = hf_hub_download(repo_id="Tifinagh/Tifinagh-Unigram-Tokenizer-32K", filename="tokenizer.model") | |
| # Load the tokenizer model | |
| tokenizer = spm.SentencePieceProcessor() | |
| tokenizer.load(model_path) | |
| # Example usage | |
| text = "ⴰⵣⵓⵍ ⴰⵎⴰⴹⴰⵍ! ⵡⴰ ⵉⴳⴰ ⵉⵔⵉⵎ ⵏ ⵓⵙⵙⵎⵔⵙ ⵏ ⵓⵙⵏⵉⴳⵍ ⵏ ⵓⵙⴽⴽⵉⵍ ⵏ ⵜⴼⵉⵏⴰⵖ" | |
| tokens = tokenizer.encode(text, out_type=str) | |
| decoded = tokenizer.decode(tokenizer.encode(text)) | |
| print(f"Text: {text}") | |
| print(f"Tokens: {tokens}") | |
| print(f"Decoded: {decoded}") |