Akseltinfat's picture
Update usage.py
9b3d5a6 verified
Raw History Blame Contribute Delete
692 Bytes
import sentencepiece as spm
from huggingface_hub import hf_hub_download
# Download the tokenizer model directly from Hugging Face
model_path = hf_hub_download(repo_id="Tifinagh/Tifinagh-Unigram-Tokenizer-32K", filename="tokenizer.model")
# Load the tokenizer model
tokenizer = spm.SentencePieceProcessor()
tokenizer.load(model_path)
# Example usage
text = "ⴰⵣⵓⵍ ⴰⵎⴰⴹⴰⵍ! ⵡⴰ ⵉⴳⴰ ⵉⵔⵉⵎ ⵏ ⵓⵙⵙⵎⵔⵙ ⵏ ⵓⵙⵏⵉⴳⵍ ⵏ ⵓⵙⴽⴽⵉⵍ ⵏ ⵜⴼⵉⵏⴰⵖ"
tokens = tokenizer.encode(text, out_type=str)
decoded = tokenizer.decode(tokenizer.encode(text))
print(f"Text: {text}")
print(f"Tokens: {tokens}")
print(f"Decoded: {decoded}")