import sentencepiece as spm from huggingface_hub import hf_hub_download # Download the tokenizer model directly from Hugging Face model_path = hf_hub_download(repo_id="Tifinagh/Tifinagh-Unigram-Tokenizer-32K", filename="tokenizer.model") # Load the tokenizer model tokenizer = spm.SentencePieceProcessor() tokenizer.load(model_path) # Example usage text = "ⴰⵣⵓⵍ ⴰⵎⴰⴹⴰⵍ! ⵡⴰ ⵉⴳⴰ ⵉⵔⵉⵎ ⵏ ⵓⵙⵙⵎⵔⵙ ⵏ ⵓⵙⵏⵉⴳⵍ ⵏ ⵓⵙⴽⴽⵉⵍ ⵏ ⵜⴼⵉⵏⴰⵖ" tokens = tokenizer.encode(text, out_type=str) decoded = tokenizer.decode(tokenizer.encode(text)) print(f"Text: {text}") print(f"Tokens: {tokens}") print(f"Decoded: {decoded}")