{"id":46643,"library":"tokie","title":"Tokie","description":"A blazingly fast tokenizer library that is 50x faster and 10x smaller than alternatives, providing 100% accurate encoding/decoding for large language models. Current version 0.0.10 targets Python >=3.10, with weekly release cadence.","status":"active","version":"0.0.10","language":"python","source_language":"en","source_url":"https://github.com/chonkie-inc/tokie","tags":["tokenizer","fast","llm"],"install":[{"cmd":"pip install tokie","lang":"bash","label":"Latest release"}],"dependencies":[{"reason":"Underlying tokenizer implementation for most models","package":"tokenizers","optional":false}],"imports":[{"wrong":"from tokie import Tokie","symbol":"Tokenizer","correct":"from tokie import Tokenizer"},{"symbol":"Encoding","correct":"from tokie import Encoding"},{"symbol":"tokie","correct":"from tokie import tokie"}],"quickstart":{"code":"from tokie import Tokie\ntok = Tokie(model='gpt2')\nencoded = tok.encode('Hello, world!')\nprint(encoded)\ndecoded = tok.decode(encoded)\nprint(decoded)","lang":"python","description":"Initialize a Tokie instance with a model name, then encode/decode strings."},"warnings":[{"fix":"Use `backend='huggingface'` instead of `tokenizer='huggingface'`","message":"The `tokenizer` parameter renaming to `backend` in v0.1.0","severity":"deprecated","affected_versions":">=0.1.0"},{"fix":"Install with appropriate extras: `pip install tokie[all]` to cover all backends","message":"Tokie must be installed with extra dependencies for non-HuggingFace backends (e.g., `pip install tokie[sentencepiece]`)","severity":"gotcha","affected_versions":">=0.0.10"},{"fix":"Provide a valid model name (e.g., 'gpt2', 'bert-base-uncased')","message":"The `model` parameter is required for initialization; no default model","severity":"gotcha","affected_versions":"all"}],"env_vars":null,"search_vec":"'0.0.10':26 '100':17 '10x':12 '3.10':29 '50x':9 'accur':18 'altern':15 'blaze':3 'cadenc':33 'current':24 'encoding/decoding':19 'fast':4,35 'faster':10 'languag':22 'larg':21 'librari':6 'llm':36 'model':23 'provid':16 'python':28 'releas':32 'smaller':13 'target':27 'token':5,34 'toki':1 'version':25 'week':31","created_at":"2026-06-07T13:00:55.200884+00:00","updated_at":"2026-06-07T13:00:55.200884+00:00","problems":null,"ecosystem":"pypi","meta_description":null,"install_score":null,"quickstart_score":null,"quickstart_tag":null,"pypi_latest":null,"cli_name":null,"cli_version":null,"type":"library","homepage":"https://github.com/chonkie-inc/tokie","github":"https://github.com/chonkie-inc/tokie","docs":null,"changelog":null,"pypi":null,"npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"categories":["ai-ml"],"base_url":null,"auth_type":null,"provenance":{"verified_status":"passing","verified_at":"2026-07-03","last_verified":"2026-07-03","next_check":"2026-08-02","install_tag":null}}