{"id":58782,"library":"tiktoken-rs","title":"Tiktoken-rs","description":"Rust library for encoding and decoding text using OpenAI's tiktoken tokenizer.","status":"active","version":"0.12.0","language":"rust","source_language":null,"source_url":"https://github.com/zurawiki/tiktoken-rs","tags":["tokenizer","openai","encoding","nlp"],"install":[{"cmd":"# Cargo.toml\n[dependencies]\ntiktoken-rs = \"0.12.0\"","lang":"toml","label":"Cargo.toml"},{"cmd":"cargo add tiktoken-rs","lang":"bash","label":"cargo add"}],"dependencies":[],"imports":[{"symbol":"cl100k_base","correct":"use tiktoken_rs::cl100k_base;"}],"quickstart":{"code":"use tiktoken_rs::cl100k_base;\n\nfn main() {\n    let bpe = cl100k_base().unwrap();\n    let tokens = bpe.encode_with_special_tokens(\"Hello, world!\");\n    println!(\"Token count: {}\", tokens.len());\n}","lang":"rust","description":"Encode a string using the cl100k_base tokenizer."},"warnings":[{"fix":"Ensure network access or pre-cache the tokenizer files.","message":"Requires downloading tokenizer files on first use; may fail without network access.","severity":"gotcha","affected_versions":">=0.1.0"}],"env_vars":null,"search_vec":"'decod':9 'encod':7,18 'librari':5 'nlp':19 'openai':12,17 'rs':3 'rust':4 'text':10 'tiktoken':2,14 'tiktoken-r':1 'token':15,16 'use':11","created_at":"2026-06-16T16:38:28.198070+00:00","updated_at":"2026-06-16T16:38:28.198070+00:00","problems":[],"ecosystem":"crates","meta_description":null,"install_score":null,"quickstart_score":null,"quickstart_tag":null,"pypi_latest":null,"cli_name":null,"cli_version":null,"type":"library","homepage":"https://github.com/zurawiki/tiktoken-rs","github":"https://github.com/zurawiki/tiktoken-rs","docs":"https://docs.rs/tiktoken-rs/","changelog":null,"pypi":null,"npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"categories":["utility"],"base_url":null,"auth_type":null,"provenance":{"verified_status":null,"verified_at":null,"last_verified":"2026-06-16","next_check":"2026-12-13","install_tag":null}}