{"id":69518,"library":"cohere-ai-tokenizer","title":"Cohere Tokenizer","description":"A Go tokenizer for Cohere's language models, converting text into tokens for API consumption.","status":"active","version":"1.1.2","language":"go","source_language":null,"source_url":"https://github.com/cohere-ai/tokenizer","tags":["cohere","tokenizer","nlp","ai"],"install":[{"cmd":"go get github.com/cohere-ai/tokenizer","lang":"bash","label":"Install github.com/cohere-ai/tokenizer"}],"dependencies":[],"imports":[{"symbol":"tokenizer","correct":"github.com/cohere-ai/tokenizer"}],"quickstart":{"code":"package main\n\nimport (\n\t\"fmt\"\n\t\"github.com/cohere-ai/tokenizer\"\n)\n\nfunc main() {\n\ttokens := tokenizer.Tokenize(\"Hello, world!\")\n\tfmt.Println(tokens)\n}","lang":"go","description":"Tokenizes a string into Cohere tokens."},"warnings":[],"env_vars":null,"search_vec":"'ai':21 'api':16 'coher':1,7,18 'consumpt':17 'convert':11 'go':4 'languag':9 'model':10 'nlp':20 'text':12 'token':2,5,14,19","created_at":"2026-06-17T18:02:52.896121+00:00","updated_at":"2026-06-17T18:02:52.896121+00:00","problems":[],"ecosystem":"go","meta_description":null,"install_score":null,"quickstart_score":null,"quickstart_tag":null,"pypi_latest":null,"cli_name":null,"cli_version":null,"type":"library","homepage":"https://github.com/cohere-ai/tokenizer","github":"https://github.com/cohere-ai/tokenizer","docs":"https://pkg.go.dev/github.com/cohere-ai/tokenizer#section-documentation","changelog":null,"pypi":null,"npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"categories":["utility"],"base_url":null,"auth_type":null,"provenance":{"verified_status":null,"verified_at":null,"last_verified":"2026-06-17","next_check":"2026-12-14","install_tag":null}}