{"id":81219,"library":"tokenizer","title":"Tokenizer","description":"A multilingual tokenizer for NLP tasks supporting German, English, and French texts.","status":"active","version":"0.3.0","language":"ruby","source_language":null,"source_url":"https://github.com/arbox/tokenizer","tags":["nlp","tokenization","multilingual","cli"],"install":[{"cmd":"gem install tokenizer","lang":"bash","label":"gem install"},{"cmd":"# Gemfile\ngem 'tokenizer', '~> 0.3.0'","lang":"ruby","label":"Gemfile"}],"dependencies":[],"imports":[{"symbol":"Tokenizer","correct":"require 'tokenizer'"}],"quickstart":{"code":"require 'tokenizer'\ntokens = Tokenizer.tokenize('Hello world!', language: :english)\nputs tokens.inspect","lang":"ruby","description":"Tokenize an English sentence."},"warnings":[],"env_vars":null,"search_vec":"'cli':17 'english':10 'french':12 'german':9 'multilingu':3,16 'nlp':6,14 'support':8 'task':7 'text':13 'token':1,4,15","created_at":"2026-06-18T05:47:45.053452+00:00","updated_at":"2026-06-18T05:47:45.053452+00:00","problems":[],"ecosystem":"rubygems","meta_description":null,"install_score":null,"quickstart_score":null,"quickstart_tag":null,"pypi_latest":null,"cli_name":null,"cli_version":null,"type":"library","homepage":"https://github.com/arbox/tokenizer","github":"https://github.com/arbox/tokenizer","docs":"http://www.rubydoc.info/gems/tokenizer/","changelog":null,"pypi":null,"npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"categories":["utility"],"base_url":null,"auth_type":null,"provenance":{"verified_status":null,"verified_at":null,"last_verified":"2026-06-18","next_check":"2026-12-15","install_tag":null}}