{"id":27408,"library":"soynlp","title":"soynlp","description":"An unsupervised Korean Natural Language Processing toolkit for tokenization, stemming, part-of-speech tagging, and noun extraction. The current version is 0.0.493. Development has stalled since 2020; the repository is archived and no longer maintained as of version 0.1.1 (which is not on PyPI).","status":"deprecated","version":"0.0.493","language":"python","source_language":"en","source_url":"https://github.com/lovit/soynlp","tags":["korean-nlp","unsupervised","tokenization","noun-extraction"],"install":[{"cmd":"pip install soynlp","lang":"bash","label":"PyPI install"}],"dependencies":[{"reason":"Used for system resource monitoring during tokenization","package":"psutil","optional":false},{"reason":"Used for some vectorization utilities","package":"scikit-learn","optional":true}],"imports":[{"note":"Correct import for noun extraction","symbol":"soynlp.noun.LRNounExtractor_v2","correct":"from soynlp.noun import LRNounExtractor_v2"},{"note":"Correct import for tokenization","symbol":"soynlp.tokenizer.MaxScoreTokenizer","correct":"from soynlp.tokenizer import MaxScoreTokenizer"},{"note":"Correct import for text normalization (repeat char, emoticon, etc.)","symbol":"soynlp.normalize","correct":"from soynlp import normalize"},{"note":"Correct import for word extraction (note: 'word' not 'words')","symbol":"soynlp.words.WordExtractor","correct":"from soynlp.word import WordExtractor"}],"quickstart":{"code":"from soynlp import DoublespaceLineCorpus\nfrom soynlp.word import WordExtractor\nfrom soynlp.tokenizer import LTokenizer\n\ncorpus = DoublespaceLineCorpus('dataset.txt', iter_sent=True)\nword_extractor = WordExtractor(max_iter_learning_steps=100)\nword_extractor.train(corpus)\nscores = word_extractor.extract()\nscores = {word:score.cohesion_forward for word, score in scores.items()}\ntokenizer = LTokenizer(scores=scores)\ntext = '한국어 자연어 처리'\nprint(tokenizer.tokenize(text))","lang":"python","description":"Basic unsupervised tokenization using word extraction scores."},"warnings":[{"fix":"Consider migrating to modern Korean NLP libraries such as Kiwi (kiwipiepy), KoNLPy, or Hugging Face tokenizers.","message":"The repository is archived on GitHub (last release 0.1.1, not on PyPI). PyPI version 0.0.493 is several years old and will not receive updates.","severity":"deprecated","affected_versions":"all"},{"fix":"Replace 'from soynlp.words import WordExtractor' with 'from soynlp.word import WordExtractor'.","message":"In some versions, import paths changed. Using 'soynlp.words' (with an 's') will fail; use 'soynlp.word' instead.","severity":"breaking","affected_versions":">=0.0.400"},{"fix":"Extract scores as shown in quickstart: scores = {word:score.cohesion_forward for word, score in scores.items()}.","message":"The LTokenzier sorce dictionary must be a dict mapping word to a float score (e.g., cohesion_forward). Passing raw WordExtractor output will cause a TypeError.","severity":"gotcha","affected_versions":"all"},{"fix":"Ensure input file has tokens separated by two spaces, or preprocess accordingly.","message":"DoublespaceLineCorpus expects double-space separated tokens per line, not arbitrary text. Using raw text lines will produce garbage corpus sentences.","severity":"gotcha","affected_versions":"all"}],"env_vars":null,"search_vec":"'0.0.493':24 '0.1.1':41 '2020':29 'archiv':33 'current':21 'develop':25 'extract':19,54 'korean':4,48 'korean-nlp':47 'languag':6 'longer':36 'maintain':37 'natur':5 'nlp':49 'noun':18,53 'noun-extract':52 'part':13 'part-of-speech':12 'process':7 'pypi':46 'repositori':31 'sinc':28 'soynlp':1 'speech':15 'stall':27 'stem':11 'tag':16 'token':10,51 'toolkit':8 'unsupervis':3,50 'version':22,40","created_at":"2026-05-01T17:46:06.764372+00:00","updated_at":"2026-05-01T17:46:06.764372+00:00","problems":[{"fix":"Use 'from soynlp.word import WordExtractor' instead.","cause":"Incorrect import path; the correct module is 'soynlp.word' (singular).","error":"ModuleNotFoundError: No module named 'soynlp.words'"},{"fix":"Extract scores: scores = {word:score.cohesion_forward for word, score in word_extractor.extract().items()}","cause":"Passing the WordExtractor object directly as a tokenizer score dictionary; expected a dict.","error":"TypeError: 'WordExtractor' object is not iterable"},{"fix":"Call word_extractor.train(corpus) before word_extractor.extract()","cause":"Calling extract() without calling train() first on a corpus.","error":"ValueError: No corpus was trained"}],"ecosystem":"pypi","meta_description":null,"install_score":null,"quickstart_score":null,"quickstart_tag":null,"pypi_latest":null,"cli_name":"","cli_version":null,"type":"library","homepage":"https://soynlp.github.io/soynlp/","github":"https://github.com/lovit/soynlp","docs":null,"changelog":null,"pypi":"https://pypi.org/project/soynlp/","npm":null,"openapi_spec":null,"status_page":null,"smithery":null,"categories":["ai-ml"],"base_url":null,"auth_type":null,"provenance":{"verified_status":"passing","verified_at":"2026-06-28","last_verified":"2026-06-28","next_check":"2026-07-29","install_tag":null}}