feat(noun): LRNounExtractor 멀티프로세싱 지원 (#160)

.pre-commit-config.yaml

@@ -6,6 +6,7 @@ repos:
     hooks:
       - id: pyright
         entry: pyright
+        language_version: python3.12
         additional_dependencies:
           - pytest==8.3.0
           - dacite==1.9.1

soynlp/core/lrgraph.py

@@ -151,5 +151,73 @@ def load(cls, path: str) -> "LRGraph":
         return cls(lr_data)
 
 
-def corpus_to_lrgraph(texts: list[str], l_max_length: int = 10, r_max_length: int = 9) -> LRGraph:
-    return LRGraph.from_sents(texts, max_l_length=l_max_length, max_r_length=r_max_length)
+def _build_partial_counter(args: tuple[list[str], int, int]) -> dict[str, dict[str, int]]:
+    """Module-level worker function: builds a partial LR counter from a chunk of texts.
+
+    Args:
+        args: tuple of (texts_chunk, l_max_length, r_max_length)
+
+    Returns:
+        dict of {L: {R: frequency}}
+    """
+    texts_chunk, l_max_length, r_max_length = args
+    counter: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+    for sent in texts_chunk:
+        for word in sent.split():
+            word = word.strip()
+            for e in range(1, min(len(word), l_max_length) + 1):
+                L, R = word[:e], word[e:]
+                if len(R) > r_max_length:
+                    continue
+                counter[L][R] += 1
+    return {L: dict(R_freq) for L, R_freq in counter.items()}
+
+
+def _merge_counters(counters: list[dict[str, dict[str, int]]]) -> dict[str, dict[str, int]]:
+    """Merge a list of partial LR counters into one."""
+    merged: dict[str, dict[str, int]] = {}
+    for counter in counters:
+        for L, R_freq in counter.items():
+            if L not in merged:
+                merged[L] = {}
+            for R, freq in R_freq.items():
+                merged[L][R] = merged[L].get(R, 0) + freq
+    return merged
+
+
+def corpus_to_lrgraph(texts: list[str], l_max_length: int = 10, r_max_length: int = 9, n_workers: int = 1) -> LRGraph:
+    """Build an LRGraph from a list of texts.
+
+    Args:
+        texts: list of sentences
+        l_max_length: maximum length of L parts
+        r_max_length: maximum length of R parts
+        n_workers: number of worker processes. Use -1 to use all CPU cores.
+
+    Returns:
+        LRGraph built from the input texts
+    """
+    if n_workers == -1:
+        n_workers = os.cpu_count() or 1
+
+    if n_workers <= 1:
+        return LRGraph.from_sents(texts, max_l_length=l_max_length, max_r_length=r_max_length)
+
+    # Ensure texts is a list for chunking
+    if not isinstance(texts, list):
+        texts = list(texts)
+
+    # Split texts into n_workers chunks
+    chunk_size = max(1, len(texts) // n_workers)
+    chunks = [texts[i : i + chunk_size] for i in range(0, len(texts), chunk_size)]
+
+    # Build partial counters in parallel
+    from multiprocessing import Pool
+
+    worker_args = [(chunk, l_max_length, r_max_length) for chunk in chunks]
+    with Pool(processes=n_workers) as pool:
+        partial_counters = pool.map(_build_partial_counter, worker_args)
+
+    # Merge all partial counters and build LRGraph
+    merged = _merge_counters(partial_counters)
+    return LRGraph(merged, max_l_length=l_max_length, max_r_length=r_max_length)

soynlp/noun/lr.py

@@ -109,6 +109,7 @@ def extract(
         exclude_syllables: bool = False,
         exclude_numbers: bool = True,
         custom_exclude_function: Callable[[str], bool] | None = None,
+        n_workers: int = 1,
     ) -> dict[str, NounScore]:
         """Extract nouns from `train_data` or trained L-R graph
 
@@ -187,7 +188,9 @@ def extract(
             raise ValueError("`train_data` must not be `None` if noun extractor has no LRGraph")
 
         if train_data is not None:
-            self.lrgraph = train_lrgraph(train_data, min_eojeol_frequency, self.max_l_length, self.max_r_length, self.verbose)
+            self.lrgraph = train_lrgraph(
+                train_data, min_eojeol_frequency, self.max_l_length, self.max_r_length, self.verbose, n_workers
+            )
         else:
             if self.lrgraph is None:
                 raise ValueError("`train_data` must not be `None` if noun extractor has no LRGraph")
@@ -404,7 +407,10 @@ def train_lrgraph(
     max_l_length: int,
     max_r_length: int,
     verbose: bool,
+    n_workers: int = 1,
 ) -> LRGraph:
+    from soynlp.core import corpus_to_lrgraph
+
     if isinstance(train_data, LRGraph):
         logger.info("input is LRGraph")
         return train_data
@@ -418,6 +424,26 @@ def train_lrgraph(
         fmt = "jsonl" if train_data.endswith(".jsonl") else "text"
         train_data = CorpusLoader(train_data, format=fmt)
 
+    if n_workers != 1:
+        # Use corpus_to_lrgraph with multiprocessing support
+        texts_list: list[str] = train_data if isinstance(train_data, list) else [str(s) for s in train_data]
+        # Apply min_eojeol_frequency filter via EojeolCounter first if needed
+        if min_eojeol_frequency > 1:
+            # min_eojeol_frequency > 1이면 빈도 필터링이 필요하므로 EojeolCounter 경로 사용 (단일 프로세스)
+            logger.info("min_eojeol_frequency > 1: n_workers 무시, 단일 프로세스로 LRGraph 구축")
+            eojeol_counter = EojeolCounter(
+                sents=texts_list,
+                min_count=min_eojeol_frequency,
+                max_length=(max_l_length + max_r_length),
+                verbose=verbose,
+            )
+            lrgraph = eojeol_counter.to_lrgraph(max_l_length, max_r_length)
+            logger.info(f"finished building LRGraph from {len(eojeol_counter)} eojeols")
+            return lrgraph
+        lrgraph = corpus_to_lrgraph(texts_list, l_max_length=max_l_length, r_max_length=max_r_length, n_workers=n_workers)
+        logger.info(f"finished building LRGraph with n_workers={n_workers}")
+        return lrgraph
+
     eojeol_counter = EojeolCounter(
         sents=train_data,
         min_count=min_eojeol_frequency,

tests/integration/examples/extract_frequent_nouns/verify.py

@@ -1,15 +1,32 @@
 """Verify frequent noun extraction results."""
 
+import os
+
+from soynlp.noun import LRNounExtractor
+from soynlp.utils import CorpusLoader
+
+ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../.."))
+DATA_PATH = os.path.join(ROOT_DIR, "tests/integration/data/news-text/2016-10-20.jsonl")
+
 
 def _read_lines(path: str) -> list[str]:
     with open(path, encoding="utf-8") as f:
         return [line for line in f.read().splitlines() if line.strip()]
 
 
 def verify(parameters: dict, answers_dir: str) -> None:
+    # 단일 프로세스 결과 검증 (pipeline 실행 결과)
     nouns = parameters["nouns"]
     top_nouns = sorted(nouns.items(), key=lambda x: -x[1].frequency)[:100]
     lines = [f"{noun}\t{score.frequency}\t{score.score:.4f}" for noun, score in top_nouns]
-
     expected = _read_lines(f"{answers_dir}/top_nouns.txt")
     assert lines == expected
+
+    # 멀티프로세싱 결과 검증 (n_workers=4 결과가 단일 프로세스와 동일한지 확인)
+    corpus = CorpusLoader(DATA_PATH, format="jsonl", verbose=False)
+    sents = [item["text"] for item in corpus]
+    extractor_multi = LRNounExtractor(verbose=False)
+    nouns_multi = extractor_multi.extract(sents, min_noun_frequency=10, n_workers=4)
+    assert set(nouns.keys()) == set(nouns_multi.keys()), (
+        f"멀티프로세싱 결과 불일치: single={len(nouns)}, multi={len(nouns_multi)}"
+    )

tests/integration/examples/movie_review_nouns/verify.py

@@ -64,3 +64,16 @@ def verify(answers_dir: str) -> None:
     actual_comparison = "\n".join(lines) + "\n"
     expected_comparison = _read_answer(f"{answers_dir}/domain_comparison.txt")
     assert actual_comparison == expected_comparison
+
+    # 멀티프로세싱 결과 검증 (n_workers=4 결과가 단일 프로세스와 동일한지 확인)
+    news_extractor_multi = LRNounExtractor(verbose=False)
+    news_nouns_multi = news_extractor_multi.extract(news_sents, min_noun_frequency=10, n_workers=4)
+    assert set(news_nouns.keys()) == set(news_nouns_multi.keys()), (
+        f"뉴스 멀티프로세싱 결과 불일치: single={len(news_nouns)}, multi={len(news_nouns_multi)}"
+    )
+
+    review_extractor_multi = LRNounExtractor(verbose=False)
+    review_nouns_multi = review_extractor_multi.extract(review_sents, min_noun_frequency=10, n_workers=4)
+    assert set(review_nouns.keys()) == set(review_nouns_multi.keys()), (
+        f"리뷰 멀티프로세싱 결과 불일치: single={len(review_nouns)}, multi={len(review_nouns_multi)}"
+    )

tests/unit/test_multiprocessing.py

@@ -0,0 +1,63 @@
+"""Tests for multiprocessing support in LRNounExtractor."""
+
+from soynlp.core import LRGraph, corpus_to_lrgraph
+from soynlp.noun import LRNounExtractor
+
+# 테스트용 샘플 문장 (충분한 양)
+SAMPLE_SENTS = [
+    "아이오아이가 평가단에게 높은 점수를 받았습니다",
+    "아이오아이는 아이돌 그룹입니다",
+    "자연어처리는 어렵습니다",
+    "자연어처리를 공부합니다",
+    "명사추출은 중요한 작업입니다",
+] * 100  # 충분한 반복
+
+
+def test_corpus_to_lrgraph_single():
+    """단일 프로세스로 LRGraph를 구축한다."""
+    lrgraph = corpus_to_lrgraph(SAMPLE_SENTS, n_workers=1)
+    assert isinstance(lrgraph, LRGraph)
+    assert len(lrgraph._lr) > 0
+
+
+def test_corpus_to_lrgraph_multi():
+    """멀티프로세스로 구축한 LRGraph가 단일 프로세스 결과와 동일하다."""
+    lrgraph_single = corpus_to_lrgraph(SAMPLE_SENTS, n_workers=1)
+    lrgraph_multi = corpus_to_lrgraph(SAMPLE_SENTS, n_workers=4)
+    assert lrgraph_single._lr == lrgraph_multi._lr
+
+
+def test_noun_extractor_single():
+    """단일 프로세스로 명사를 추출한다."""
+    extractor = LRNounExtractor(verbose=False)
+    nouns = extractor.extract(SAMPLE_SENTS, n_workers=1)
+    assert len(nouns) > 0
+
+
+def test_noun_extractor_multi():
+    """n_workers=4로 추출한 명사 결과가 단일 프로세스와 동일하다."""
+    extractor_single = LRNounExtractor(verbose=False)
+    nouns_single = extractor_single.extract(SAMPLE_SENTS, n_workers=1)
+
+    extractor_multi = LRNounExtractor(verbose=False)
+    nouns_multi = extractor_multi.extract(SAMPLE_SENTS, n_workers=4)
+
+    assert set(nouns_single.keys()) == set(nouns_multi.keys())
+
+
+def test_noun_extractor_auto_workers():
+    """n_workers=-1이면 CPU 코어 수를 자동으로 사용한다."""
+    extractor = LRNounExtractor(verbose=False)
+    nouns = extractor.extract(SAMPLE_SENTS, n_workers=-1)
+    assert len(nouns) > 0
+
+
+def test_noun_extractor_multi_with_min_eojeol_frequency():
+    """min_eojeol_frequency > 1이면 n_workers를 지정해도 단일 프로세스로 동작하며 결과는 동일하다."""
+    extractor_single = LRNounExtractor(verbose=False)
+    nouns_single = extractor_single.extract(SAMPLE_SENTS, min_eojeol_frequency=2, n_workers=1)
+
+    extractor_multi = LRNounExtractor(verbose=False)
+    nouns_multi = extractor_multi.extract(SAMPLE_SENTS, min_eojeol_frequency=2, n_workers=4)
+
+    assert set(nouns_single.keys()) == set(nouns_multi.keys())