diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 97d50c1b..a003ec1a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,6 +6,7 @@ repos: hooks: - id: pyright entry: pyright + language_version: python3.12 additional_dependencies: - pytest==8.3.0 - dacite==1.9.1 diff --git a/soynlp/core/lrgraph.py b/soynlp/core/lrgraph.py index f52064bf..27478583 100644 --- a/soynlp/core/lrgraph.py +++ b/soynlp/core/lrgraph.py @@ -151,5 +151,73 @@ def load(cls, path: str) -> "LRGraph": return cls(lr_data) -def corpus_to_lrgraph(texts: list[str], l_max_length: int = 10, r_max_length: int = 9) -> LRGraph: - return LRGraph.from_sents(texts, max_l_length=l_max_length, max_r_length=r_max_length) +def _build_partial_counter(args: tuple[list[str], int, int]) -> dict[str, dict[str, int]]: + """Module-level worker function: builds a partial LR counter from a chunk of texts. + + Args: + args: tuple of (texts_chunk, l_max_length, r_max_length) + + Returns: + dict of {L: {R: frequency}} + """ + texts_chunk, l_max_length, r_max_length = args + counter: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int)) + for sent in texts_chunk: + for word in sent.split(): + word = word.strip() + for e in range(1, min(len(word), l_max_length) + 1): + L, R = word[:e], word[e:] + if len(R) > r_max_length: + continue + counter[L][R] += 1 + return {L: dict(R_freq) for L, R_freq in counter.items()} + + +def _merge_counters(counters: list[dict[str, dict[str, int]]]) -> dict[str, dict[str, int]]: + """Merge a list of partial LR counters into one.""" + merged: dict[str, dict[str, int]] = {} + for counter in counters: + for L, R_freq in counter.items(): + if L not in merged: + merged[L] = {} + for R, freq in R_freq.items(): + merged[L][R] = merged[L].get(R, 0) + freq + return merged + + +def corpus_to_lrgraph(texts: list[str], l_max_length: int = 10, r_max_length: int = 9, n_workers: int = 1) -> LRGraph: + """Build an LRGraph from a list of texts. + + Args: + texts: list of sentences + l_max_length: maximum length of L parts + r_max_length: maximum length of R parts + n_workers: number of worker processes. Use -1 to use all CPU cores. + + Returns: + LRGraph built from the input texts + """ + if n_workers == -1: + n_workers = os.cpu_count() or 1 + + if n_workers <= 1: + return LRGraph.from_sents(texts, max_l_length=l_max_length, max_r_length=r_max_length) + + # Ensure texts is a list for chunking + if not isinstance(texts, list): + texts = list(texts) + + # Split texts into n_workers chunks + chunk_size = max(1, len(texts) // n_workers) + chunks = [texts[i : i + chunk_size] for i in range(0, len(texts), chunk_size)] + + # Build partial counters in parallel + from multiprocessing import Pool + + worker_args = [(chunk, l_max_length, r_max_length) for chunk in chunks] + with Pool(processes=n_workers) as pool: + partial_counters = pool.map(_build_partial_counter, worker_args) + + # Merge all partial counters and build LRGraph + merged = _merge_counters(partial_counters) + return LRGraph(merged, max_l_length=l_max_length, max_r_length=r_max_length) diff --git a/soynlp/noun/lr.py b/soynlp/noun/lr.py index c8ba1317..6daaeb01 100644 --- a/soynlp/noun/lr.py +++ b/soynlp/noun/lr.py @@ -109,6 +109,7 @@ def extract( exclude_syllables: bool = False, exclude_numbers: bool = True, custom_exclude_function: Callable[[str], bool] | None = None, + n_workers: int = 1, ) -> dict[str, NounScore]: """Extract nouns from `train_data` or trained L-R graph @@ -187,7 +188,9 @@ def extract( raise ValueError("`train_data` must not be `None` if noun extractor has no LRGraph") if train_data is not None: - self.lrgraph = train_lrgraph(train_data, min_eojeol_frequency, self.max_l_length, self.max_r_length, self.verbose) + self.lrgraph = train_lrgraph( + train_data, min_eojeol_frequency, self.max_l_length, self.max_r_length, self.verbose, n_workers + ) else: if self.lrgraph is None: raise ValueError("`train_data` must not be `None` if noun extractor has no LRGraph") @@ -404,7 +407,10 @@ def train_lrgraph( max_l_length: int, max_r_length: int, verbose: bool, + n_workers: int = 1, ) -> LRGraph: + from soynlp.core import corpus_to_lrgraph + if isinstance(train_data, LRGraph): logger.info("input is LRGraph") return train_data @@ -418,6 +424,26 @@ def train_lrgraph( fmt = "jsonl" if train_data.endswith(".jsonl") else "text" train_data = CorpusLoader(train_data, format=fmt) + if n_workers != 1: + # Use corpus_to_lrgraph with multiprocessing support + texts_list: list[str] = train_data if isinstance(train_data, list) else [str(s) for s in train_data] + # Apply min_eojeol_frequency filter via EojeolCounter first if needed + if min_eojeol_frequency > 1: + # min_eojeol_frequency > 1이면 빈도 필터링이 필요하므로 EojeolCounter 경로 사용 (단일 프로세스) + logger.info("min_eojeol_frequency > 1: n_workers 무시, 단일 프로세스로 LRGraph 구축") + eojeol_counter = EojeolCounter( + sents=texts_list, + min_count=min_eojeol_frequency, + max_length=(max_l_length + max_r_length), + verbose=verbose, + ) + lrgraph = eojeol_counter.to_lrgraph(max_l_length, max_r_length) + logger.info(f"finished building LRGraph from {len(eojeol_counter)} eojeols") + return lrgraph + lrgraph = corpus_to_lrgraph(texts_list, l_max_length=max_l_length, r_max_length=max_r_length, n_workers=n_workers) + logger.info(f"finished building LRGraph with n_workers={n_workers}") + return lrgraph + eojeol_counter = EojeolCounter( sents=train_data, min_count=min_eojeol_frequency, diff --git a/tests/integration/examples/extract_frequent_nouns/verify.py b/tests/integration/examples/extract_frequent_nouns/verify.py index e3a0636b..29676e71 100644 --- a/tests/integration/examples/extract_frequent_nouns/verify.py +++ b/tests/integration/examples/extract_frequent_nouns/verify.py @@ -1,5 +1,13 @@ """Verify frequent noun extraction results.""" +import os + +from soynlp.noun import LRNounExtractor +from soynlp.utils import CorpusLoader + +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..")) +DATA_PATH = os.path.join(ROOT_DIR, "tests/integration/data/news-text/2016-10-20.jsonl") + def _read_lines(path: str) -> list[str]: with open(path, encoding="utf-8") as f: @@ -7,9 +15,18 @@ def _read_lines(path: str) -> list[str]: def verify(parameters: dict, answers_dir: str) -> None: + # 단일 프로세스 결과 검증 (pipeline 실행 결과) nouns = parameters["nouns"] top_nouns = sorted(nouns.items(), key=lambda x: -x[1].frequency)[:100] lines = [f"{noun}\t{score.frequency}\t{score.score:.4f}" for noun, score in top_nouns] - expected = _read_lines(f"{answers_dir}/top_nouns.txt") assert lines == expected + + # 멀티프로세싱 결과 검증 (n_workers=4 결과가 단일 프로세스와 동일한지 확인) + corpus = CorpusLoader(DATA_PATH, format="jsonl", verbose=False) + sents = [item["text"] for item in corpus] + extractor_multi = LRNounExtractor(verbose=False) + nouns_multi = extractor_multi.extract(sents, min_noun_frequency=10, n_workers=4) + assert set(nouns.keys()) == set(nouns_multi.keys()), ( + f"멀티프로세싱 결과 불일치: single={len(nouns)}, multi={len(nouns_multi)}" + ) diff --git a/tests/integration/examples/movie_review_nouns/verify.py b/tests/integration/examples/movie_review_nouns/verify.py index af783e0c..dca6cac2 100644 --- a/tests/integration/examples/movie_review_nouns/verify.py +++ b/tests/integration/examples/movie_review_nouns/verify.py @@ -64,3 +64,16 @@ def verify(answers_dir: str) -> None: actual_comparison = "\n".join(lines) + "\n" expected_comparison = _read_answer(f"{answers_dir}/domain_comparison.txt") assert actual_comparison == expected_comparison + + # 멀티프로세싱 결과 검증 (n_workers=4 결과가 단일 프로세스와 동일한지 확인) + news_extractor_multi = LRNounExtractor(verbose=False) + news_nouns_multi = news_extractor_multi.extract(news_sents, min_noun_frequency=10, n_workers=4) + assert set(news_nouns.keys()) == set(news_nouns_multi.keys()), ( + f"뉴스 멀티프로세싱 결과 불일치: single={len(news_nouns)}, multi={len(news_nouns_multi)}" + ) + + review_extractor_multi = LRNounExtractor(verbose=False) + review_nouns_multi = review_extractor_multi.extract(review_sents, min_noun_frequency=10, n_workers=4) + assert set(review_nouns.keys()) == set(review_nouns_multi.keys()), ( + f"리뷰 멀티프로세싱 결과 불일치: single={len(review_nouns)}, multi={len(review_nouns_multi)}" + ) diff --git a/tests/unit/test_multiprocessing.py b/tests/unit/test_multiprocessing.py new file mode 100644 index 00000000..c1fd819f --- /dev/null +++ b/tests/unit/test_multiprocessing.py @@ -0,0 +1,63 @@ +"""Tests for multiprocessing support in LRNounExtractor.""" + +from soynlp.core import LRGraph, corpus_to_lrgraph +from soynlp.noun import LRNounExtractor + +# 테스트용 샘플 문장 (충분한 양) +SAMPLE_SENTS = [ + "아이오아이가 평가단에게 높은 점수를 받았습니다", + "아이오아이는 아이돌 그룹입니다", + "자연어처리는 어렵습니다", + "자연어처리를 공부합니다", + "명사추출은 중요한 작업입니다", +] * 100 # 충분한 반복 + + +def test_corpus_to_lrgraph_single(): + """단일 프로세스로 LRGraph를 구축한다.""" + lrgraph = corpus_to_lrgraph(SAMPLE_SENTS, n_workers=1) + assert isinstance(lrgraph, LRGraph) + assert len(lrgraph._lr) > 0 + + +def test_corpus_to_lrgraph_multi(): + """멀티프로세스로 구축한 LRGraph가 단일 프로세스 결과와 동일하다.""" + lrgraph_single = corpus_to_lrgraph(SAMPLE_SENTS, n_workers=1) + lrgraph_multi = corpus_to_lrgraph(SAMPLE_SENTS, n_workers=4) + assert lrgraph_single._lr == lrgraph_multi._lr + + +def test_noun_extractor_single(): + """단일 프로세스로 명사를 추출한다.""" + extractor = LRNounExtractor(verbose=False) + nouns = extractor.extract(SAMPLE_SENTS, n_workers=1) + assert len(nouns) > 0 + + +def test_noun_extractor_multi(): + """n_workers=4로 추출한 명사 결과가 단일 프로세스와 동일하다.""" + extractor_single = LRNounExtractor(verbose=False) + nouns_single = extractor_single.extract(SAMPLE_SENTS, n_workers=1) + + extractor_multi = LRNounExtractor(verbose=False) + nouns_multi = extractor_multi.extract(SAMPLE_SENTS, n_workers=4) + + assert set(nouns_single.keys()) == set(nouns_multi.keys()) + + +def test_noun_extractor_auto_workers(): + """n_workers=-1이면 CPU 코어 수를 자동으로 사용한다.""" + extractor = LRNounExtractor(verbose=False) + nouns = extractor.extract(SAMPLE_SENTS, n_workers=-1) + assert len(nouns) > 0 + + +def test_noun_extractor_multi_with_min_eojeol_frequency(): + """min_eojeol_frequency > 1이면 n_workers를 지정해도 단일 프로세스로 동작하며 결과는 동일하다.""" + extractor_single = LRNounExtractor(verbose=False) + nouns_single = extractor_single.extract(SAMPLE_SENTS, min_eojeol_frequency=2, n_workers=1) + + extractor_multi = LRNounExtractor(verbose=False) + nouns_multi = extractor_multi.extract(SAMPLE_SENTS, min_eojeol_frequency=2, n_workers=4) + + assert set(nouns_single.keys()) == set(nouns_multi.keys())