fix(tokenizer): tokenize offset 계산이 다중 공백에서 틀리는 버그 수정

soynlp/tokenizer/tokenizer.py

@@ -108,11 +108,9 @@ def tokenize(self, sentence, return_words=True):
                Token(hank`s, score=1, position=(27, 33), eojeol_id=2),
                Token(report, score=1, position=(34, 40), eojeol_id=3)]
         """
-        offset = 0
         tokens = []
-        for eojeol_id, token in enumerate(sentence.split()):
-            tokens += self._tokenize(token, offset, eojeol_id)
-            offset += len(token) + 1
+        for eojeol_id, m in enumerate(re.finditer(r"\S+", sentence)):
+            tokens += self._tokenize(m.group(), m.start(), eojeol_id)
         if return_words:
             tokens = [token.word for token in tokens]
         return tokens

tests/unit/test_tokenizers.py

@@ -4,6 +4,7 @@
 from soynlp.tokenizer.tokenizer_builder import EojeolPatternTrainer
 
 
+<<<<<<< HEAD
 def test_regex_tokenizer_empty_string():
     tokenizer = RegexTokenizer()
     assert tokenizer.tokenize("") == []
@@ -19,6 +20,14 @@ def test_regex_tokenizer_version_number():
     assert tokenizer.tokenize("42") == ["42"]
 
 
+def test_regex_tokenizer_multi_space_offset():
+    # 연속 공백이 있을 때 Token의 position이 정확해야 한다 (issue #280)
+    tokenizer = RegexTokenizer()
+    tokens = tokenizer.tokenize("hello  world", return_words=False)
+    assert tokens[0].word == "hello" and tokens[0].begin == 0 and tokens[0].end == 5
+    assert tokens[1].word == "world" and tokens[1].begin == 7 and tokens[1].end == 12
+
+
 def test_regex_tokenizer():
     sentence = "abc123가나다 alphabet!!3.14한글 hank`s report"
     expected_words = ["abc", "123", "가나다", "alphabet", "!!", "3.14", "한글", "hank`s", "report"]