Merge pull request #39 from arushadev/paragraph

Fix paragraph tokenizer

piraye/tokenizer.py

@@ -72,7 +72,7 @@ def paragraph_span_tokenize(self, text: str) -> List[Tuple[int, int, str]]:
         for _, sentence_end, _ in sentences:
             if last_index + 1 >= text2_len:
                 break
-            pointer = sentence_end + 1
+            pointer = sentence_end
             while True:
                 if pointer + 1 >= text2_len:
                     paragraphs.append((last_index, pointer, text[last_index:pointer]))

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "piraye"
-version = "0.6.0"
+version = "0.6.1"
 authors = [
     { name = "Hamed Khademi Khaledi", email = "khaledihkh@gmail.com" },
     { name = "HosseiN Khademi khaeldi", email = "hossein@arusha.dev" },

tests/test_tokenizer.py

@@ -34,10 +34,19 @@ def test_sentence_tokenizer():
 
 
 def test_paragraph_tokenizer():
-    text = "par1 sen1 sad.  par1 \n sen2. par1 \n\n sen3.  \n par2 sen1.\n\n\n\n   par3 sen1.\n\n"
+    text = "par1 sen1 sad.  par1 \n sen2. par1 \n\n sen3.\n par2 sen1.\n\n\n\n   par3 sen1. \n par4 sen1.\n\n"
     tokenizer = NltkTokenizer()
-    assert len(tokenizer.paragraph_tokenize(text)) == 3
-    assert len(tokenizer.paragraph_span_tokenize(text)) == 3
+    assert len(tokenizer.paragraph_tokenize(text)) == 4
+    assert len(tokenizer.paragraph_span_tokenize(text)) == 4
+    assert len(tokenizer.paragraph_tokenize("par1 sen1 sad.")) == 1
+    assert len(tokenizer.paragraph_tokenize("par1 sen1 sad. par1 \n sen2. ")) == 1
+
+
+def test_paragraph_tokenizer_spacy():
+    text = "par1 sen1 sad.  par1 \n sen2. par1 \n\n sen3.\n par2 sen1.\n\n\n\n   par3 sen1. \n par4 sen1.\n\n"
+    tokenizer = SpacyTokenizer()
+    assert len(tokenizer.paragraph_tokenize(text)) == 4
+    assert len(tokenizer.paragraph_span_tokenize(text)) == 4
     assert len(tokenizer.paragraph_tokenize("par1 sen1 sad.")) == 1
     assert len(tokenizer.paragraph_tokenize("par1 sen1 sad. par1 \n sen2. ")) == 1