Update nltk version (#25)

nltk/nltk#2877
arushadev · Dec 15, 2022 · 64efbac · 64efbac
1 parent 4e9e97e
commit 64efbac
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 2 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,5 @@
 tqdm>=4.62.3
 spacy>=3.1.3
 hazm>=0.7.0
-nltk>=3.2.0
+nltk>=3.8.0
 setuptools>=58.1.0
diff --git a/setup.py b/setup.py
@@ -19,7 +19,7 @@
         "Normalization",
     ],
     url="https://github.com/arushadev/piraye",
-    version="0.1.4",
+    version="0.1.5",
     package_dir={"piraye": "src"},
     packages=["piraye"],
     package_data={"piraye": ["data/*/*.json"]},

diff --git a/tests/test_normalizer.py b/tests/test_normalizer.py
@@ -1,6 +1,7 @@
 # testing Fibonacci number function
 # pylint: skip-file
 
+from ..src import NltkTokenizer
 from ..src import NormalizerBuilder
 
 
@@ -46,3 +47,8 @@ def test_quotes():
     norm = NormalizerBuilder().digit_en().punctuation_en().alphabet_fa() \
         .tokenizing().remove_extra_spaces().build()
     norm.normalize(text)
+
+
+def test_normalizer():
+    tokens = NltkTokenizer().word_tokenize('\'\'Y\'"')
+    print(tokens)