Use global regex for TweetTokenizer'ss remove_handles (#2795)

nltk/tokenize/casual.py

@@ -177,6 +177,11 @@
 # These are for regularizing HTML entities to Unicode:
 ENT_RE = regex.compile(r"&(#?(x?))([^&;\s]+);")
 
+# For stripping away handles from a tweet:
+HANDLES_RE = regex.compile(
+    r"(?<![A-Za-z0-9_!@#\$%&*])@(([A-Za-z0-9_]){20}(?!@))|"
+    r"(?<![A-Za-z0-9_!@#\$%&*])@(([A-Za-z0-9_]){1,19})(?![A-Za-z0-9_]*@)"
+)
 
 ######################################################################
 # Functions for converting html entities
@@ -322,13 +327,8 @@ def remove_handles(text):
     """
     Remove Twitter username handles from text.
     """
-    pattern = regex.compile(
-        r"(?<![A-Za-z0-9_!@#\$%&*])@(([A-Za-z0-9_]){20}(?!@))|"
-        r"(?<![A-Za-z0-9_!@#\$%&*])@(([A-Za-z0-9_]){1,19})(?![A-Za-z0-9_]*@)"
-    )
-    # Substitute handles with ' ' to ensure that text on either
-    # side of removed handles are tokenized correctly
-    return pattern.sub(" ", text)
+    # Substitute handles with ' ' to ensure that text on either side of removed handles are tokenized correctly
+    return HANDLES_RE.sub(" ", text)
 
 
 ######################################################################