forked from nltk/nltk
/
treebank.py
383 lines (318 loc) · 15.1 KB
/
treebank.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
# Natural Language Toolkit: Tokenizers
#
# Copyright (C) 2001-2021 NLTK Project
# Author: Edward Loper <edloper@gmail.com>
# Michael Heilman <mheilman@cmu.edu> (re-port from http://www.cis.upenn.edu/~treebank/tokenizer.sed)
# Tom Aarsen <> (modifications)
#
# URL: <https://www.nltk.org>
# For license information, see LICENSE.TXT
r"""
Penn Treebank Tokenizer
The Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank.
This implementation is a port of the tokenizer sed script written by Robert McIntyre
and available at http://www.cis.upenn.edu/~treebank/tokenizer.sed.
"""
import re
import warnings
from nltk.tokenize.api import TokenizerI
from nltk.tokenize.destructive import MacIntyreContractions
from nltk.tokenize.util import align_tokens
class TreebankWordTokenizer(TokenizerI):
r"""
The Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank.
This is the method that is invoked by ``word_tokenize()``. It assumes that the
text has already been segmented into sentences, e.g. using ``sent_tokenize()``.
This tokenizer performs the following steps:
- split standard contractions, e.g. ``don't`` -> ``do n't`` and ``they'll`` -> ``they 'll``
- treat most punctuation characters as separate tokens
- split off commas and single quotes, when followed by whitespace
- separate periods that appear at the end of line
>>> from nltk.tokenize import TreebankWordTokenizer
>>> s = '''Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\nThanks.'''
>>> TreebankWordTokenizer().tokenize(s)
['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two', 'of', 'them.', 'Thanks', '.']
>>> s = "They'll save and invest more."
>>> TreebankWordTokenizer().tokenize(s)
['They', "'ll", 'save', 'and', 'invest', 'more', '.']
>>> s = "hi, my name can't hello,"
>>> TreebankWordTokenizer().tokenize(s)
['hi', ',', 'my', 'name', 'ca', "n't", 'hello', ',']
"""
# starting quotes
STARTING_QUOTES = [
(re.compile(r"^\""), r"``"),
(re.compile(r"(``)"), r" \1 "),
(re.compile(r"([ \(\[{<])(\"|\'{2})"), r"\1 `` "),
]
# punctuation
PUNCTUATION = [
(re.compile(r"([:,])([^\d])"), r" \1 \2"),
(re.compile(r"([:,])$"), r" \1 "),
(re.compile(r"\.\.\."), r" ... "),
(re.compile(r"[;@#$%&]"), r" \g<0> "),
(
re.compile(r'([^\.])(\.)([\]\)}>"\']*)\s*$'),
r"\1 \2\3 ",
), # Handles the final period.
(re.compile(r"[?!]"), r" \g<0> "),
(re.compile(r"([^'])' "), r"\1 ' "),
]
# Pads parentheses
PARENS_BRACKETS = (re.compile(r"[\]\[\(\)\{\}\<\>]"), r" \g<0> ")
# Optionally: Convert parentheses, brackets and converts them to PTB symbols.
CONVERT_PARENTHESES = [
(re.compile(r"\("), "-LRB-"),
(re.compile(r"\)"), "-RRB-"),
(re.compile(r"\["), "-LSB-"),
(re.compile(r"\]"), "-RSB-"),
(re.compile(r"\{"), "-LCB-"),
(re.compile(r"\}"), "-RCB-"),
]
DOUBLE_DASHES = (re.compile(r"--"), r" -- ")
# ending quotes
ENDING_QUOTES = [
(re.compile(r"''"), " '' "),
(re.compile(r'"'), " '' "),
(re.compile(r"([^' ])('[sS]|'[mM]|'[dD]|') "), r"\1 \2 "),
(re.compile(r"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) "), r"\1 \2 "),
]
# List of contractions adapted from Robert MacIntyre's tokenizer.
_contractions = MacIntyreContractions()
CONTRACTIONS2 = list(map(re.compile, _contractions.CONTRACTIONS2))
CONTRACTIONS3 = list(map(re.compile, _contractions.CONTRACTIONS3))
def tokenize(self, text, convert_parentheses=False, return_str=False):
if return_str is not False:
warnings.warn(
"Parameter 'return_str' has been deprecated and should no "
"longer be used.",
category=DeprecationWarning,
stacklevel=2,
)
for regexp, substitution in self.STARTING_QUOTES:
text = regexp.sub(substitution, text)
for regexp, substitution in self.PUNCTUATION:
text = regexp.sub(substitution, text)
# Handles parentheses.
regexp, substitution = self.PARENS_BRACKETS
text = regexp.sub(substitution, text)
# Optionally convert parentheses
if convert_parentheses:
for regexp, substitution in self.CONVERT_PARENTHESES:
text = regexp.sub(substitution, text)
# Handles double dash.
regexp, substitution = self.DOUBLE_DASHES
text = regexp.sub(substitution, text)
# add extra space to make things easier
text = " " + text + " "
for regexp, substitution in self.ENDING_QUOTES:
text = regexp.sub(substitution, text)
for regexp in self.CONTRACTIONS2:
text = regexp.sub(r" \1 \2 ", text)
for regexp in self.CONTRACTIONS3:
text = regexp.sub(r" \1 \2 ", text)
# We are not using CONTRACTIONS4 since
# they are also commented out in the SED scripts
# for regexp in self._contractions.CONTRACTIONS4:
# text = regexp.sub(r' \1 \2 \3 ', text)
return text.split()
def span_tokenize(self, text):
r"""
Uses the post-hoc nltk.tokens.align_tokens to return the offset spans.
>>> from nltk.tokenize import TreebankWordTokenizer
>>> s = '''Good muffins cost $3.88\nin New (York). Please (buy) me\ntwo of them.\n(Thanks).'''
>>> expected = [(0, 4), (5, 12), (13, 17), (18, 19), (19, 23),
... (24, 26), (27, 30), (31, 32), (32, 36), (36, 37), (37, 38),
... (40, 46), (47, 48), (48, 51), (51, 52), (53, 55), (56, 59),
... (60, 62), (63, 68), (69, 70), (70, 76), (76, 77), (77, 78)]
>>> list(TreebankWordTokenizer().span_tokenize(s)) == expected
True
>>> expected = ['Good', 'muffins', 'cost', '$', '3.88', 'in',
... 'New', '(', 'York', ')', '.', 'Please', '(', 'buy', ')',
... 'me', 'two', 'of', 'them.', '(', 'Thanks', ')', '.']
>>> [s[start:end] for start, end in TreebankWordTokenizer().span_tokenize(s)] == expected
True
Additional example
>>> from nltk.tokenize import TreebankWordTokenizer
>>> s = '''I said, "I'd like to buy some ''good muffins" which cost $3.88\n each in New (York)."'''
>>> expected = [(0, 1), (2, 6), (6, 7), (8, 9), (9, 10), (10, 12),
... (13, 17), (18, 20), (21, 24), (25, 29), (30, 32), (32, 36),
... (37, 44), (44, 45), (46, 51), (52, 56), (57, 58), (58, 62),
... (64, 68), (69, 71), (72, 75), (76, 77), (77, 81), (81, 82),
... (82, 83), (83, 84)]
>>> list(TreebankWordTokenizer().span_tokenize(s)) == expected
True
>>> expected = ['I', 'said', ',', '"', 'I', "'d", 'like', 'to',
... 'buy', 'some', "''", "good", 'muffins', '"', 'which', 'cost',
... '$', '3.88', 'each', 'in', 'New', '(', 'York', ')', '.', '"']
>>> [s[start:end] for start, end in TreebankWordTokenizer().span_tokenize(s)] == expected
True
"""
raw_tokens = self.tokenize(text)
# Convert converted quotes back to original double quotes
# Do this only if original text contains double quote(s) or double
# single-quotes (because '' might be transformed to `` if it is
# treated as starting quotes).
if ('"' in text) or ("''" in text):
# Find double quotes and converted quotes
matched = [m.group() for m in re.finditer(r"``|'{2}|\"", text)]
# Replace converted quotes back to double quotes
tokens = [
matched.pop(0) if tok in ['"', "``", "''"] else tok
for tok in raw_tokens
]
else:
tokens = raw_tokens
yield from align_tokens(tokens, text)
class TreebankWordDetokenizer(TokenizerI):
r"""
The Treebank detokenizer uses the reverse regex operations corresponding to
the Treebank tokenizer's regexes.
Note:
- There're additional assumption mades when undoing the padding of `[;@#$%&]`
punctuation symbols that isn't presupposed in the TreebankTokenizer.
- There're additional regexes added in reversing the parentheses tokenization,
- the `r'([\]\)\}\>])\s([:;,.])'` removes the additional right padding added
to the closing parentheses precedding `[:;,.]`.
- It's not possible to return the original whitespaces as they were because
there wasn't explicit records of where '\n', '\t' or '\s' were removed at
the text.split() operation.
>>> from nltk.tokenize.treebank import TreebankWordTokenizer, TreebankWordDetokenizer
>>> s = '''Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\nThanks.'''
>>> d = TreebankWordDetokenizer()
>>> t = TreebankWordTokenizer()
>>> toks = t.tokenize(s)
>>> d.detokenize(toks)
'Good muffins cost $3.88 in New York. Please buy me two of them. Thanks.'
The MXPOST parentheses substitution can be undone using the `convert_parentheses`
parameter:
>>> s = '''Good muffins cost $3.88\nin New (York). Please (buy) me\ntwo of them.\n(Thanks).'''
>>> expected_tokens = ['Good', 'muffins', 'cost', '$', '3.88', 'in',
... 'New', '-LRB-', 'York', '-RRB-', '.', 'Please', '-LRB-', 'buy',
... '-RRB-', 'me', 'two', 'of', 'them.', '-LRB-', 'Thanks', '-RRB-', '.']
>>> expected_tokens == t.tokenize(s, convert_parentheses=True)
True
>>> expected_detoken = 'Good muffins cost $3.88 in New (York). Please (buy) me two of them. (Thanks).'
>>> expected_detoken == d.detokenize(t.tokenize(s, convert_parentheses=True), convert_parentheses=True)
True
During tokenization it's safe to add more spaces but during detokenization,
simply undoing the padding doesn't really help.
- During tokenization, left and right pad is added to [!?], when
detokenizing, only left shift the [!?] is needed.
Thus (re.compile(r'\s([?!])'), r'\g<1>')
- During tokenization [:,] are left and right padded but when detokenizing,
only left shift is necessary and we keep right pad after comma/colon
if the string after is a non-digit.
Thus (re.compile(r'\s([:,])\s([^\d])'), r'\1 \2')
>>> from nltk.tokenize.treebank import TreebankWordDetokenizer
>>> toks = ['hello', ',', 'i', 'ca', "n't", 'feel', 'my', 'feet', '!', 'Help', '!', '!']
>>> twd = TreebankWordDetokenizer()
>>> twd.detokenize(toks)
"hello, i can't feel my feet! Help!!"
>>> toks = ['hello', ',', 'i', "can't", 'feel', ';', 'my', 'feet', '!',
... 'Help', '!', '!', 'He', 'said', ':', 'Help', ',', 'help', '?', '!']
>>> twd.detokenize(toks)
"hello, i can't feel; my feet! Help!! He said: Help, help?!"
"""
_contractions = MacIntyreContractions()
CONTRACTIONS2 = [
re.compile(pattern.replace("(?#X)", r"\s"))
for pattern in _contractions.CONTRACTIONS2
]
CONTRACTIONS3 = [
re.compile(pattern.replace("(?#X)", r"\s"))
for pattern in _contractions.CONTRACTIONS3
]
# ending quotes
ENDING_QUOTES = [
(re.compile(r"([^' ])\s('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) "), r"\1\2 "),
(re.compile(r"([^' ])\s('[sS]|'[mM]|'[dD]|') "), r"\1\2 "),
(re.compile(r"(\S)\s(\'\')"), r"\1\2"),
(
re.compile(r"(\'\')\s([.,:)\]>};%])"),
r"\1\2",
), # Quotes followed by no-left-padded punctuations.
(re.compile(r"''"), '"'),
]
# Handles double dashes
DOUBLE_DASHES = (re.compile(r" -- "), r"--")
# Optionally: Convert parentheses, brackets and converts them from PTB symbols.
CONVERT_PARENTHESES = [
(re.compile("-LRB-"), "("),
(re.compile("-RRB-"), ")"),
(re.compile("-LSB-"), "["),
(re.compile("-RSB-"), "]"),
(re.compile("-LCB-"), "{"),
(re.compile("-RCB-"), "}"),
]
# Undo padding on parentheses.
PARENS_BRACKETS = [
(re.compile(r"([\[\(\{\<])\s"), r"\g<1>"),
(re.compile(r"\s([\]\)\}\>])"), r"\g<1>"),
(re.compile(r"([\]\)\}\>])\s([:;,.])"), r"\1\2"),
]
# punctuation
PUNCTUATION = [
(re.compile(r"([^'])\s'\s"), r"\1' "),
(re.compile(r"\s([?!])"), r"\g<1>"), # Strip left pad for [?!]
# (re.compile(r'\s([?!])\s'), r'\g<1>'),
(re.compile(r'([^\.])\s(\.)([\]\)}>"\']*)\s*$'), r"\1\2\3"),
# When tokenizing, [;@#$%&] are padded with whitespace regardless of
# whether there are spaces before or after them.
# But during detokenization, we need to distinguish between left/right
# pad, so we split this up.
(re.compile(r"([#$])\s"), r"\g<1>"), # Left pad.
(re.compile(r"\s([;%])"), r"\g<1>"), # Right pad.
# (re.compile(r"\s([&*])\s"), r" \g<1> "), # Unknown pad.
(re.compile(r"\s\.\.\.\s"), r"..."),
# (re.compile(r"\s([:,])\s$"), r"\1"), # .strip() takes care of it.
(
re.compile(r"\s([:,])"),
r"\1",
), # Just remove left padding. Punctuation in numbers won't be padded.
]
# starting quotes
STARTING_QUOTES = [
(re.compile(r"([ (\[{<])\s``"), r"\1``"),
(re.compile(r"(``)\s"), r"\1"),
(re.compile(r"``"), r'"'),
]
def tokenize(self, tokens, convert_parentheses=False):
"""
Treebank detokenizer, created by undoing the regexes from
the TreebankWordTokenizer.tokenize.
:param tokens: A list of strings, i.e. tokenized text.
:type tokens: list(str)
:return: str
"""
text = " ".join(tokens)
# Reverse the contractions regexes.
# Note: CONTRACTIONS4 are not used in tokenization.
for regexp in self.CONTRACTIONS3:
text = regexp.sub(r"\1\2", text)
for regexp in self.CONTRACTIONS2:
text = regexp.sub(r"\1\2", text)
# Reverse the regexes applied for ending quotes.
for regexp, substitution in self.ENDING_QUOTES:
text = regexp.sub(substitution, text)
# Undo the space padding.
text = text.strip()
# Reverse the padding on double dashes.
regexp, substitution = self.DOUBLE_DASHES
text = regexp.sub(substitution, text)
if convert_parentheses:
for regexp, substitution in self.CONVERT_PARENTHESES:
text = regexp.sub(substitution, text)
# Reverse the padding regexes applied for parenthesis/brackets.
for regexp, substitution in self.PARENS_BRACKETS:
text = regexp.sub(substitution, text)
# Reverse the regexes applied for punctuations.
for regexp, substitution in self.PUNCTUATION:
text = regexp.sub(substitution, text)
# Reverse the regexes applied for starting quotes.
for regexp, substitution in self.STARTING_QUOTES:
text = regexp.sub(substitution, text)
return text.strip()
def detokenize(self, tokens, convert_parentheses=False):
"""Duck-typing the abstract *tokenize()*."""
return self.tokenize(tokens, convert_parentheses)