-
Notifications
You must be signed in to change notification settings - Fork 2.8k
/
corenlp.py
748 lines (615 loc) · 23.7 KB
/
corenlp.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
# Natural Language Toolkit: Interface to the CoreNLP REST API.
#
# Copyright (C) 2001-2021 NLTK Project
# Author: Dmitrijs Milajevs <dimazest@gmail.com>
#
# URL: <https://www.nltk.org/>
# For license information, see LICENSE.TXT
import json
import re
import socket
import time
from nltk.internals import _java_options, config_java, find_jar_iter, java
from nltk.parse.api import ParserI
from nltk.parse.dependencygraph import DependencyGraph
from nltk.tag.api import TaggerI
from nltk.tokenize.api import TokenizerI
from nltk.tree import Tree
_stanford_url = "https://stanfordnlp.github.io/CoreNLP/"
class CoreNLPServerError(EnvironmentError):
"""Exceptions associated with the Core NLP server."""
def try_port(port=0):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(("", port))
p = sock.getsockname()[1]
sock.close()
return p
class CoreNLPServer:
_MODEL_JAR_PATTERN = r"stanford-corenlp-(\d+)\.(\d+)\.(\d+)-models\.jar"
_JAR = r"stanford-corenlp-(\d+)\.(\d+)\.(\d+)\.jar"
def __init__(
self,
path_to_jar=None,
path_to_models_jar=None,
verbose=False,
java_options=None,
corenlp_options=None,
port=None,
):
if corenlp_options is None:
corenlp_options = ["-preload", "tokenize,ssplit,pos,lemma,parse,depparse"]
jars = list(
find_jar_iter(
self._JAR,
path_to_jar,
env_vars=("CORENLP",),
searchpath=(),
url=_stanford_url,
verbose=verbose,
is_regex=True,
)
)
# find the most recent code and model jar
stanford_jar = max(jars, key=lambda model_name: re.match(self._JAR, model_name))
if port is None:
try:
port = try_port(9000)
except OSError:
port = try_port()
corenlp_options.append(str(port))
else:
try_port(port)
self.url = f"http://localhost:{port}"
model_jar = max(
find_jar_iter(
self._MODEL_JAR_PATTERN,
path_to_models_jar,
env_vars=("CORENLP_MODELS",),
searchpath=(),
url=_stanford_url,
verbose=verbose,
is_regex=True,
),
key=lambda model_name: re.match(self._MODEL_JAR_PATTERN, model_name),
)
self.verbose = verbose
self._classpath = stanford_jar, model_jar
self.corenlp_options = corenlp_options
self.java_options = java_options or ["-mx2g"]
def start(self, stdout="devnull", stderr="devnull"):
"""Starts the CoreNLP server
:param stdout, stderr: Specifies where CoreNLP output is redirected. Valid values are 'devnull', 'stdout', 'pipe'
"""
import requests
cmd = ["edu.stanford.nlp.pipeline.StanfordCoreNLPServer"]
if self.corenlp_options:
cmd.extend(self.corenlp_options)
# Configure java.
default_options = " ".join(_java_options)
config_java(options=self.java_options, verbose=self.verbose)
try:
self.popen = java(
cmd,
classpath=self._classpath,
blocking=False,
stdout=stdout,
stderr=stderr,
)
finally:
# Return java configurations to their default values.
config_java(options=default_options, verbose=self.verbose)
# Check that the server is istill running.
returncode = self.popen.poll()
if returncode is not None:
_, stderrdata = self.popen.communicate()
raise CoreNLPServerError(
returncode,
"Could not start the server. "
"The error was: {}".format(stderrdata.decode("ascii")),
)
for i in range(30):
try:
response = requests.get(requests.compat.urljoin(self.url, "live"))
except requests.exceptions.ConnectionError:
time.sleep(1)
else:
if response.ok:
break
else:
raise CoreNLPServerError("Could not connect to the server.")
for i in range(60):
try:
response = requests.get(requests.compat.urljoin(self.url, "ready"))
except requests.exceptions.ConnectionError:
time.sleep(1)
else:
if response.ok:
break
else:
raise CoreNLPServerError("The server is not ready.")
def stop(self):
self.popen.terminate()
self.popen.wait()
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.stop()
return False
class GenericCoreNLPParser(ParserI, TokenizerI, TaggerI):
"""Interface to the CoreNLP Parser."""
def __init__(self, url="http://localhost:9000", encoding="utf8", tagtype=None):
import requests
self.url = url
self.encoding = encoding
if tagtype not in ["pos", "ner", None]:
raise ValueError("tagtype must be either 'pos', 'ner' or None")
self.tagtype = tagtype
self.session = requests.Session()
def parse_sents(self, sentences, *args, **kwargs):
"""Parse multiple sentences.
Takes multiple sentences as a list where each sentence is a list of
words. Each sentence will be automatically tagged with this
CoreNLPParser instance's tagger.
If a whitespace exists inside a token, then the token will be treated as
several tokens.
:param sentences: Input sentences to parse
:type sentences: list(list(str))
:rtype: iter(iter(Tree))
"""
# Converting list(list(str)) -> list(str)
sentences = (" ".join(words) for words in sentences)
return self.raw_parse_sents(sentences, *args, **kwargs)
def raw_parse(self, sentence, properties=None, *args, **kwargs):
"""Parse a sentence.
Takes a sentence as a string; before parsing, it will be automatically
tokenized and tagged by the CoreNLP Parser.
:param sentence: Input sentence to parse
:type sentence: str
:rtype: iter(Tree)
"""
default_properties = {"tokenize.whitespace": "false"}
default_properties.update(properties or {})
return next(
self.raw_parse_sents(
[sentence], properties=default_properties, *args, **kwargs
)
)
def api_call(self, data, properties=None, timeout=60):
default_properties = {
"outputFormat": "json",
"annotators": "tokenize,pos,lemma,ssplit,{parser_annotator}".format(
parser_annotator=self.parser_annotator
),
}
default_properties.update(properties or {})
response = self.session.post(
self.url,
params={"properties": json.dumps(default_properties)},
data=data.encode(self.encoding),
headers={"Content-Type": f"text/plain; charset={self.encoding}"},
timeout=timeout,
)
response.raise_for_status()
return response.json()
def raw_parse_sents(
self, sentences, verbose=False, properties=None, *args, **kwargs
):
"""Parse multiple sentences.
Takes multiple sentences as a list of strings. Each sentence will be
automatically tokenized and tagged.
:param sentences: Input sentences to parse.
:type sentences: list(str)
:rtype: iter(iter(Tree))
"""
default_properties = {
# Only splits on '\n', never inside the sentence.
"ssplit.eolonly": "true"
}
default_properties.update(properties or {})
"""
for sentence in sentences:
parsed_data = self.api_call(sentence, properties=default_properties)
assert len(parsed_data['sentences']) == 1
for parse in parsed_data['sentences']:
tree = self.make_tree(parse)
yield iter([tree])
"""
parsed_data = self.api_call("\n".join(sentences), properties=default_properties)
for parsed_sent in parsed_data["sentences"]:
tree = self.make_tree(parsed_sent)
yield iter([tree])
def parse_text(self, text, *args, **kwargs):
"""Parse a piece of text.
The text might contain several sentences which will be split by CoreNLP.
:param str text: text to be split.
:returns: an iterable of syntactic structures. # TODO: should it be an iterable of iterables?
"""
parsed_data = self.api_call(text, *args, **kwargs)
for parse in parsed_data["sentences"]:
yield self.make_tree(parse)
def tokenize(self, text, properties=None):
"""Tokenize a string of text.
>>> parser = CoreNLPParser(url='http://localhost:9000')
>>> text = 'Good muffins cost $3.88\\nin New York. Please buy me\\ntwo of them.\\nThanks.'
>>> list(parser.tokenize(text))
['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York', '.', 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
>>> s = "The colour of the wall is blue."
>>> list(
... parser.tokenize(
... 'The colour of the wall is blue.',
... properties={'tokenize.options': 'americanize=true'},
... )
... )
['The', 'color', 'of', 'the', 'wall', 'is', 'blue', '.']
"""
default_properties = {"annotators": "tokenize,ssplit"}
default_properties.update(properties or {})
result = self.api_call(text, properties=default_properties)
for sentence in result["sentences"]:
for token in sentence["tokens"]:
yield token["originalText"] or token["word"]
def tag_sents(self, sentences, properties=None):
"""
Tag multiple sentences.
Takes multiple sentences as a list where each sentence is a list of
tokens.
:param sentences: Input sentences to tag
:type sentences: list(list(str))
:rtype: list(list(tuple(str, str))
"""
# Converting list(list(str)) -> list(str)
sentences = (" ".join(words) for words in sentences)
if properties is None:
properties = {"tokenize.whitespace": "true"}
return [sentences[0] for sentences in self.raw_tag_sents(sentences, properties)]
def tag(self, sentence, properties=None):
"""
Tag a list of tokens.
:rtype: list(tuple(str, str))
>>> parser = CoreNLPParser(url='http://localhost:9000', tagtype='ner')
>>> tokens = 'Rami Eid is studying at Stony Brook University in NY'.split()
>>> parser.tag(tokens)
[('Rami', 'PERSON'), ('Eid', 'PERSON'), ('is', 'O'), ('studying', 'O'), ('at', 'O'), ('Stony', 'ORGANIZATION'),
('Brook', 'ORGANIZATION'), ('University', 'ORGANIZATION'), ('in', 'O'), ('NY', 'O')]
>>> parser = CoreNLPParser(url='http://localhost:9000', tagtype='pos')
>>> tokens = "What is the airspeed of an unladen swallow ?".split()
>>> parser.tag(tokens)
[('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'),
('airspeed', 'NN'), ('of', 'IN'), ('an', 'DT'),
('unladen', 'JJ'), ('swallow', 'VB'), ('?', '.')]
"""
return self.tag_sents([sentence], properties)[0]
def raw_tag_sents(self, sentences, properties=None):
"""
Tag multiple sentences.
Takes multiple sentences as a list where each sentence is a string.
:param sentences: Input sentences to tag
:type sentences: list(str)
:rtype: list(list(list(tuple(str, str)))
"""
default_properties = {
"ssplit.isOneSentence": "true",
"annotators": "tokenize,ssplit,",
}
default_properties.update(properties or {})
# Supports only 'pos' or 'ner' tags.
assert self.tagtype in [
"pos",
"ner",
], "CoreNLP tagger supports only 'pos' or 'ner' tags."
default_properties["annotators"] += self.tagtype
for sentence in sentences:
tagged_data = self.api_call(sentence, properties=default_properties)
yield [
[
(token["word"], token[self.tagtype])
for token in tagged_sentence["tokens"]
]
for tagged_sentence in tagged_data["sentences"]
]
class CoreNLPParser(GenericCoreNLPParser):
"""
>>> parser = CoreNLPParser(url='http://localhost:9000')
>>> next(
... parser.raw_parse('The quick brown fox jumps over the lazy dog.')
... ).pretty_print() # doctest: +NORMALIZE_WHITESPACE
ROOT
|
S
_______________|__________________________
| VP |
| _________|___ |
| | PP |
| | ________|___ |
NP | | NP |
____|__________ | | _______|____ |
DT JJ JJ NN VBZ IN DT JJ NN .
| | | | | | | | | |
The quick brown fox jumps over the lazy dog .
>>> (parse_fox, ), (parse_wolf, ) = parser.raw_parse_sents(
... [
... 'The quick brown fox jumps over the lazy dog.',
... 'The quick grey wolf jumps over the lazy fox.',
... ]
... )
>>> parse_fox.pretty_print() # doctest: +NORMALIZE_WHITESPACE
ROOT
|
S
_______________|__________________________
| VP |
| _________|___ |
| | PP |
| | ________|___ |
NP | | NP |
____|__________ | | _______|____ |
DT JJ JJ NN VBZ IN DT JJ NN .
| | | | | | | | | |
The quick brown fox jumps over the lazy dog .
>>> parse_wolf.pretty_print() # doctest: +NORMALIZE_WHITESPACE
ROOT
|
S
_______________|__________________________
| VP |
| _________|___ |
| | PP |
| | ________|___ |
NP | | NP |
____|_________ | | _______|____ |
DT JJ JJ NN VBZ IN DT JJ NN .
| | | | | | | | | |
The quick grey wolf jumps over the lazy fox .
>>> (parse_dog, ), (parse_friends, ) = parser.parse_sents(
... [
... "I 'm a dog".split(),
... "This is my friends ' cat ( the tabby )".split(),
... ]
... )
>>> parse_dog.pretty_print() # doctest: +NORMALIZE_WHITESPACE
ROOT
|
S
_______|____
| VP
| ________|___
NP | NP
| | ___|___
PRP VBP DT NN
| | | |
I 'm a dog
>>> parse_friends.pretty_print() # doctest: +NORMALIZE_WHITESPACE
ROOT
|
S
____|___________
| VP
| ___________|_____________
| | NP
| | _______|_________
| | NP PRN
| | _____|_______ ____|______________
NP | NP | | NP |
| | ______|_________ | | ___|____ |
DT VBZ PRP$ NNS POS NN -LRB- DT NN -RRB-
| | | | | | | | | |
This is my friends ' cat -LRB- the tabby -RRB-
>>> parse_john, parse_mary, = parser.parse_text(
... 'John loves Mary. Mary walks.'
... )
>>> parse_john.pretty_print() # doctest: +NORMALIZE_WHITESPACE
ROOT
|
S
_____|_____________
| VP |
| ____|___ |
NP | NP |
| | | |
NNP VBZ NNP .
| | | |
John loves Mary .
>>> parse_mary.pretty_print() # doctest: +NORMALIZE_WHITESPACE
ROOT
|
S
_____|____
NP VP |
| | |
NNP VBZ .
| | |
Mary walks .
Special cases
>>> next(
... parser.raw_parse(
... 'NASIRIYA, Iraq—Iraqi doctors who treated former prisoner of war '
... 'Jessica Lynch have angrily dismissed claims made in her biography '
... 'that she was raped by her Iraqi captors.'
... )
... ).height()
20
>>> next(
... parser.raw_parse(
... "The broader Standard & Poor's 500 Index <.SPX> was 0.46 points lower, or "
... '0.05 percent, at 997.02.'
... )
... ).height()
9
"""
_OUTPUT_FORMAT = "penn"
parser_annotator = "parse"
def make_tree(self, result):
return Tree.fromstring(result["parse"])
class CoreNLPDependencyParser(GenericCoreNLPParser):
"""Dependency parser.
>>> dep_parser = CoreNLPDependencyParser(url='http://localhost:9000')
>>> parse, = dep_parser.raw_parse(
... 'The quick brown fox jumps over the lazy dog.'
... )
>>> print(parse.to_conll(4)) # doctest: +NORMALIZE_WHITESPACE
The DT 4 det
quick JJ 4 amod
brown JJ 4 amod
fox NN 5 nsubj
jumps VBZ 0 ROOT
over IN 9 case
the DT 9 det
lazy JJ 9 amod
dog NN 5 nmod
. . 5 punct
>>> print(parse.tree()) # doctest: +NORMALIZE_WHITESPACE
(jumps (fox The quick brown) (dog over the lazy) .)
>>> for governor, dep, dependent in parse.triples():
... print(governor, dep, dependent) # doctest: +NORMALIZE_WHITESPACE
('jumps', 'VBZ') nsubj ('fox', 'NN')
('fox', 'NN') det ('The', 'DT')
('fox', 'NN') amod ('quick', 'JJ')
('fox', 'NN') amod ('brown', 'JJ')
('jumps', 'VBZ') nmod ('dog', 'NN')
('dog', 'NN') case ('over', 'IN')
('dog', 'NN') det ('the', 'DT')
('dog', 'NN') amod ('lazy', 'JJ')
('jumps', 'VBZ') punct ('.', '.')
>>> (parse_fox, ), (parse_dog, ) = dep_parser.raw_parse_sents(
... [
... 'The quick brown fox jumps over the lazy dog.',
... 'The quick grey wolf jumps over the lazy fox.',
... ]
... )
>>> print(parse_fox.to_conll(4)) # doctest: +NORMALIZE_WHITESPACE
The DT 4 det
quick JJ 4 amod
brown JJ 4 amod
fox NN 5 nsubj
jumps VBZ 0 ROOT
over IN 9 case
the DT 9 det
lazy JJ 9 amod
dog NN 5 nmod
. . 5 punct
>>> print(parse_dog.to_conll(4)) # doctest: +NORMALIZE_WHITESPACE
The DT 4 det
quick JJ 4 amod
grey JJ 4 amod
wolf NN 5 nsubj
jumps VBZ 0 ROOT
over IN 9 case
the DT 9 det
lazy JJ 9 amod
fox NN 5 nmod
. . 5 punct
>>> (parse_dog, ), (parse_friends, ) = dep_parser.parse_sents(
... [
... "I 'm a dog".split(),
... "This is my friends ' cat ( the tabby )".split(),
... ]
... )
>>> print(parse_dog.to_conll(4)) # doctest: +NORMALIZE_WHITESPACE
I PRP 4 nsubj
'm VBP 4 cop
a DT 4 det
dog NN 0 ROOT
>>> print(parse_friends.to_conll(4)) # doctest: +NORMALIZE_WHITESPACE
This DT 6 nsubj
is VBZ 6 cop
my PRP$ 4 nmod:poss
friends NNS 6 nmod:poss
' POS 4 case
cat NN 0 ROOT
-LRB- -LRB- 9 punct
the DT 9 det
tabby NN 6 appos
-RRB- -RRB- 9 punct
>>> parse_john, parse_mary, = dep_parser.parse_text(
... 'John loves Mary. Mary walks.'
... )
>>> print(parse_john.to_conll(4)) # doctest: +NORMALIZE_WHITESPACE
John NNP 2 nsubj
loves VBZ 0 ROOT
Mary NNP 2 dobj
. . 2 punct
>>> print(parse_mary.to_conll(4)) # doctest: +NORMALIZE_WHITESPACE
Mary NNP 2 nsubj
walks VBZ 0 ROOT
. . 2 punct
Special cases
Non-breaking space inside of a token.
>>> len(
... next(
... dep_parser.raw_parse(
... 'Anhalt said children typically treat a 20-ounce soda bottle as one '
... 'serving, while it actually contains 2 1/2 servings.'
... )
... ).nodes
... )
21
Phone numbers.
>>> len(
... next(
... dep_parser.raw_parse('This is not going to crash: 01 111 555.')
... ).nodes
... )
10
>>> print(
... next(
... dep_parser.raw_parse('The underscore _ should not simply disappear.')
... ).to_conll(4)
... ) # doctest: +NORMALIZE_WHITESPACE
The DT 3 det
underscore VBP 3 amod
_ NN 7 nsubj
should MD 7 aux
not RB 7 neg
simply RB 7 advmod
disappear VB 0 ROOT
. . 7 punct
>>> print(
... '\\n'.join(
... next(
... dep_parser.raw_parse(
... 'for all of its insights into the dream world of teen life , and its electronic expression through '
... 'cyber culture , the film gives no quarter to anyone seeking to pull a cohesive story out of its 2 '
... '1/2-hour running time .'
... )
... ).to_conll(4).split('\\n')[-8:]
... )
... )
its PRP$ 40 nmod:poss
2 1/2 CD 40 nummod
- : 40 punct
hour NN 31 nmod
running VBG 42 amod
time NN 40 dep
. . 24 punct
<BLANKLINE>
"""
_OUTPUT_FORMAT = "conll2007"
parser_annotator = "depparse"
def make_tree(self, result):
return DependencyGraph(
(
" ".join(n_items[1:]) # NLTK expects an iterable of strings...
for n_items in sorted(transform(result))
),
cell_separator=" ", # To make sure that a non-breaking space is kept inside of a token.
)
def transform(sentence):
for dependency in sentence["basicDependencies"]:
dependent_index = dependency["dependent"]
token = sentence["tokens"][dependent_index - 1]
# Return values that we don't know as '_'. Also, consider tag and ctag
# to be equal.
yield (
dependent_index,
"_",
token["word"],
token["lemma"],
token["pos"],
token["pos"],
"_",
str(dependency["governor"]),
dependency["dep"],
"_",
"_",
)