-
Notifications
You must be signed in to change notification settings - Fork 2.8k
/
wordnet.py
2525 lines (2113 loc) · 90.5 KB
/
wordnet.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Natural Language Toolkit: WordNet
#
# Copyright (C) 2001-2023 NLTK Project
# Author: Steven Bethard <Steven.Bethard@colorado.edu>
# Steven Bird <stevenbird1@gmail.com>
# Edward Loper <edloper@gmail.com>
# Nitin Madnani <nmadnani@ets.org>
# Nasruddin A’aidil Shari
# Sim Wei Ying Geraldine
# Soe Lynn
# Francis Bond <bond@ieee.org>
# Eric Kafe <kafe.eric@gmail.com>
# URL: <https://www.nltk.org/>
# For license information, see LICENSE.TXT
"""
An NLTK interface for WordNet
WordNet is a lexical database of English.
Using synsets, helps find conceptual relationships between words
such as hypernyms, hyponyms, synonyms, antonyms etc.
For details about WordNet see:
https://wordnet.princeton.edu/
This module also allows you to find lemmas in languages
other than English from the Open Multilingual Wordnet
https://omwn.org/
"""
import math
import os
import re
import warnings
from collections import defaultdict, deque
from functools import total_ordering
from itertools import chain, islice
from operator import itemgetter
from nltk.corpus.reader import CorpusReader
from nltk.internals import deprecated
from nltk.probability import FreqDist
from nltk.util import binary_search_file as _binary_search_file
######################################################################
# Table of Contents
######################################################################
# - Constants
# - Data Classes
# - WordNetError
# - Lemma
# - Synset
# - WordNet Corpus Reader
# - WordNet Information Content Corpus Reader
# - Similarity Metrics
# - Demo
######################################################################
# Constants
######################################################################
#: Positive infinity (for similarity functions)
_INF = 1e300
# { Part-of-speech constants
ADJ, ADJ_SAT, ADV, NOUN, VERB = "a", "s", "r", "n", "v"
# }
POS_LIST = [NOUN, VERB, ADJ, ADV]
# A table of strings that are used to express verb frames.
VERB_FRAME_STRINGS = (
None,
"Something %s",
"Somebody %s",
"It is %sing",
"Something is %sing PP",
"Something %s something Adjective/Noun",
"Something %s Adjective/Noun",
"Somebody %s Adjective",
"Somebody %s something",
"Somebody %s somebody",
"Something %s somebody",
"Something %s something",
"Something %s to somebody",
"Somebody %s on something",
"Somebody %s somebody something",
"Somebody %s something to somebody",
"Somebody %s something from somebody",
"Somebody %s somebody with something",
"Somebody %s somebody of something",
"Somebody %s something on somebody",
"Somebody %s somebody PP",
"Somebody %s something PP",
"Somebody %s PP",
"Somebody's (body part) %s",
"Somebody %s somebody to INFINITIVE",
"Somebody %s somebody INFINITIVE",
"Somebody %s that CLAUSE",
"Somebody %s to somebody",
"Somebody %s to INFINITIVE",
"Somebody %s whether INFINITIVE",
"Somebody %s somebody into V-ing something",
"Somebody %s something with something",
"Somebody %s INFINITIVE",
"Somebody %s VERB-ing",
"It %s that CLAUSE",
"Something %s INFINITIVE",
# OEWN additions:
"Somebody %s at something",
"Somebody %s for something",
"Somebody %s on somebody",
"Somebody %s out of somebody",
)
SENSENUM_RE = re.compile(r"\.[\d]+\.")
######################################################################
# Data Classes
######################################################################
class WordNetError(Exception):
"""An exception class for wordnet-related errors."""
@total_ordering
class _WordNetObject:
"""A common base class for lemmas and synsets."""
def hypernyms(self):
return self._related("@")
def _hypernyms(self):
return self._related("@")
def instance_hypernyms(self):
return self._related("@i")
def _instance_hypernyms(self):
return self._related("@i")
def hyponyms(self):
return self._related("~")
def instance_hyponyms(self):
return self._related("~i")
def member_holonyms(self):
return self._related("#m")
def substance_holonyms(self):
return self._related("#s")
def part_holonyms(self):
return self._related("#p")
def member_meronyms(self):
return self._related("%m")
def substance_meronyms(self):
return self._related("%s")
def part_meronyms(self):
return self._related("%p")
def topic_domains(self):
return self._related(";c")
def in_topic_domains(self):
return self._related("-c")
def region_domains(self):
return self._related(";r")
def in_region_domains(self):
return self._related("-r")
def usage_domains(self):
return self._related(";u")
def in_usage_domains(self):
return self._related("-u")
def attributes(self):
return self._related("=")
def entailments(self):
return self._related("*")
def causes(self):
return self._related(">")
def also_sees(self):
return self._related("^")
def verb_groups(self):
return self._related("$")
def similar_tos(self):
return self._related("&")
def __hash__(self):
return hash(self._name)
def __eq__(self, other):
return self._name == other._name
def __ne__(self, other):
return self._name != other._name
def __lt__(self, other):
return self._name < other._name
class Lemma(_WordNetObject):
"""
The lexical entry for a single morphological form of a
sense-disambiguated word.
Create a Lemma from a "<word>.<pos>.<number>.<lemma>" string where:
<word> is the morphological stem identifying the synset
<pos> is one of the module attributes ADJ, ADJ_SAT, ADV, NOUN or VERB
<number> is the sense number, counting from 0.
<lemma> is the morphological form of interest
Note that <word> and <lemma> can be different, e.g. the Synset
'salt.n.03' has the Lemmas 'salt.n.03.salt', 'salt.n.03.saltiness' and
'salt.n.03.salinity'.
Lemma attributes, accessible via methods with the same name:
- name: The canonical name of this lemma.
- synset: The synset that this lemma belongs to.
- syntactic_marker: For adjectives, the WordNet string identifying the
syntactic position relative modified noun. See:
https://wordnet.princeton.edu/documentation/wninput5wn
For all other parts of speech, this attribute is None.
- count: The frequency of this lemma in wordnet.
Lemma methods:
Lemmas have the following methods for retrieving related Lemmas. They
correspond to the names for the pointer symbols defined here:
https://wordnet.princeton.edu/documentation/wninput5wn
These methods all return lists of Lemmas:
- antonyms
- hypernyms, instance_hypernyms
- hyponyms, instance_hyponyms
- member_holonyms, substance_holonyms, part_holonyms
- member_meronyms, substance_meronyms, part_meronyms
- topic_domains, region_domains, usage_domains
- attributes
- derivationally_related_forms
- entailments
- causes
- also_sees
- verb_groups
- similar_tos
- pertainyms
"""
__slots__ = [
"_wordnet_corpus_reader",
"_name",
"_syntactic_marker",
"_synset",
"_frame_strings",
"_frame_ids",
"_lexname_index",
"_lex_id",
"_lang",
"_key",
]
def __init__(
self,
wordnet_corpus_reader,
synset,
name,
lexname_index,
lex_id,
syntactic_marker,
):
self._wordnet_corpus_reader = wordnet_corpus_reader
self._name = name
self._syntactic_marker = syntactic_marker
self._synset = synset
self._frame_strings = []
self._frame_ids = []
self._lexname_index = lexname_index
self._lex_id = lex_id
self._lang = "eng"
self._key = None # gets set later.
def name(self):
return self._name
def syntactic_marker(self):
return self._syntactic_marker
def synset(self):
return self._synset
def frame_strings(self):
return self._frame_strings
def frame_ids(self):
return self._frame_ids
def lang(self):
return self._lang
def key(self):
return self._key
def __repr__(self):
tup = type(self).__name__, self._synset._name, self._name
return "%s('%s.%s')" % tup
def _related(self, relation_symbol):
"""Returns the lemma's relation targets for the given relation_symbol.
Includes both the lemma ("lexical") relations and the synset ("semantic") relations.
The in_topic_domain() relation (-c) is hybrid, because a few lemmas have
both lemma and synset targets. For ex:
>>> from nltk.corpus import wordnet as wn
>>> print(wn.lemmas("insect")[0].in_topic_domains())
[Lemma('chirpy.a.01.chirpy'), Synset('holometabolism.n.01')]
:param relation_symbol: pointer symbol denoting a WordNet relation
:type relation_symbol: string
:return: list of target lemmas and/or synsets
"""
get_synset = self._wordnet_corpus_reader.synset_from_pos_and_offset
ssrels = self._synset._related(relation_symbol)
if (self._name, relation_symbol) not in self._synset._lemma_pointers:
return ssrels
return [
get_synset(pos, offset)._lemmas[lemma_index]
for pos, offset, lemma_index in self._synset._lemma_pointers[
self._name, relation_symbol
]
] + ssrels # a few domain relations concern both lemmas and synsets
def count(self):
"""Return the frequency count for this Lemma"""
return self._wordnet_corpus_reader.lemma_count(self)
def antonyms(self):
return self._related("!")
def derivationally_related_forms(self):
return self._related("+")
def pertainyms(self):
return self._related("\\")
class Synset(_WordNetObject):
"""Create a Synset from a "<lemma>.<pos>.<number>" string where:
<lemma> is the word's morphological stem
<pos> is one of the module attributes ADJ, ADJ_SAT, ADV, NOUN or VERB
<number> is the sense number, counting from 0.
Synset attributes, accessible via methods with the same name:
- name: The canonical name of this synset, formed using the first lemma
of this synset. Note that this may be different from the name
passed to the constructor if that string used a different lemma to
identify the synset.
- pos: The synset's part of speech, matching one of the module level
attributes ADJ, ADJ_SAT, ADV, NOUN or VERB.
- lemmas: A list of the Lemma objects for this synset.
- definition: The definition for this synset.
- examples: A list of example strings for this synset.
- offset: The offset in the WordNet dict file of this synset.
- lexname: The name of the lexicographer file containing this synset.
Synset methods:
Synsets have the following methods for retrieving related Synsets.
They correspond to the names for the pointer symbols defined here:
https://wordnet.princeton.edu/documentation/wninput5wn
These methods all return lists of Synsets.
- hypernyms, instance_hypernyms
- hyponyms, instance_hyponyms
- member_holonyms, substance_holonyms, part_holonyms
- member_meronyms, substance_meronyms, part_meronyms
- attributes
- entailments
- causes
- also_sees
- verb_groups
- similar_tos
Additionally, Synsets support the following methods specific to the
hypernym relation:
- root_hypernyms
- common_hypernyms
- lowest_common_hypernyms
Note that Synsets do not support the following relations because
these are defined by WordNet as lexical relations:
- antonyms
- derivationally_related_forms
- pertainyms
"""
__slots__ = [
"_pos",
"_offset",
"_name",
"_frame_ids",
"_lemmas",
"_lemma_names",
"_definition",
"_examples",
"_lexname",
"_pointers",
"_lemma_pointers",
"_max_depth",
"_min_depth",
]
def __init__(self, wordnet_corpus_reader):
self._wordnet_corpus_reader = wordnet_corpus_reader
# All of these attributes get initialized by
# WordNetCorpusReader._synset_from_pos_and_line()
self._pos = None
self._offset = None
self._name = None
self._frame_ids = []
self._lemmas = []
self._lemma_names = []
self._definition = None
self._examples = []
self._lexname = None # lexicographer name
self._all_hypernyms = None
self._pointers = defaultdict(set)
self._lemma_pointers = defaultdict(list)
def pos(self):
return self._pos
def offset(self):
return self._offset
def name(self):
return self._name
def frame_ids(self):
return self._frame_ids
def _doc(self, doc_type, default, lang="eng"):
"""Helper method for Synset.definition and Synset.examples"""
corpus = self._wordnet_corpus_reader
if lang not in corpus.langs():
return None
elif lang == "eng":
return default
else:
corpus._load_lang_data(lang)
of = corpus.ss2of(self)
i = corpus.lg_attrs.index(doc_type)
if of in corpus._lang_data[lang][i]:
return corpus._lang_data[lang][i][of]
else:
return None
def definition(self, lang="eng"):
"""Return definition in specified language"""
return self._doc("def", self._definition, lang=lang)
def examples(self, lang="eng"):
"""Return examples in specified language"""
return self._doc("exe", self._examples, lang=lang)
def lexname(self):
return self._lexname
def _needs_root(self):
if self._pos == NOUN and self._wordnet_corpus_reader.get_version() != "1.6":
return False
else:
return True
def lemma_names(self, lang="eng"):
"""Return all the lemma_names associated with the synset"""
if lang == "eng":
return self._lemma_names
else:
reader = self._wordnet_corpus_reader
reader._load_lang_data(lang)
i = reader.ss2of(self)
if i in reader._lang_data[lang][0]:
return reader._lang_data[lang][0][i]
else:
return []
def lemmas(self, lang="eng"):
"""Return all the lemma objects associated with the synset"""
if lang == "eng":
return self._lemmas
elif self._name:
self._wordnet_corpus_reader._load_lang_data(lang)
lemmark = []
lemmy = self.lemma_names(lang)
for lem in lemmy:
temp = Lemma(
self._wordnet_corpus_reader,
self,
lem,
self._wordnet_corpus_reader._lexnames.index(self.lexname()),
0,
None,
)
temp._lang = lang
lemmark.append(temp)
return lemmark
def root_hypernyms(self):
"""Get the topmost hypernyms of this synset in WordNet."""
result = []
seen = set()
todo = [self]
while todo:
next_synset = todo.pop()
if next_synset not in seen:
seen.add(next_synset)
next_hypernyms = (
next_synset.hypernyms() + next_synset.instance_hypernyms()
)
if not next_hypernyms:
result.append(next_synset)
else:
todo.extend(next_hypernyms)
return result
# Simpler implementation which makes incorrect assumption that
# hypernym hierarchy is acyclic:
#
# if not self.hypernyms():
# return [self]
# else:
# return list(set(root for h in self.hypernyms()
# for root in h.root_hypernyms()))
def max_depth(self):
"""
:return: The length of the longest hypernym path from this
synset to the root.
"""
if "_max_depth" not in self.__dict__:
hypernyms = self.hypernyms() + self.instance_hypernyms()
if not hypernyms:
self._max_depth = 0
else:
self._max_depth = 1 + max(h.max_depth() for h in hypernyms)
return self._max_depth
def min_depth(self):
"""
:return: The length of the shortest hypernym path from this
synset to the root.
"""
if "_min_depth" not in self.__dict__:
hypernyms = self.hypernyms() + self.instance_hypernyms()
if not hypernyms:
self._min_depth = 0
else:
self._min_depth = 1 + min(h.min_depth() for h in hypernyms)
return self._min_depth
def closure(self, rel, depth=-1):
"""
Return the transitive closure of source under the rel
relationship, breadth-first, discarding cycles:
>>> from nltk.corpus import wordnet as wn
>>> computer = wn.synset('computer.n.01')
>>> topic = lambda s:s.topic_domains()
>>> print(list(computer.closure(topic)))
[Synset('computer_science.n.01')]
UserWarning: Discarded redundant search for Synset('computer.n.01') at depth 2
Include redundant paths (but only once), avoiding duplicate searches
(from 'animal.n.01' to 'entity.n.01'):
>>> dog = wn.synset('dog.n.01')
>>> hyp = lambda s:s.hypernyms()
>>> print(list(dog.closure(hyp)))
[Synset('canine.n.02'), Synset('domestic_animal.n.01'), Synset('carnivore.n.01'),\
Synset('animal.n.01'), Synset('placental.n.01'), Synset('organism.n.01'),\
Synset('mammal.n.01'), Synset('living_thing.n.01'), Synset('vertebrate.n.01'),\
Synset('whole.n.02'), Synset('chordate.n.01'), Synset('object.n.01'),\
Synset('physical_entity.n.01'), Synset('entity.n.01')]
UserWarning: Discarded redundant search for Synset('animal.n.01') at depth 7
"""
from nltk.util import acyclic_breadth_first
for synset in acyclic_breadth_first(self, rel, depth):
if synset != self:
yield synset
from nltk.util import acyclic_depth_first as acyclic_tree
from nltk.util import unweighted_minimum_spanning_tree as mst
# Also add this shortcut?
# from nltk.util import unweighted_minimum_spanning_digraph as umsd
def tree(self, rel, depth=-1, cut_mark=None):
"""
Return the full relation tree, including self,
discarding cycles:
>>> from nltk.corpus import wordnet as wn
>>> from pprint import pprint
>>> computer = wn.synset('computer.n.01')
>>> topic = lambda s:s.topic_domains()
>>> pprint(computer.tree(topic))
[Synset('computer.n.01'), [Synset('computer_science.n.01')]]
UserWarning: Discarded redundant search for Synset('computer.n.01') at depth -3
But keep duplicate branches (from 'animal.n.01' to 'entity.n.01'):
>>> dog = wn.synset('dog.n.01')
>>> hyp = lambda s:s.hypernyms()
>>> pprint(dog.tree(hyp))
[Synset('dog.n.01'),
[Synset('canine.n.02'),
[Synset('carnivore.n.01'),
[Synset('placental.n.01'),
[Synset('mammal.n.01'),
[Synset('vertebrate.n.01'),
[Synset('chordate.n.01'),
[Synset('animal.n.01'),
[Synset('organism.n.01'),
[Synset('living_thing.n.01'),
[Synset('whole.n.02'),
[Synset('object.n.01'),
[Synset('physical_entity.n.01'),
[Synset('entity.n.01')]]]]]]]]]]]]],
[Synset('domestic_animal.n.01'),
[Synset('animal.n.01'),
[Synset('organism.n.01'),
[Synset('living_thing.n.01'),
[Synset('whole.n.02'),
[Synset('object.n.01'),
[Synset('physical_entity.n.01'), [Synset('entity.n.01')]]]]]]]]]
"""
from nltk.util import acyclic_branches_depth_first
return acyclic_branches_depth_first(self, rel, depth, cut_mark)
def hypernym_paths(self):
"""
Get the path(s) from this synset to the root, where each path is a
list of the synset nodes traversed on the way to the root.
:return: A list of lists, where each list gives the node sequence
connecting the initial ``Synset`` node and a root node.
"""
paths = []
hypernyms = self.hypernyms() + self.instance_hypernyms()
if len(hypernyms) == 0:
paths = [[self]]
for hypernym in hypernyms:
for ancestor_list in hypernym.hypernym_paths():
ancestor_list.append(self)
paths.append(ancestor_list)
return paths
def common_hypernyms(self, other):
"""
Find all synsets that are hypernyms of this synset and the
other synset.
:type other: Synset
:param other: other input synset.
:return: The synsets that are hypernyms of both synsets.
"""
if not self._all_hypernyms:
self._all_hypernyms = {
self_synset
for self_synsets in self._iter_hypernym_lists()
for self_synset in self_synsets
}
if not other._all_hypernyms:
other._all_hypernyms = {
other_synset
for other_synsets in other._iter_hypernym_lists()
for other_synset in other_synsets
}
return list(self._all_hypernyms.intersection(other._all_hypernyms))
def lowest_common_hypernyms(self, other, simulate_root=False, use_min_depth=False):
"""
Get a list of lowest synset(s) that both synsets have as a hypernym.
When `use_min_depth == False` this means that the synset which appears
as a hypernym of both `self` and `other` with the lowest maximum depth
is returned or if there are multiple such synsets at the same depth
they are all returned
However, if `use_min_depth == True` then the synset(s) which has/have
the lowest minimum depth and appear(s) in both paths is/are returned.
By setting the use_min_depth flag to True, the behavior of NLTK2 can be
preserved. This was changed in NLTK3 to give more accurate results in a
small set of cases, generally with synsets concerning people. (eg:
'chef.n.01', 'fireman.n.01', etc.)
This method is an implementation of Ted Pedersen's "Lowest Common
Subsumer" method from the Perl Wordnet module. It can return either
"self" or "other" if they are a hypernym of the other.
:type other: Synset
:param other: other input synset
:type simulate_root: bool
:param simulate_root: The various verb taxonomies do not
share a single root which disallows this metric from working for
synsets that are not connected. This flag (False by default)
creates a fake root that connects all the taxonomies. Set it
to True to enable this behavior. For the noun taxonomy,
there is usually a default root except for WordNet version 1.6.
If you are using wordnet 1.6, a fake root will need to be added
for nouns as well.
:type use_min_depth: bool
:param use_min_depth: This setting mimics older (v2) behavior of NLTK
wordnet If True, will use the min_depth function to calculate the
lowest common hypernyms. This is known to give strange results for
some synset pairs (eg: 'chef.n.01', 'fireman.n.01') but is retained
for backwards compatibility
:return: The synsets that are the lowest common hypernyms of both
synsets
"""
synsets = self.common_hypernyms(other)
if simulate_root:
fake_synset = Synset(None)
fake_synset._name = "*ROOT*"
fake_synset.hypernyms = lambda: []
fake_synset.instance_hypernyms = lambda: []
synsets.append(fake_synset)
try:
if use_min_depth:
max_depth = max(s.min_depth() for s in synsets)
unsorted_lch = [s for s in synsets if s.min_depth() == max_depth]
else:
max_depth = max(s.max_depth() for s in synsets)
unsorted_lch = [s for s in synsets if s.max_depth() == max_depth]
return sorted(unsorted_lch)
except ValueError:
return []
def hypernym_distances(self, distance=0, simulate_root=False):
"""
Get the path(s) from this synset to the root, counting the distance
of each node from the initial node on the way. A set of
(synset, distance) tuples is returned.
:type distance: int
:param distance: the distance (number of edges) from this hypernym to
the original hypernym ``Synset`` on which this method was called.
:return: A set of ``(Synset, int)`` tuples where each ``Synset`` is
a hypernym of the first ``Synset``.
"""
distances = {(self, distance)}
for hypernym in self._hypernyms() + self._instance_hypernyms():
distances |= hypernym.hypernym_distances(distance + 1, simulate_root=False)
if simulate_root:
fake_synset = Synset(None)
fake_synset._name = "*ROOT*"
fake_synset_distance = max(distances, key=itemgetter(1))[1]
distances.add((fake_synset, fake_synset_distance + 1))
return distances
def _shortest_hypernym_paths(self, simulate_root):
if self._name == "*ROOT*":
return {self: 0}
queue = deque([(self, 0)])
path = {}
while queue:
s, depth = queue.popleft()
if s in path:
continue
path[s] = depth
depth += 1
queue.extend((hyp, depth) for hyp in s._hypernyms())
queue.extend((hyp, depth) for hyp in s._instance_hypernyms())
if simulate_root:
fake_synset = Synset(None)
fake_synset._name = "*ROOT*"
path[fake_synset] = max(path.values()) + 1
return path
def shortest_path_distance(self, other, simulate_root=False):
"""
Returns the distance of the shortest path linking the two synsets (if
one exists). For each synset, all the ancestor nodes and their
distances are recorded and compared. The ancestor node common to both
synsets that can be reached with the minimum number of traversals is
used. If no ancestor nodes are common, None is returned. If a node is
compared with itself 0 is returned.
:type other: Synset
:param other: The Synset to which the shortest path will be found.
:return: The number of edges in the shortest path connecting the two
nodes, or None if no path exists.
"""
if self == other:
return 0
dist_dict1 = self._shortest_hypernym_paths(simulate_root)
dist_dict2 = other._shortest_hypernym_paths(simulate_root)
# For each ancestor synset common to both subject synsets, find the
# connecting path length. Return the shortest of these.
inf = float("inf")
path_distance = inf
for synset, d1 in dist_dict1.items():
d2 = dist_dict2.get(synset, inf)
path_distance = min(path_distance, d1 + d2)
return None if math.isinf(path_distance) else path_distance
# interface to similarity methods
def path_similarity(self, other, verbose=False, simulate_root=True):
"""
Path Distance Similarity:
Return a score denoting how similar two word senses are, based on the
shortest path that connects the senses in the is-a (hypernym/hypnoym)
taxonomy. The score is in the range 0 to 1, except in those cases where
a path cannot be found (will only be true for verbs as there are many
distinct verb taxonomies), in which case None is returned. A score of
1 represents identity i.e. comparing a sense with itself will return 1.
:type other: Synset
:param other: The ``Synset`` that this ``Synset`` is being compared to.
:type simulate_root: bool
:param simulate_root: The various verb taxonomies do not
share a single root which disallows this metric from working for
synsets that are not connected. This flag (True by default)
creates a fake root that connects all the taxonomies. Set it
to false to disable this behavior. For the noun taxonomy,
there is usually a default root except for WordNet version 1.6.
If you are using wordnet 1.6, a fake root will be added for nouns
as well.
:return: A score denoting the similarity of the two ``Synset`` objects,
normally between 0 and 1. None is returned if no connecting path
could be found. 1 is returned if a ``Synset`` is compared with
itself.
"""
distance = self.shortest_path_distance(
other,
simulate_root=simulate_root and (self._needs_root() or other._needs_root()),
)
if distance is None or distance < 0:
return None
return 1.0 / (distance + 1)
def lch_similarity(self, other, verbose=False, simulate_root=True):
"""
Leacock Chodorow Similarity:
Return a score denoting how similar two word senses are, based on the
shortest path that connects the senses (as above) and the maximum depth
of the taxonomy in which the senses occur. The relationship is given as
-log(p/2d) where p is the shortest path length and d is the taxonomy
depth.
:type other: Synset
:param other: The ``Synset`` that this ``Synset`` is being compared to.
:type simulate_root: bool
:param simulate_root: The various verb taxonomies do not
share a single root which disallows this metric from working for
synsets that are not connected. This flag (True by default)
creates a fake root that connects all the taxonomies. Set it
to false to disable this behavior. For the noun taxonomy,
there is usually a default root except for WordNet version 1.6.
If you are using wordnet 1.6, a fake root will be added for nouns
as well.
:return: A score denoting the similarity of the two ``Synset`` objects,
normally greater than 0. None is returned if no connecting path
could be found. If a ``Synset`` is compared with itself, the
maximum score is returned, which varies depending on the taxonomy
depth.
"""
if self._pos != other._pos:
raise WordNetError(
"Computing the lch similarity requires "
"%s and %s to have the same part of speech." % (self, other)
)
need_root = self._needs_root()
if self._pos not in self._wordnet_corpus_reader._max_depth:
self._wordnet_corpus_reader._compute_max_depth(self._pos, need_root)
depth = self._wordnet_corpus_reader._max_depth[self._pos]
distance = self.shortest_path_distance(
other, simulate_root=simulate_root and need_root
)
if distance is None or distance < 0 or depth == 0:
return None
return -math.log((distance + 1) / (2.0 * depth))
def wup_similarity(self, other, verbose=False, simulate_root=True):
"""
Wu-Palmer Similarity:
Return a score denoting how similar two word senses are, based on the
depth of the two senses in the taxonomy and that of their Least Common
Subsumer (most specific ancestor node). Previously, the scores computed
by this implementation did _not_ always agree with those given by
Pedersen's Perl implementation of WordNet Similarity. However, with
the addition of the simulate_root flag (see below), the score for
verbs now almost always agree but not always for nouns.
The LCS does not necessarily feature in the shortest path connecting
the two senses, as it is by definition the common ancestor deepest in
the taxonomy, not closest to the two senses. Typically, however, it
will so feature. Where multiple candidates for the LCS exist, that
whose shortest path to the root node is the longest will be selected.
Where the LCS has multiple paths to the root, the longer path is used
for the purposes of the calculation.
:type other: Synset
:param other: The ``Synset`` that this ``Synset`` is being compared to.
:type simulate_root: bool
:param simulate_root: The various verb taxonomies do not
share a single root which disallows this metric from working for
synsets that are not connected. This flag (True by default)
creates a fake root that connects all the taxonomies. Set it
to false to disable this behavior. For the noun taxonomy,
there is usually a default root except for WordNet version 1.6.
If you are using wordnet 1.6, a fake root will be added for nouns
as well.
:return: A float score denoting the similarity of the two ``Synset``
objects, normally greater than zero. If no connecting path between
the two senses can be found, None is returned.
"""
need_root = self._needs_root() or other._needs_root()
# Note that to preserve behavior from NLTK2 we set use_min_depth=True
# It is possible that more accurate results could be obtained by
# removing this setting and it should be tested later on
subsumers = self.lowest_common_hypernyms(
other, simulate_root=simulate_root and need_root, use_min_depth=True
)
# If no LCS was found return None
if len(subsumers) == 0:
return None
subsumer = self if self in subsumers else subsumers[0]
# Get the longest path from the LCS to the root,
# including a correction:
# - add one because the calculations include both the start and end
# nodes
depth = subsumer.max_depth() + 1
# Note: No need for an additional add-one correction for non-nouns
# to account for an imaginary root node because that is now
# automatically handled by simulate_root
# if subsumer._pos != NOUN:
# depth += 1