diff --git a/nltk/__init__.py b/nltk/__init__.py index 1cd72fb459..a96ac22b25 100644 --- a/nltk/__init__.py +++ b/nltk/__init__.py @@ -135,7 +135,6 @@ def _fake_Popen(*args, **kwargs): from nltk.grammar import * from nltk.probability import * from nltk.text import * -from nltk.tree import * from nltk.util import * from nltk.jsontags import * @@ -151,6 +150,7 @@ def _fake_Popen(*args, **kwargs): from nltk.tag import * from nltk.tokenize import * from nltk.translate import * +from nltk.tree import * from nltk.sem import * from nltk.stem import * @@ -200,7 +200,7 @@ def _fake_Popen(*args, **kwargs): from nltk import ccg, chunk, classify, collocations from nltk import data, featstruct, grammar, help, inference, metrics from nltk import misc, parse, probability, sem, stem, wsd -from nltk import tag, tbl, text, tokenize, translate, tree, treetransforms, util +from nltk import tag, tbl, text, tokenize, translate, tree, util # FIXME: override any accidentally imported demo, see https://github.com/nltk/nltk/issues/2116 diff --git a/nltk/test/tree.doctest b/nltk/test/tree.doctest index a479273283..feea8201bb 100644 --- a/nltk/test/tree.doctest +++ b/nltk/test/tree.doctest @@ -113,7 +113,7 @@ type: >>> print(tree) (VP (V enjoyed) (NP my cookie)) >>> print(type(tree)) - + >>> tree[1] = 'x' Traceback (most recent call last): . . . @@ -345,7 +345,7 @@ Parented trees can be created from strings using the classmethod >>> print(ptree) (VP (VERB saw) (NP (DET the) (NOUN dog))) >>> print(type(ptree)) - + Parented trees can also be created by using the classmethod `ParentedTree.convert` to convert another type of tree to a parented @@ -356,7 +356,7 @@ tree: >>> print(ptree) (VP (VERB saw) (NP (DET the) (NOUN dog))) >>> print(type(ptree)) - + .. clean-up: @@ -802,13 +802,59 @@ Test that a tree can not be given multiple parents: [more to be written] +Shallow copying can be tricky for Tree and several of its subclasses. +For shallow copies of Tree, only the root node is reconstructed, while +all the children are shared between the two trees. Modify the children +of one tree - and the shallowly copied tree will also update. + + >>> from nltk.tree import Tree, ParentedTree, MultiParentedTree + >>> tree = Tree.fromstring("(TOP (S (NP (NNP Bell,)) (NP (NP (DT a) (NN company)) (SBAR (WHNP (WDT which)) (S (VP (VBZ is) (VP (VBN based) (PP (IN in) (NP (NNP LA,)))))))) (VP (VBZ makes) (CC and) (VBZ distributes) (NP (NN computer))) (. products.)))") + >>> copy_tree = tree.copy(deep=False) + >>> tree == copy_tree # Ensure identical labels and nodes + True + >>> id(copy_tree[0]) == id(tree[0]) # Ensure shallow copy - the children are the same objects in memory + True + +For ParentedTree objects, this behaviour is not possible. With a shallow +copy, the children of the root node would be reused for both the original +and the shallow copy. For this to be possible, some children would need +to have multiple parents. As this is forbidden for ParentedTree objects, +attempting to make a shallow copy will cause a warning, and a deep copy +is made instead. + + >>> ptree = ParentedTree.fromstring("(TOP (S (NP (NNP Bell,)) (NP (NP (DT a) (NN company)) (SBAR (WHNP (WDT which)) (S (VP (VBZ is) (VP (VBN based) (PP (IN in) (NP (NNP LA,)))))))) (VP (VBZ makes) (CC and) (VBZ distributes) (NP (NN computer))) (. products.)))") + >>> copy_ptree = ptree.copy(deep=False) + >>> copy_ptree == ptree # Ensure identical labels and nodes + True + >>> id(copy_ptree[0]) != id(ptree[0]) # Shallow copying isn't supported - it defaults to deep copy. + True + +For MultiParentedTree objects, the issue of only allowing one parent that +can be seen for ParentedTree objects is no more. Shallow copying a +MultiParentedTree gives the children of the root node two parents: +the original and the newly copied root. + + >>> mptree = MultiParentedTree.fromstring("(TOP (S (NP (NNP Bell,)) (NP (NP (DT a) (NN company)) (SBAR (WHNP (WDT which)) (S (VP (VBZ is) (VP (VBN based) (PP (IN in) (NP (NNP LA,)))))))) (VP (VBZ makes) (CC and) (VBZ distributes) (NP (NN computer))) (. products.)))") + >>> len(mptree[0].parents()) + 1 + >>> copy_mptree = mptree.copy(deep=False) + >>> copy_mptree == mptree # Ensure identical labels and nodes + True + >>> len(mptree[0].parents()) + 2 + >>> len(copy_mptree[0].parents()) + 2 + +Shallow copying a MultiParentedTree is similar to creating a second root +which is identically labeled as the root on which the copy method was called. + ImmutableParentedTree Regression Tests -------------------------------------- >>> iptree = ImmutableParentedTree.convert(ptree) >>> type(iptree) - + >>> del iptree[0] Traceback (most recent call last): . . . @@ -1110,7 +1156,7 @@ ImmutableMultiParentedTree Regression Tests >>> imptree = ImmutableMultiParentedTree.convert(mptree) >>> type(imptree) - + >>> del imptree[0] Traceback (most recent call last): . . . @@ -1137,7 +1183,7 @@ ProbabilisticTree Regression Tests >>> imprtree = ImmutableProbabilisticTree.convert(prtree) >>> type(imprtree) - + >>> del imprtree[0] Traceback (most recent call last): . . . @@ -1155,3 +1201,23 @@ This used to discard the ``(B b)`` subtree (fixed in svn 6270): >>> print(Tree.fromstring('((A a) (B b))')) ( (A a) (B b)) + +Pickling ParentedTree instances didn't work for Python 3.7 onwards (See #2478) + + >>> import pickle + >>> tree = ParentedTree.fromstring('(S (NN x) (NP x) (NN x))') + >>> print(tree) + (S (NN x) (NP x) (NN x)) + + >>> pickled = pickle.dumps(tree) + >>> tree_loaded = pickle.loads(pickled) + >>> print(tree_loaded) + (S (NN x) (NP x) (NN x)) + +ParentedTree used to be impossible to (deep)copy. (See #1324) + + >>> from nltk.tree import ParentedTree + >>> import copy + >>> tree = ParentedTree.fromstring("(TOP (S (NP (NNP Bell,)) (NP (NP (DT a) (NN company)) (SBAR (WHNP (WDT which)) (S (VP (VBZ is) (VP (VBN based) (PP (IN in) (NP (NNP LA,)))))))) (VP (VBZ makes) (CC and) (VBZ distributes) (NP (NN computer))) (. products.)))") + >>> tree == copy.deepcopy(tree) == copy.copy(tree) == tree.copy(deep=True) == tree.copy() + True diff --git a/nltk/test/treeprettyprinter.doctest b/nltk/test/treeprettyprinter.doctest index cea049c98e..9f11af21e4 100644 --- a/nltk/test/treeprettyprinter.doctest +++ b/nltk/test/treeprettyprinter.doctest @@ -1,12 +1,11 @@ .. Copyright (C) 2001-2021 NLTK Project .. For license information, see LICENSE.TXT -======================================================== - Unit tests for nltk.treeprettyprinter.TreePrettyPrinter -======================================================== +========================================================= + Unit tests for nltk.tree.prettyprinter.TreePrettyPrinter +========================================================= - >>> from nltk.tree import Tree - >>> from nltk.treeprettyprinter import TreePrettyPrinter + >>> from nltk.tree import Tree, TreePrettyPrinter Tree nr 2170 from nltk.corpus.treebank: @@ -124,3 +123,55 @@ A discontinuous tree: noun verb prep det noun verb verb verb punct verb vg verb punct │ │ │ │ │ │ │ │ │ │ │ │ │ Ze had met haar moeder kunnen gaan winkelen , zwemmen of terrassen . + +Importing TreePrettyPrinter +--------------------------- + +First of all, a simple tree will be constructed:: + + >>> from nltk.tree import Tree + >>> tree = Tree.fromstring('(S (NP Mary) (VP walks))') + +We'll use this sample tree to show that the method of importing `TreePrettyPrinter` work correctly: + +- Recommended:: + + >>> from nltk.tree import TreePrettyPrinter + >>> print(TreePrettyPrinter(tree).text()) + S + ____|____ + NP VP + | | + Mary walks + +- Alternative but valid options:: + + >>> from nltk import TreePrettyPrinter + >>> print(TreePrettyPrinter(tree).text()) + S + ____|____ + NP VP + | | + Mary walks + + >>> from nltk.tree.prettyprinter import TreePrettyPrinter + >>> print(TreePrettyPrinter(tree).text()) + S + ____|____ + NP VP + | | + Mary walks + +- Deprecated, do not use:: + + >>> from nltk.treeprettyprinter import TreePrettyPrinter + >>> print(TreePrettyPrinter(tree).text()) + S + ____|____ + NP VP + | | + Mary walks + + This method will throw a DeprecationWarning:: + + Import `TreePrettyPrinter` using `from nltk.tree import TreePrettyPrinter` instead. diff --git a/nltk/test/treetransforms.doctest b/nltk/test/treetransforms.doctest index a3870138d6..411499340c 100644 --- a/nltk/test/treetransforms.doctest +++ b/nltk/test/treetransforms.doctest @@ -6,8 +6,7 @@ Unit tests for the TreeTransformation class ------------------------------------------- >>> from copy import deepcopy - >>> from nltk.tree import * - >>> from nltk.treetransforms import * + >>> from nltk.tree import Tree, collapse_unary, chomsky_normal_form, un_chomsky_normal_form >>> tree_string = "(TOP (S (S (VP (VBN Turned) (ADVP (RB loose)) (PP (IN in) (NP (NP (NNP Shane) (NNP Longman) (POS 's)) (NN trading) (NN room))))) (, ,) (NP (DT the) (NN yuppie) (NNS dealers)) (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right)))) (. .)))" diff --git a/nltk/tree/__init__.py b/nltk/tree/__init__.py new file mode 100644 index 0000000000..9b052ac6dd --- /dev/null +++ b/nltk/tree/__init__.py @@ -0,0 +1,35 @@ +# Natural Language Toolkit: Machine Translation +# +# Copyright (C) 2001-2021 NLTK Project +# Author: Edward Loper +# Steven Bird +# Peter Ljunglöf +# Tom Aarsen <> +# URL: +# For license information, see LICENSE.TXT + +""" +NLTK Tree Package + +This package may be used for representing hierarchical language +structures, such as syntax trees and morphological trees. +""" + +# TODO: add LabelledTree (can be used for dependency trees) + +from nltk.tree.immutable import ( + ImmutableMultiParentedTree, + ImmutableParentedTree, + ImmutableProbabilisticTree, + ImmutableTree, +) +from nltk.tree.parented import MultiParentedTree, ParentedTree +from nltk.tree.parsing import bracket_parse, sinica_parse +from nltk.tree.prettyprinter import TreePrettyPrinter +from nltk.tree.probabilistic import ProbabilisticTree +from nltk.tree.transforms import ( + chomsky_normal_form, + collapse_unary, + un_chomsky_normal_form, +) +from nltk.tree.tree import Tree diff --git a/nltk/tree/immutable.py b/nltk/tree/immutable.py new file mode 100644 index 0000000000..072193927d --- /dev/null +++ b/nltk/tree/immutable.py @@ -0,0 +1,124 @@ +# Natural Language Toolkit: Text Trees +# +# Copyright (C) 2001-2021 NLTK Project +# Author: Edward Loper +# Steven Bird +# Peter Ljunglöf +# Tom Aarsen <> +# URL: +# For license information, see LICENSE.TXT + +from nltk.probability import ProbabilisticMixIn +from nltk.tree.parented import MultiParentedTree, ParentedTree +from nltk.tree.tree import Tree + + +class ImmutableTree(Tree): + def __init__(self, node, children=None): + super().__init__(node, children) + # Precompute our hash value. This ensures that we're really + # immutable. It also means we only have to calculate it once. + try: + self._hash = hash((self._label, tuple(self))) + except (TypeError, ValueError) as e: + raise ValueError( + "%s: node value and children " "must be immutable" % type(self).__name__ + ) from e + + def __setitem__(self, index, value): + raise ValueError("%s may not be modified" % type(self).__name__) + + def __setslice__(self, i, j, value): + raise ValueError("%s may not be modified" % type(self).__name__) + + def __delitem__(self, index): + raise ValueError("%s may not be modified" % type(self).__name__) + + def __delslice__(self, i, j): + raise ValueError("%s may not be modified" % type(self).__name__) + + def __iadd__(self, other): + raise ValueError("%s may not be modified" % type(self).__name__) + + def __imul__(self, other): + raise ValueError("%s may not be modified" % type(self).__name__) + + def append(self, v): + raise ValueError("%s may not be modified" % type(self).__name__) + + def extend(self, v): + raise ValueError("%s may not be modified" % type(self).__name__) + + def pop(self, v=None): + raise ValueError("%s may not be modified" % type(self).__name__) + + def remove(self, v): + raise ValueError("%s may not be modified" % type(self).__name__) + + def reverse(self): + raise ValueError("%s may not be modified" % type(self).__name__) + + def sort(self): + raise ValueError("%s may not be modified" % type(self).__name__) + + def __hash__(self): + return self._hash + + def set_label(self, value): + """ + Set the node label. This will only succeed the first time the + node label is set, which should occur in ImmutableTree.__init__(). + """ + if hasattr(self, "_label"): + raise ValueError("%s may not be modified" % type(self).__name__) + self._label = value + + +class ImmutableProbabilisticTree(ImmutableTree, ProbabilisticMixIn): + def __init__(self, node, children=None, **prob_kwargs): + ImmutableTree.__init__(self, node, children) + ProbabilisticMixIn.__init__(self, **prob_kwargs) + self._hash = hash((self._label, tuple(self), self.prob())) + + # We have to patch up these methods to make them work right: + def _frozen_class(self): + return ImmutableProbabilisticTree + + def __repr__(self): + return f"{Tree.__repr__(self)} [{self.prob()}]" + + def __str__(self): + return f"{self.pformat(margin=60)} [{self.prob()}]" + + def copy(self, deep=False): + if not deep: + return type(self)(self._label, self, prob=self.prob()) + else: + return type(self).convert(self) + + @classmethod + def convert(cls, val): + if isinstance(val, Tree): + children = [cls.convert(child) for child in val] + if isinstance(val, ProbabilisticMixIn): + return cls(val._label, children, prob=val.prob()) + else: + return cls(val._label, children, prob=1.0) + else: + return val + + +class ImmutableParentedTree(ImmutableTree, ParentedTree): + pass + + +class ImmutableMultiParentedTree(ImmutableTree, MultiParentedTree): + pass + + +__all__ = [ + "ImmutableProbabilisticTree", + "ImmutableTree", + "ImmutableParentedTree", + "ImmutableMultiParentedTree", +] diff --git a/nltk/tree/parented.py b/nltk/tree/parented.py new file mode 100644 index 0000000000..31955ec0d5 --- /dev/null +++ b/nltk/tree/parented.py @@ -0,0 +1,590 @@ +# Natural Language Toolkit: Text Trees +# +# Copyright (C) 2001-2021 NLTK Project +# Author: Edward Loper +# Steven Bird +# Peter Ljunglöf +# Tom Aarsen <> +# URL: +# For license information, see LICENSE.TXT + +import warnings +from abc import ABCMeta, abstractmethod + +from nltk.tree.tree import Tree +from nltk.util import slice_bounds + + +###################################################################### +## Parented trees +###################################################################### +class AbstractParentedTree(Tree, metaclass=ABCMeta): + """ + An abstract base class for a ``Tree`` that automatically maintains + pointers to parent nodes. These parent pointers are updated + whenever any change is made to a tree's structure. Two subclasses + are currently defined: + + - ``ParentedTree`` is used for tree structures where each subtree + has at most one parent. This class should be used in cases + where there is no"sharing" of subtrees. + + - ``MultiParentedTree`` is used for tree structures where a + subtree may have zero or more parents. This class should be + used in cases where subtrees may be shared. + + Subclassing + =========== + The ``AbstractParentedTree`` class redefines all operations that + modify a tree's structure to call two methods, which are used by + subclasses to update parent information: + + - ``_setparent()`` is called whenever a new child is added. + - ``_delparent()`` is called whenever a child is removed. + """ + + def __init__(self, node, children=None): + super().__init__(node, children) + # If children is None, the tree is read from node, and + # all parents will be set during parsing. + if children is not None: + # Otherwise we have to set the parent of the children. + # Iterate over self, and *not* children, because children + # might be an iterator. + for i, child in enumerate(self): + if isinstance(child, Tree): + self._setparent(child, i, dry_run=True) + for i, child in enumerate(self): + if isinstance(child, Tree): + self._setparent(child, i) + + # //////////////////////////////////////////////////////////// + # Parent management + # //////////////////////////////////////////////////////////// + @abstractmethod + def _setparent(self, child, index, dry_run=False): + """ + Update the parent pointer of ``child`` to point to ``self``. This + method is only called if the type of ``child`` is ``Tree``; + i.e., it is not called when adding a leaf to a tree. This method + is always called before the child is actually added to the + child list of ``self``. + + :type child: Tree + :type index: int + :param index: The index of ``child`` in ``self``. + :raise TypeError: If ``child`` is a tree with an impropriate + type. Typically, if ``child`` is a tree, then its type needs + to match the type of ``self``. This prevents mixing of + different tree types (single-parented, multi-parented, and + non-parented). + :param dry_run: If true, the don't actually set the child's + parent pointer; just check for any error conditions, and + raise an exception if one is found. + """ + + @abstractmethod + def _delparent(self, child, index): + """ + Update the parent pointer of ``child`` to not point to self. This + method is only called if the type of ``child`` is ``Tree``; i.e., it + is not called when removing a leaf from a tree. This method + is always called before the child is actually removed from the + child list of ``self``. + + :type child: Tree + :type index: int + :param index: The index of ``child`` in ``self``. + """ + + # //////////////////////////////////////////////////////////// + # Methods that add/remove children + # //////////////////////////////////////////////////////////// + # Every method that adds or removes a child must make + # appropriate calls to _setparent() and _delparent(). + + def __delitem__(self, index): + # del ptree[start:stop] + if isinstance(index, slice): + start, stop, step = slice_bounds(self, index, allow_step=True) + # Clear all the children pointers. + for i in range(start, stop, step): + if isinstance(self[i], Tree): + self._delparent(self[i], i) + # Delete the children from our child list. + super().__delitem__(index) + + # del ptree[i] + elif isinstance(index, int): + if index < 0: + index += len(self) + if index < 0: + raise IndexError("index out of range") + # Clear the child's parent pointer. + if isinstance(self[index], Tree): + self._delparent(self[index], index) + # Remove the child from our child list. + super().__delitem__(index) + + elif isinstance(index, (list, tuple)): + # del ptree[()] + if len(index) == 0: + raise IndexError("The tree position () may not be deleted.") + # del ptree[(i,)] + elif len(index) == 1: + del self[index[0]] + # del ptree[i1, i2, i3] + else: + del self[index[0]][index[1:]] + + else: + raise TypeError( + "%s indices must be integers, not %s" + % (type(self).__name__, type(index).__name__) + ) + + def __setitem__(self, index, value): + # ptree[start:stop] = value + if isinstance(index, slice): + start, stop, step = slice_bounds(self, index, allow_step=True) + # make a copy of value, in case it's an iterator + if not isinstance(value, (list, tuple)): + value = list(value) + # Check for any error conditions, so we can avoid ending + # up in an inconsistent state if an error does occur. + for i, child in enumerate(value): + if isinstance(child, Tree): + self._setparent(child, start + i * step, dry_run=True) + # clear the child pointers of all parents we're removing + for i in range(start, stop, step): + if isinstance(self[i], Tree): + self._delparent(self[i], i) + # set the child pointers of the new children. We do this + # after clearing *all* child pointers, in case we're e.g. + # reversing the elements in a tree. + for i, child in enumerate(value): + if isinstance(child, Tree): + self._setparent(child, start + i * step) + # finally, update the content of the child list itself. + super().__setitem__(index, value) + + # ptree[i] = value + elif isinstance(index, int): + if index < 0: + index += len(self) + if index < 0: + raise IndexError("index out of range") + # if the value is not changing, do nothing. + if value is self[index]: + return + # Set the new child's parent pointer. + if isinstance(value, Tree): + self._setparent(value, index) + # Remove the old child's parent pointer + if isinstance(self[index], Tree): + self._delparent(self[index], index) + # Update our child list. + super().__setitem__(index, value) + + elif isinstance(index, (list, tuple)): + # ptree[()] = value + if len(index) == 0: + raise IndexError("The tree position () may not be assigned to.") + # ptree[(i,)] = value + elif len(index) == 1: + self[index[0]] = value + # ptree[i1, i2, i3] = value + else: + self[index[0]][index[1:]] = value + + else: + raise TypeError( + "%s indices must be integers, not %s" + % (type(self).__name__, type(index).__name__) + ) + + def append(self, child): + if isinstance(child, Tree): + self._setparent(child, len(self)) + super().append(child) + + def extend(self, children): + for child in children: + if isinstance(child, Tree): + self._setparent(child, len(self)) + super().append(child) + + def insert(self, index, child): + # Handle negative indexes. Note that if index < -len(self), + # we do *not* raise an IndexError, unlike __getitem__. This + # is done for consistency with list.__getitem__ and list.index. + if index < 0: + index += len(self) + if index < 0: + index = 0 + # Set the child's parent, and update our child list. + if isinstance(child, Tree): + self._setparent(child, index) + super().insert(index, child) + + def pop(self, index=-1): + if index < 0: + index += len(self) + if index < 0: + raise IndexError("index out of range") + if isinstance(self[index], Tree): + self._delparent(self[index], index) + return super().pop(index) + + # n.b.: like `list`, this is done by equality, not identity! + # To remove a specific child, use del ptree[i]. + def remove(self, child): + index = self.index(child) + if isinstance(self[index], Tree): + self._delparent(self[index], index) + super().remove(child) + + # We need to implement __getslice__ and friends, even though + # they're deprecated, because otherwise list.__getslice__ will get + # called (since we're subclassing from list). Just delegate to + # __getitem__ etc., but use max(0, start) and max(0, stop) because + # because negative indices are already handled *before* + # __getslice__ is called; and we don't want to double-count them. + if hasattr(list, "__getslice__"): + + def __getslice__(self, start, stop): + return self.__getitem__(slice(max(0, start), max(0, stop))) + + def __delslice__(self, start, stop): + return self.__delitem__(slice(max(0, start), max(0, stop))) + + def __setslice__(self, start, stop, value): + return self.__setitem__(slice(max(0, start), max(0, stop)), value) + + def __getnewargs__(self): + """Method used by the pickle module when un-pickling. + This method provides the arguments passed to ``__new__`` + upon un-pickling. Without this method, ParentedTree instances + cannot be pickled and unpickled in Python 3.7+ onwards. + + :return: Tuple of arguments for ``__new__``, i.e. the label + and the children of this node. + :rtype: Tuple[Any, List[AbstractParentedTree]] + """ + return (self._label, list(self)) + + +class ParentedTree(AbstractParentedTree): + """ + A ``Tree`` that automatically maintains parent pointers for + single-parented trees. The following are methods for querying + the structure of a parented tree: ``parent``, ``parent_index``, + ``left_sibling``, ``right_sibling``, ``root``, ``treeposition``. + + Each ``ParentedTree`` may have at most one parent. In + particular, subtrees may not be shared. Any attempt to reuse a + single ``ParentedTree`` as a child of more than one parent (or + as multiple children of the same parent) will cause a + ``ValueError`` exception to be raised. + + ``ParentedTrees`` should never be used in the same tree as ``Trees`` + or ``MultiParentedTrees``. Mixing tree implementations may result + in incorrect parent pointers and in ``TypeError`` exceptions. + """ + + def __init__(self, node, children=None): + self._parent = None + """The parent of this Tree, or None if it has no parent.""" + super().__init__(node, children) + if children is None: + # If children is None, the tree is read from node. + # After parsing, the parent of the immediate children + # will point to an intermediate tree, not self. + # We fix this by brute force: + for i, child in enumerate(self): + if isinstance(child, Tree): + child._parent = None + self._setparent(child, i) + + def _frozen_class(self): + from nltk.tree.immutable import ImmutableParentedTree + + return ImmutableParentedTree + + def copy(self, deep=False): + if not deep: + warnings.warn( + f"{self.__class__.__name__} objects do not support shallow copies. Defaulting to a deep copy." + ) + return super().copy(deep=True) + + # ///////////////////////////////////////////////////////////////// + # Methods + # ///////////////////////////////////////////////////////////////// + + def parent(self): + """The parent of this tree, or None if it has no parent.""" + return self._parent + + def parent_index(self): + """ + The index of this tree in its parent. I.e., + ``ptree.parent()[ptree.parent_index()] is ptree``. Note that + ``ptree.parent_index()`` is not necessarily equal to + ``ptree.parent.index(ptree)``, since the ``index()`` method + returns the first child that is equal to its argument. + """ + if self._parent is None: + return None + for i, child in enumerate(self._parent): + if child is self: + return i + assert False, "expected to find self in self._parent!" + + def left_sibling(self): + """The left sibling of this tree, or None if it has none.""" + parent_index = self.parent_index() + if self._parent and parent_index > 0: + return self._parent[parent_index - 1] + return None # no left sibling + + def right_sibling(self): + """The right sibling of this tree, or None if it has none.""" + parent_index = self.parent_index() + if self._parent and parent_index < (len(self._parent) - 1): + return self._parent[parent_index + 1] + return None # no right sibling + + def root(self): + """ + The root of this tree. I.e., the unique ancestor of this tree + whose parent is None. If ``ptree.parent()`` is None, then + ``ptree`` is its own root. + """ + root = self + while root.parent() is not None: + root = root.parent() + return root + + def treeposition(self): + """ + The tree position of this tree, relative to the root of the + tree. I.e., ``ptree.root[ptree.treeposition] is ptree``. + """ + if self.parent() is None: + return () + else: + return self.parent().treeposition() + (self.parent_index(),) + + # ///////////////////////////////////////////////////////////////// + # Parent Management + # ///////////////////////////////////////////////////////////////// + + def _delparent(self, child, index): + # Sanity checks + assert isinstance(child, ParentedTree) + assert self[index] is child + assert child._parent is self + + # Delete child's parent pointer. + child._parent = None + + def _setparent(self, child, index, dry_run=False): + # If the child's type is incorrect, then complain. + if not isinstance(child, ParentedTree): + raise TypeError("Can not insert a non-ParentedTree into a ParentedTree") + + # If child already has a parent, then complain. + if hasattr(child, "_parent") and child._parent is not None: + raise ValueError("Can not insert a subtree that already has a parent.") + + # Set child's parent pointer & index. + if not dry_run: + child._parent = self + + +class MultiParentedTree(AbstractParentedTree): + """ + A ``Tree`` that automatically maintains parent pointers for + multi-parented trees. The following are methods for querying the + structure of a multi-parented tree: ``parents()``, ``parent_indices()``, + ``left_siblings()``, ``right_siblings()``, ``roots``, ``treepositions``. + + Each ``MultiParentedTree`` may have zero or more parents. In + particular, subtrees may be shared. If a single + ``MultiParentedTree`` is used as multiple children of the same + parent, then that parent will appear multiple times in its + ``parents()`` method. + + ``MultiParentedTrees`` should never be used in the same tree as + ``Trees`` or ``ParentedTrees``. Mixing tree implementations may + result in incorrect parent pointers and in ``TypeError`` exceptions. + """ + + def __init__(self, node, children=None): + self._parents = [] + """A list of this tree's parents. This list should not + contain duplicates, even if a parent contains this tree + multiple times.""" + super().__init__(node, children) + if children is None: + # If children is None, the tree is read from node. + # After parsing, the parent(s) of the immediate children + # will point to an intermediate tree, not self. + # We fix this by brute force: + for i, child in enumerate(self): + if isinstance(child, Tree): + child._parents = [] + self._setparent(child, i) + + def _frozen_class(self): + from nltk.tree.immutable import ImmutableMultiParentedTree + + return ImmutableMultiParentedTree + + # ///////////////////////////////////////////////////////////////// + # Methods + # ///////////////////////////////////////////////////////////////// + + def parents(self): + """ + The set of parents of this tree. If this tree has no parents, + then ``parents`` is the empty set. To check if a tree is used + as multiple children of the same parent, use the + ``parent_indices()`` method. + + :type: list(MultiParentedTree) + """ + return list(self._parents) + + def left_siblings(self): + """ + A list of all left siblings of this tree, in any of its parent + trees. A tree may be its own left sibling if it is used as + multiple contiguous children of the same parent. A tree may + appear multiple times in this list if it is the left sibling + of this tree with respect to multiple parents. + + :type: list(MultiParentedTree) + """ + return [ + parent[index - 1] + for (parent, index) in self._get_parent_indices() + if index > 0 + ] + + def right_siblings(self): + """ + A list of all right siblings of this tree, in any of its parent + trees. A tree may be its own right sibling if it is used as + multiple contiguous children of the same parent. A tree may + appear multiple times in this list if it is the right sibling + of this tree with respect to multiple parents. + + :type: list(MultiParentedTree) + """ + return [ + parent[index + 1] + for (parent, index) in self._get_parent_indices() + if index < (len(parent) - 1) + ] + + def _get_parent_indices(self): + return [ + (parent, index) + for parent in self._parents + for index, child in enumerate(parent) + if child is self + ] + + def roots(self): + """ + The set of all roots of this tree. This set is formed by + tracing all possible parent paths until trees with no parents + are found. + + :type: list(MultiParentedTree) + """ + return list(self._get_roots_helper({}).values()) + + def _get_roots_helper(self, result): + if self._parents: + for parent in self._parents: + parent._get_roots_helper(result) + else: + result[id(self)] = self + return result + + def parent_indices(self, parent): + """ + Return a list of the indices where this tree occurs as a child + of ``parent``. If this child does not occur as a child of + ``parent``, then the empty list is returned. The following is + always true:: + + for parent_index in ptree.parent_indices(parent): + parent[parent_index] is ptree + """ + if parent not in self._parents: + return [] + else: + return [index for (index, child) in enumerate(parent) if child is self] + + def treepositions(self, root): + """ + Return a list of all tree positions that can be used to reach + this multi-parented tree starting from ``root``. I.e., the + following is always true:: + + for treepos in ptree.treepositions(root): + root[treepos] is ptree + """ + if self is root: + return [()] + else: + return [ + treepos + (index,) + for parent in self._parents + for treepos in parent.treepositions(root) + for (index, child) in enumerate(parent) + if child is self + ] + + # ///////////////////////////////////////////////////////////////// + # Parent Management + # ///////////////////////////////////////////////////////////////// + + def _delparent(self, child, index): + # Sanity checks + assert isinstance(child, MultiParentedTree) + assert self[index] is child + assert len([p for p in child._parents if p is self]) == 1 + + # If the only copy of child in self is at index, then delete + # self from child's parent list. + for i, c in enumerate(self): + if c is child and i != index: + break + else: + child._parents.remove(self) + + def _setparent(self, child, index, dry_run=False): + # If the child's type is incorrect, then complain. + if not isinstance(child, MultiParentedTree): + raise TypeError( + "Can not insert a non-MultiParentedTree into a MultiParentedTree" + ) + + # Add self as a parent pointer if it's not already listed. + if not dry_run: + for parent in child._parents: + if parent is self: + break + else: + child._parents.append(self) + + +__all__ = [ + "ParentedTree", + "MultiParentedTree", +] diff --git a/nltk/tree/parsing.py b/nltk/tree/parsing.py new file mode 100644 index 0000000000..0626622336 --- /dev/null +++ b/nltk/tree/parsing.py @@ -0,0 +1,66 @@ +# Natural Language Toolkit: Text Trees +# +# Copyright (C) 2001-2021 NLTK Project +# Author: Edward Loper +# Steven Bird +# Peter Ljunglöf +# Tom Aarsen <> +# URL: +# For license information, see LICENSE.TXT + +import re + +from nltk.tree.tree import Tree + +###################################################################### +## Parsing +###################################################################### + + +def bracket_parse(s): + """ + Use Tree.read(s, remove_empty_top_bracketing=True) instead. + """ + raise NameError("Use Tree.read(s, remove_empty_top_bracketing=True) instead.") + + +def sinica_parse(s): + """ + Parse a Sinica Treebank string and return a tree. Trees are represented as nested brackettings, + as shown in the following example (X represents a Chinese character): + S(goal:NP(Head:Nep:XX)|theme:NP(Head:Nhaa:X)|quantity:Dab:X|Head:VL2:X)#0(PERIODCATEGORY) + + :return: A tree corresponding to the string representation. + :rtype: Tree + :param s: The string to be converted + :type s: str + """ + tokens = re.split(r"([()| ])", s) + for i in range(len(tokens)): + if tokens[i] == "(": + tokens[i - 1], tokens[i] = ( + tokens[i], + tokens[i - 1], + ) # pull nonterminal inside parens + elif ":" in tokens[i]: + fields = tokens[i].split(":") + if len(fields) == 2: # non-terminal + tokens[i] = fields[1] + else: + tokens[i] = "(" + fields[-2] + " " + fields[-1] + ")" + elif tokens[i] == "|": + tokens[i] = "" + + treebank_string = " ".join(tokens) + return Tree.fromstring(treebank_string, remove_empty_top_bracketing=True) + + +# s = re.sub(r'^#[^\s]*\s', '', s) # remove leading identifier +# s = re.sub(r'\w+:', '', s) # remove role tags + +# return s + +__all__ = [ + "bracket_parse", + "sinica_parse", +] diff --git a/nltk/tree/prettyprinter.py b/nltk/tree/prettyprinter.py new file mode 100644 index 0000000000..2ce106e055 --- /dev/null +++ b/nltk/tree/prettyprinter.py @@ -0,0 +1,628 @@ +# Natural Language Toolkit: ASCII visualization of NLTK trees +# +# Copyright (C) 2001-2021 NLTK Project +# Author: Andreas van Cranenburgh +# Peter Ljunglöf +# URL: +# For license information, see LICENSE.TXT + +""" +Pretty-printing of discontinuous trees. +Adapted from the disco-dop project, by Andreas van Cranenburgh. +https://github.com/andreasvc/disco-dop + +Interesting reference (not used for this code): +T. Eschbach et al., Orth. Hypergraph Drawing, Journal of +Graph Algorithms and Applications, 10(2) 141--157 (2006)149. +https://jgaa.info/accepted/2006/EschbachGuentherBecker2006.10.2.pdf +""" + +import re + +try: + from html import escape +except ImportError: + from cgi import escape + +from collections import defaultdict +from operator import itemgetter + +from nltk.tree.tree import Tree +from nltk.util import OrderedDict + +ANSICOLOR = { + "black": 30, + "red": 31, + "green": 32, + "yellow": 33, + "blue": 34, + "magenta": 35, + "cyan": 36, + "white": 37, +} + + +class TreePrettyPrinter: + """ + Pretty-print a tree in text format, either as ASCII or Unicode. + The tree can be a normal tree, or discontinuous. + + ``TreePrettyPrinter(tree, sentence=None, highlight=())`` + creates an object from which different visualizations can be created. + + :param tree: a Tree object. + :param sentence: a list of words (strings). If `sentence` is given, + `tree` must contain integers as leaves, which are taken as indices + in `sentence`. Using this you can display a discontinuous tree. + :param highlight: Optionally, a sequence of Tree objects in `tree` which + should be highlighted. Has the effect of only applying colors to nodes + in this sequence (nodes should be given as Tree objects, terminals as + indices). + + >>> from nltk.tree import Tree + >>> tree = Tree.fromstring('(S (NP Mary) (VP walks))') + >>> print(TreePrettyPrinter(tree).text()) + ... # doctest: +NORMALIZE_WHITESPACE + S + ____|____ + NP VP + | | + Mary walks + """ + + def __init__(self, tree, sentence=None, highlight=()): + if sentence is None: + leaves = tree.leaves() + if ( + leaves + and not any(len(a) == 0 for a in tree.subtrees()) + and all(isinstance(a, int) for a in leaves) + ): + sentence = [str(a) for a in leaves] + else: + # this deals with empty nodes (frontier non-terminals) + # and multiple/mixed terminals under non-terminals. + tree = tree.copy(True) + sentence = [] + for a in tree.subtrees(): + if len(a) == 0: + a.append(len(sentence)) + sentence.append(None) + elif any(not isinstance(b, Tree) for b in a): + for n, b in enumerate(a): + if not isinstance(b, Tree): + a[n] = len(sentence) + if type(b) == tuple: + b = "/".join(b) + sentence.append("%s" % b) + self.nodes, self.coords, self.edges, self.highlight = self.nodecoords( + tree, sentence, highlight + ) + + def __str__(self): + return self.text() + + def __repr__(self): + return "" % len(self.nodes) + + @staticmethod + def nodecoords(tree, sentence, highlight): + """ + Produce coordinates of nodes on a grid. + + Objective: + + - Produce coordinates for a non-overlapping placement of nodes and + horizontal lines. + - Order edges so that crossing edges cross a minimal number of previous + horizontal lines (never vertical lines). + + Approach: + + - bottom up level order traversal (start at terminals) + - at each level, identify nodes which cannot be on the same row + - identify nodes which cannot be in the same column + - place nodes into a grid at (row, column) + - order child-parent edges with crossing edges last + + Coordinates are (row, column); the origin (0, 0) is at the top left; + the root node is on row 0. Coordinates do not consider the size of a + node (which depends on font, &c), so the width of a column of the grid + should be automatically determined by the element with the greatest + width in that column. Alternatively, the integer coordinates could be + converted to coordinates in which the distances between adjacent nodes + are non-uniform. + + Produces tuple (nodes, coords, edges, highlighted) where: + + - nodes[id]: Tree object for the node with this integer id + - coords[id]: (n, m) coordinate where to draw node with id in the grid + - edges[id]: parent id of node with this id (ordered dictionary) + - highlighted: set of ids that should be highlighted + """ + + def findcell(m, matrix, startoflevel, children): + """ + Find vacant row, column index for node ``m``. + Iterate over current rows for this level (try lowest first) + and look for cell between first and last child of this node, + add new row to level if no free row available. + """ + candidates = [a for _, a in children[m]] + minidx, maxidx = min(candidates), max(candidates) + leaves = tree[m].leaves() + center = scale * sum(leaves) // len(leaves) # center of gravity + if minidx < maxidx and not minidx < center < maxidx: + center = sum(candidates) // len(candidates) + if max(candidates) - min(candidates) > 2 * scale: + center -= center % scale # round to unscaled coordinate + if minidx < maxidx and not minidx < center < maxidx: + center += scale + if ids[m] == 0: + startoflevel = len(matrix) + for rowidx in range(startoflevel, len(matrix) + 1): + if rowidx == len(matrix): # need to add a new row + matrix.append( + [ + vertline if a not in (corner, None) else None + for a in matrix[-1] + ] + ) + row = matrix[rowidx] + i = j = center + if len(children[m]) == 1: # place unaries directly above child + return rowidx, next(iter(children[m]))[1] + elif all( + a is None or a == vertline + for a in row[min(candidates) : max(candidates) + 1] + ): + # find free column + for n in range(scale): + i = j = center + n + while j > minidx or i < maxidx: + if i < maxidx and ( + matrix[rowidx][i] is None or i in candidates + ): + return rowidx, i + elif j > minidx and ( + matrix[rowidx][j] is None or j in candidates + ): + return rowidx, j + i += scale + j -= scale + raise ValueError( + "could not find a free cell for:\n%s\n%s" + "min=%d; max=%d" % (tree[m], minidx, maxidx, dumpmatrix()) + ) + + def dumpmatrix(): + """Dump matrix contents for debugging purposes.""" + return "\n".join( + "%2d: %s" % (n, " ".join(("%2r" % i)[:2] for i in row)) + for n, row in enumerate(matrix) + ) + + leaves = tree.leaves() + if not all(isinstance(n, int) for n in leaves): + raise ValueError("All leaves must be integer indices.") + if len(leaves) != len(set(leaves)): + raise ValueError("Indices must occur at most once.") + if not all(0 <= n < len(sentence) for n in leaves): + raise ValueError( + "All leaves must be in the interval 0..n " + "with n=len(sentence)\ntokens: %d indices: " + "%r\nsentence: %s" % (len(sentence), tree.leaves(), sentence) + ) + vertline, corner = -1, -2 # constants + tree = tree.copy(True) + for a in tree.subtrees(): + a.sort(key=lambda n: min(n.leaves()) if isinstance(n, Tree) else n) + scale = 2 + crossed = set() + # internal nodes and lexical nodes (no frontiers) + positions = tree.treepositions() + maxdepth = max(map(len, positions)) + 1 + childcols = defaultdict(set) + matrix = [[None] * (len(sentence) * scale)] + nodes = {} + ids = {a: n for n, a in enumerate(positions)} + highlighted_nodes = { + n for a, n in ids.items() if not highlight or tree[a] in highlight + } + levels = {n: [] for n in range(maxdepth - 1)} + terminals = [] + for a in positions: + node = tree[a] + if isinstance(node, Tree): + levels[maxdepth - node.height()].append(a) + else: + terminals.append(a) + + for n in levels: + levels[n].sort(key=lambda n: max(tree[n].leaves()) - min(tree[n].leaves())) + terminals.sort() + positions = set(positions) + + for m in terminals: + i = int(tree[m]) * scale + assert matrix[0][i] is None, (matrix[0][i], m, i) + matrix[0][i] = ids[m] + nodes[ids[m]] = sentence[tree[m]] + if nodes[ids[m]] is None: + nodes[ids[m]] = "..." + highlighted_nodes.discard(ids[m]) + positions.remove(m) + childcols[m[:-1]].add((0, i)) + + # add other nodes centered on their children, + # if the center is already taken, back off + # to the left and right alternately, until an empty cell is found. + for n in sorted(levels, reverse=True): + nodesatdepth = levels[n] + startoflevel = len(matrix) + matrix.append( + [vertline if a not in (corner, None) else None for a in matrix[-1]] + ) + for m in nodesatdepth: # [::-1]: + if n < maxdepth - 1 and childcols[m]: + _, pivot = min(childcols[m], key=itemgetter(1)) + if { + a[:-1] + for row in matrix[:-1] + for a in row[:pivot] + if isinstance(a, tuple) + } & { + a[:-1] + for row in matrix[:-1] + for a in row[pivot:] + if isinstance(a, tuple) + }: + crossed.add(m) + + rowidx, i = findcell(m, matrix, startoflevel, childcols) + positions.remove(m) + + # block positions where children of this node branch out + for _, x in childcols[m]: + matrix[rowidx][x] = corner + # assert m == () or matrix[rowidx][i] in (None, corner), ( + # matrix[rowidx][i], m, str(tree), ' '.join(sentence)) + # node itself + matrix[rowidx][i] = ids[m] + nodes[ids[m]] = tree[m] + # add column to the set of children for its parent + if m != (): + childcols[m[:-1]].add((rowidx, i)) + assert len(positions) == 0 + + # remove unused columns, right to left + for m in range(scale * len(sentence) - 1, -1, -1): + if not any(isinstance(row[m], (Tree, int)) for row in matrix): + for row in matrix: + del row[m] + + # remove unused rows, reverse + matrix = [ + row + for row in reversed(matrix) + if not all(a is None or a == vertline for a in row) + ] + + # collect coordinates of nodes + coords = {} + for n, _ in enumerate(matrix): + for m, i in enumerate(matrix[n]): + if isinstance(i, int) and i >= 0: + coords[i] = n, m + + # move crossed edges last + positions = sorted( + (a for level in levels.values() for a in level), + key=lambda a: a[:-1] in crossed, + ) + + # collect edges from node to node + edges = OrderedDict() + for i in reversed(positions): + for j, _ in enumerate(tree[i]): + edges[ids[i + (j,)]] = ids[i] + + return nodes, coords, edges, highlighted_nodes + + def text( + self, + nodedist=1, + unicodelines=False, + html=False, + ansi=False, + nodecolor="blue", + leafcolor="red", + funccolor="green", + abbreviate=None, + maxwidth=16, + ): + """ + :return: ASCII art for a discontinuous tree. + + :param unicodelines: whether to use Unicode line drawing characters + instead of plain (7-bit) ASCII. + :param html: whether to wrap output in html code (default plain text). + :param ansi: whether to produce colors with ANSI escape sequences + (only effective when html==False). + :param leafcolor, nodecolor: specify colors of leaves and phrasal + nodes; effective when either html or ansi is True. + :param abbreviate: if True, abbreviate labels longer than 5 characters. + If integer, abbreviate labels longer than `abbr` characters. + :param maxwidth: maximum number of characters before a label starts to + wrap; pass None to disable. + """ + if abbreviate == True: + abbreviate = 5 + if unicodelines: + horzline = "\u2500" + leftcorner = "\u250c" + rightcorner = "\u2510" + vertline = " \u2502 " + tee = horzline + "\u252C" + horzline + bottom = horzline + "\u2534" + horzline + cross = horzline + "\u253c" + horzline + ellipsis = "\u2026" + else: + horzline = "_" + leftcorner = rightcorner = " " + vertline = " | " + tee = 3 * horzline + cross = bottom = "_|_" + ellipsis = "." + + def crosscell(cur, x=vertline): + """Overwrite center of this cell with a vertical branch.""" + splitl = len(cur) - len(cur) // 2 - len(x) // 2 - 1 + lst = list(cur) + lst[splitl : splitl + len(x)] = list(x) + return "".join(lst) + + result = [] + matrix = defaultdict(dict) + maxnodewith = defaultdict(lambda: 3) + maxnodeheight = defaultdict(lambda: 1) + maxcol = 0 + minchildcol = {} + maxchildcol = {} + childcols = defaultdict(set) + labels = {} + wrapre = re.compile( + "(.{%d,%d}\\b\\W*|.{%d})" % (maxwidth - 4, maxwidth, maxwidth) + ) + # collect labels and coordinates + for a in self.nodes: + row, column = self.coords[a] + matrix[row][column] = a + maxcol = max(maxcol, column) + label = ( + self.nodes[a].label() + if isinstance(self.nodes[a], Tree) + else self.nodes[a] + ) + if abbreviate and len(label) > abbreviate: + label = label[:abbreviate] + ellipsis + if maxwidth and len(label) > maxwidth: + label = wrapre.sub(r"\1\n", label).strip() + label = label.split("\n") + maxnodeheight[row] = max(maxnodeheight[row], len(label)) + maxnodewith[column] = max(maxnodewith[column], max(map(len, label))) + labels[a] = label + if a not in self.edges: + continue # e.g., root + parent = self.edges[a] + childcols[parent].add((row, column)) + minchildcol[parent] = min(minchildcol.get(parent, column), column) + maxchildcol[parent] = max(maxchildcol.get(parent, column), column) + # bottom up level order traversal + for row in sorted(matrix, reverse=True): + noderows = [ + ["".center(maxnodewith[col]) for col in range(maxcol + 1)] + for _ in range(maxnodeheight[row]) + ] + branchrow = ["".center(maxnodewith[col]) for col in range(maxcol + 1)] + for col in matrix[row]: + n = matrix[row][col] + node = self.nodes[n] + text = labels[n] + if isinstance(node, Tree): + # draw horizontal branch towards children for this node + if n in minchildcol and minchildcol[n] < maxchildcol[n]: + i, j = minchildcol[n], maxchildcol[n] + a, b = (maxnodewith[i] + 1) // 2 - 1, maxnodewith[j] // 2 + branchrow[i] = ((" " * a) + leftcorner).ljust( + maxnodewith[i], horzline + ) + branchrow[j] = (rightcorner + (" " * b)).rjust( + maxnodewith[j], horzline + ) + for i in range(minchildcol[n] + 1, maxchildcol[n]): + if i == col and any(a == i for _, a in childcols[n]): + line = cross + elif i == col: + line = bottom + elif any(a == i for _, a in childcols[n]): + line = tee + else: + line = horzline + branchrow[i] = line.center(maxnodewith[i], horzline) + else: # if n and n in minchildcol: + branchrow[col] = crosscell(branchrow[col]) + text = [a.center(maxnodewith[col]) for a in text] + color = nodecolor if isinstance(node, Tree) else leafcolor + if isinstance(node, Tree) and node.label().startswith("-"): + color = funccolor + if html: + text = [escape(a, quote=False) for a in text] + if n in self.highlight: + text = [f"{a}" for a in text] + elif ansi and n in self.highlight: + text = ["\x1b[%d;1m%s\x1b[0m" % (ANSICOLOR[color], a) for a in text] + for x in range(maxnodeheight[row]): + # draw vertical lines in partially filled multiline node + # labels, but only if it's not a frontier node. + noderows[x][col] = ( + text[x] + if x < len(text) + else (vertline if childcols[n] else " ").center( + maxnodewith[col], " " + ) + ) + # for each column, if there is a node below us which has a parent + # above us, draw a vertical branch in that column. + if row != max(matrix): + for n, (childrow, col) in self.coords.items(): + if n > 0 and self.coords[self.edges[n]][0] < row < childrow: + branchrow[col] = crosscell(branchrow[col]) + if col not in matrix[row]: + for noderow in noderows: + noderow[col] = crosscell(noderow[col]) + branchrow = [ + a + ((a[-1] if a[-1] != " " else b[0]) * nodedist) + for a, b in zip(branchrow, branchrow[1:] + [" "]) + ] + result.append("".join(branchrow)) + result.extend( + (" " * nodedist).join(noderow) for noderow in reversed(noderows) + ) + return "\n".join(reversed(result)) + "\n" + + def svg(self, nodecolor="blue", leafcolor="red", funccolor="green"): + """ + :return: SVG representation of a tree. + """ + fontsize = 12 + hscale = 40 + vscale = 25 + hstart = vstart = 20 + width = max(col for _, col in self.coords.values()) + height = max(row for row, _ in self.coords.values()) + result = [ + '' + % ( + width * 3, + height * 2.5, + -hstart, + -vstart, + width * hscale + 3 * hstart, + height * vscale + 3 * vstart, + ) + ] + + children = defaultdict(set) + for n in self.nodes: + if n: + children[self.edges[n]].add(n) + + # horizontal branches from nodes to children + for node in self.nodes: + if not children[node]: + continue + y, x = self.coords[node] + x *= hscale + y *= vscale + x += hstart + y += vstart + fontsize // 2 + childx = [self.coords[c][1] for c in children[node]] + xmin = hstart + hscale * min(childx) + xmax = hstart + hscale * max(childx) + result.append( + '\t' % (xmin, y, xmax, y) + ) + result.append( + '\t' % (x, y, x, y - fontsize // 3) + ) + + # vertical branches from children to parents + for child, parent in self.edges.items(): + y, _ = self.coords[parent] + y *= vscale + y += vstart + fontsize // 2 + childy, childx = self.coords[child] + childx *= hscale + childy *= vscale + childx += hstart + childy += vstart - fontsize + result += [ + '\t' % (childx, childy, childx, y + 5), + '\t' % (childx, childy, childx, y), + ] + + # write nodes with coordinates + for n, (row, column) in self.coords.items(): + node = self.nodes[n] + x = column * hscale + hstart + y = row * vscale + vstart + if n in self.highlight: + color = nodecolor if isinstance(node, Tree) else leafcolor + if isinstance(node, Tree) and node.label().startswith("-"): + color = funccolor + else: + color = "black" + result += [ + '\t%s' + % ( + color, + fontsize, + x, + y, + escape( + node.label() if isinstance(node, Tree) else node, quote=False + ), + ) + ] + + result += [""] + return "\n".join(result) + + +def test(): + """Do some tree drawing tests.""" + + def print_tree(n, tree, sentence=None, ansi=True, **xargs): + print() + print('{}: "{}"'.format(n, " ".join(sentence or tree.leaves()))) + print(tree) + print() + drawtree = TreePrettyPrinter(tree, sentence) + try: + print(drawtree.text(unicodelines=ansi, ansi=ansi, **xargs)) + except (UnicodeDecodeError, UnicodeEncodeError): + print(drawtree.text(unicodelines=False, ansi=False, **xargs)) + + from nltk.corpus import treebank + + for n in [0, 1440, 1591, 2771, 2170]: + tree = treebank.parsed_sents()[n] + print_tree(n, tree, nodedist=2, maxwidth=8) + print() + print("ASCII version:") + print(TreePrettyPrinter(tree).text(nodedist=2)) + + tree = Tree.fromstring( + "(top (punct 8) (smain (noun 0) (verb 1) (inf (verb 5) (inf (verb 6) " + "(conj (inf (pp (prep 2) (np (det 3) (noun 4))) (verb 7)) (inf (verb 9)) " + "(vg 10) (inf (verb 11)))))) (punct 12))", + read_leaf=int, + ) + sentence = ( + "Ze had met haar moeder kunnen gaan winkelen ," + " zwemmen of terrassen .".split() + ) + print_tree("Discontinuous tree", tree, sentence, nodedist=2) + + +__all__ = ["TreePrettyPrinter"] + +if __name__ == "__main__": + test() diff --git a/nltk/tree/probabilistic.py b/nltk/tree/probabilistic.py new file mode 100644 index 0000000000..1d7384c80e --- /dev/null +++ b/nltk/tree/probabilistic.py @@ -0,0 +1,78 @@ +# Natural Language Toolkit: Text Trees +# +# Copyright (C) 2001-2021 NLTK Project +# Author: Edward Loper +# Steven Bird +# Peter Ljunglöf +# Tom Aarsen <> +# URL: +# For license information, see LICENSE.TXT + + +from nltk.internals import raise_unorderable_types +from nltk.probability import ProbabilisticMixIn +from nltk.tree.immutable import ImmutableProbabilisticTree +from nltk.tree.tree import Tree + +###################################################################### +## Probabilistic trees +###################################################################### + + +class ProbabilisticTree(Tree, ProbabilisticMixIn): + def __init__(self, node, children=None, **prob_kwargs): + Tree.__init__(self, node, children) + ProbabilisticMixIn.__init__(self, **prob_kwargs) + + # We have to patch up these methods to make them work right: + def _frozen_class(self): + return ImmutableProbabilisticTree + + def __repr__(self): + return f"{Tree.__repr__(self)} (p={self.prob()!r})" + + def __str__(self): + return f"{self.pformat(margin=60)} (p={self.prob():.6g})" + + def copy(self, deep=False): + if not deep: + return type(self)(self._label, self, prob=self.prob()) + else: + return type(self).convert(self) + + @classmethod + def convert(cls, val): + if isinstance(val, Tree): + children = [cls.convert(child) for child in val] + if isinstance(val, ProbabilisticMixIn): + return cls(val._label, children, prob=val.prob()) + else: + return cls(val._label, children, prob=1.0) + else: + return val + + def __eq__(self, other): + return ( + self.__class__ is other.__class__ + and ( + self._label, + list(self), + self.prob(), + ) + == (other._label, list(other), other.prob()) + ) + + def __lt__(self, other): + if not isinstance(other, Tree): + raise_unorderable_types("<", self, other) + if self.__class__ is other.__class__: + return (self._label, list(self), self.prob()) < ( + other._label, + list(other), + other.prob(), + ) + else: + return self.__class__.__name__ < other.__class__.__name__ + + +__all__ = ["ProbabilisticTree"] diff --git a/nltk/tree/transforms.py b/nltk/tree/transforms.py new file mode 100644 index 0000000000..2ebaa4dbfc --- /dev/null +++ b/nltk/tree/transforms.py @@ -0,0 +1,338 @@ +# Natural Language Toolkit: Tree Transformations +# +# Copyright (C) 2005-2007 Oregon Graduate Institute +# Author: Nathan Bodenstab +# URL: +# For license information, see LICENSE.TXT + +r""" +A collection of methods for tree (grammar) transformations used +in parsing natural language. + +Although many of these methods are technically grammar transformations +(ie. Chomsky Norm Form), when working with treebanks it is much more +natural to visualize these modifications in a tree structure. Hence, +we will do all transformation directly to the tree itself. +Transforming the tree directly also allows us to do parent annotation. +A grammar can then be simply induced from the modified tree. + +The following is a short tutorial on the available transformations. + + 1. Chomsky Normal Form (binarization) + + It is well known that any grammar has a Chomsky Normal Form (CNF) + equivalent grammar where CNF is defined by every production having + either two non-terminals or one terminal on its right hand side. + When we have hierarchically structured data (ie. a treebank), it is + natural to view this in terms of productions where the root of every + subtree is the head (left hand side) of the production and all of + its children are the right hand side constituents. In order to + convert a tree into CNF, we simply need to ensure that every subtree + has either two subtrees as children (binarization), or one leaf node + (non-terminal). In order to binarize a subtree with more than two + children, we must introduce artificial nodes. + + There are two popular methods to convert a tree into CNF: left + factoring and right factoring. The following example demonstrates + the difference between them. Example:: + + Original Right-Factored Left-Factored + + A A A + / | \ / \ / \ + B C D ==> B A| OR A| D + / \ / \ + C D B C + + 2. Parent Annotation + + In addition to binarizing the tree, there are two standard + modifications to node labels we can do in the same traversal: parent + annotation and Markov order-N smoothing (or sibling smoothing). + + The purpose of parent annotation is to refine the probabilities of + productions by adding a small amount of context. With this simple + addition, a CYK (inside-outside, dynamic programming chart parse) + can improve from 74% to 79% accuracy. A natural generalization from + parent annotation is to grandparent annotation and beyond. The + tradeoff becomes accuracy gain vs. computational complexity. We + must also keep in mind data sparcity issues. Example:: + + Original Parent Annotation + + A A^ + / | \ / \ + B C D ==> B^ A|^ where ? is the + / \ parent of A + C^ D^ + + + 3. Markov order-N smoothing + + Markov smoothing combats data sparcity issues as well as decreasing + computational requirements by limiting the number of children + included in artificial nodes. In practice, most people use an order + 2 grammar. Example:: + + Original No Smoothing Markov order 1 Markov order 2 etc. + + __A__ A A A + / /|\ \ / \ / \ / \ + B C D E F ==> B A| ==> B A| ==> B A| + / \ / \ / \ + C ... C ... C ... + + + + Annotation decisions can be thought about in the vertical direction + (parent, grandparent, etc) and the horizontal direction (number of + siblings to keep). Parameters to the following functions specify + these values. For more information see: + + Dan Klein and Chris Manning (2003) "Accurate Unlexicalized + Parsing", ACL-03. https://www.aclweb.org/anthology/P03-1054 + + 4. Unary Collapsing + + Collapse unary productions (ie. subtrees with a single child) into a + new non-terminal (Tree node). This is useful when working with + algorithms that do not allow unary productions, yet you do not wish + to lose the parent information. Example:: + + A + | + B ==> A+B + / \ / \ + C D C D + +""" + +from nltk.tree.tree import Tree + + +def chomsky_normal_form( + tree, factor="right", horzMarkov=None, vertMarkov=0, childChar="|", parentChar="^" +): + # assume all subtrees have homogeneous children + # assume all terminals have no siblings + + # A semi-hack to have elegant looking code below. As a result, + # any subtree with a branching factor greater than 999 will be incorrectly truncated. + if horzMarkov is None: + horzMarkov = 999 + + # Traverse the tree depth-first keeping a list of ancestor nodes to the root. + # I chose not to use the tree.treepositions() method since it requires + # two traversals of the tree (one to get the positions, one to iterate + # over them) and node access time is proportional to the height of the node. + # This method is 7x faster which helps when parsing 40,000 sentences. + + nodeList = [(tree, [tree.label()])] + while nodeList != []: + node, parent = nodeList.pop() + if isinstance(node, Tree): + + # parent annotation + parentString = "" + originalNode = node.label() + if vertMarkov != 0 and node != tree and isinstance(node[0], Tree): + parentString = "{}<{}>".format(parentChar, "-".join(parent)) + node.set_label(node.label() + parentString) + parent = [originalNode] + parent[: vertMarkov - 1] + + # add children to the agenda before we mess with them + for child in node: + nodeList.append((child, parent)) + + # chomsky normal form factorization + if len(node) > 2: + childNodes = [child.label() for child in node] + nodeCopy = node.copy() + node[0:] = [] # delete the children + + curNode = node + numChildren = len(nodeCopy) + for i in range(1, numChildren - 1): + if factor == "right": + newHead = "{}{}<{}>{}".format( + originalNode, + childChar, + "-".join( + childNodes[i : min([i + horzMarkov, numChildren])] + ), + parentString, + ) # create new head + newNode = Tree(newHead, []) + curNode[0:] = [nodeCopy.pop(0), newNode] + else: + newHead = "{}{}<{}>{}".format( + originalNode, + childChar, + "-".join( + childNodes[max([numChildren - i - horzMarkov, 0]) : -i] + ), + parentString, + ) + newNode = Tree(newHead, []) + curNode[0:] = [newNode, nodeCopy.pop()] + + curNode = newNode + + curNode[0:] = [child for child in nodeCopy] + + +def un_chomsky_normal_form( + tree, expandUnary=True, childChar="|", parentChar="^", unaryChar="+" +): + # Traverse the tree-depth first keeping a pointer to the parent for modification purposes. + nodeList = [(tree, [])] + while nodeList != []: + node, parent = nodeList.pop() + if isinstance(node, Tree): + # if the node contains the 'childChar' character it means that + # it is an artificial node and can be removed, although we still need + # to move its children to its parent + childIndex = node.label().find(childChar) + if childIndex != -1: + nodeIndex = parent.index(node) + parent.remove(parent[nodeIndex]) + # Generated node was on the left if the nodeIndex is 0 which + # means the grammar was left factored. We must insert the children + # at the beginning of the parent's children + if nodeIndex == 0: + parent.insert(0, node[0]) + parent.insert(1, node[1]) + else: + parent.extend([node[0], node[1]]) + + # parent is now the current node so the children of parent will be added to the agenda + node = parent + else: + parentIndex = node.label().find(parentChar) + if parentIndex != -1: + # strip the node name of the parent annotation + node.set_label(node.label()[:parentIndex]) + + # expand collapsed unary productions + if expandUnary == True: + unaryIndex = node.label().find(unaryChar) + if unaryIndex != -1: + newNode = Tree( + node.label()[unaryIndex + 1 :], [i for i in node] + ) + node.set_label(node.label()[:unaryIndex]) + node[0:] = [newNode] + + for child in node: + nodeList.append((child, node)) + + +def collapse_unary(tree, collapsePOS=False, collapseRoot=False, joinChar="+"): + """ + Collapse subtrees with a single child (ie. unary productions) + into a new non-terminal (Tree node) joined by 'joinChar'. + This is useful when working with algorithms that do not allow + unary productions, and completely removing the unary productions + would require loss of useful information. The Tree is modified + directly (since it is passed by reference) and no value is returned. + + :param tree: The Tree to be collapsed + :type tree: Tree + :param collapsePOS: 'False' (default) will not collapse the parent of leaf nodes (ie. + Part-of-Speech tags) since they are always unary productions + :type collapsePOS: bool + :param collapseRoot: 'False' (default) will not modify the root production + if it is unary. For the Penn WSJ treebank corpus, this corresponds + to the TOP -> productions. + :type collapseRoot: bool + :param joinChar: A string used to connect collapsed node values (default = "+") + :type joinChar: str + """ + + if collapseRoot == False and isinstance(tree, Tree) and len(tree) == 1: + nodeList = [tree[0]] + else: + nodeList = [tree] + + # depth-first traversal of tree + while nodeList != []: + node = nodeList.pop() + if isinstance(node, Tree): + if ( + len(node) == 1 + and isinstance(node[0], Tree) + and (collapsePOS == True or isinstance(node[0, 0], Tree)) + ): + node.set_label(node.label() + joinChar + node[0].label()) + node[0:] = [child for child in node[0]] + # since we assigned the child's children to the current node, + # evaluate the current node again + nodeList.append(node) + else: + for child in node: + nodeList.append(child) + + +################################################################# +# Demonstration +################################################################# + + +def demo(): + """ + A demonstration showing how each tree transform can be used. + """ + + from copy import deepcopy + + from nltk.draw.tree import draw_trees + from nltk.tree.tree import Tree + + # original tree from WSJ bracketed text + sentence = """(TOP + (S + (S + (VP + (VBN Turned) + (ADVP (RB loose)) + (PP + (IN in) + (NP + (NP (NNP Shane) (NNP Longman) (POS 's)) + (NN trading) + (NN room))))) + (, ,) + (NP (DT the) (NN yuppie) (NNS dealers)) + (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right)))) + (. .)))""" + t = Tree.fromstring(sentence, remove_empty_top_bracketing=True) + + # collapse subtrees with only one child + collapsedTree = deepcopy(t) + collapse_unary(collapsedTree) + + # convert the tree to CNF + cnfTree = deepcopy(collapsedTree) + chomsky_normal_form(cnfTree) + + # convert the tree to CNF with parent annotation (one level) and horizontal smoothing of order two + parentTree = deepcopy(collapsedTree) + chomsky_normal_form(parentTree, horzMarkov=2, vertMarkov=1) + + # convert the tree back to its original form (used to make CYK results comparable) + original = deepcopy(parentTree) + un_chomsky_normal_form(original) + + # convert tree back to bracketed text + sentence2 = original.pprint() + print(sentence) + print(sentence2) + print("Sentences the same? ", sentence == sentence2) + + draw_trees(t, collapsedTree, cnfTree, parentTree, original) + + +if __name__ == "__main__": + demo() + +__all__ = ["chomsky_normal_form", "un_chomsky_normal_form", "collapse_unary"] diff --git a/nltk/tree.py b/nltk/tree/tree.py similarity index 52% rename from nltk/tree.py rename to nltk/tree/tree.py index c9acd7d8f2..6c983c4c6f 100644 --- a/nltk/tree.py +++ b/nltk/tree/tree.py @@ -16,14 +16,8 @@ import re import sys -from abc import ABCMeta, abstractmethod from nltk.grammar import Nonterminal, Production -from nltk.internals import raise_unorderable_types -from nltk.probability import ProbabilisticMixIn -from nltk.util import slice_bounds - -# TODO: add LabelledTree (can be used for dependency trees) ###################################################################### ## Trees @@ -362,7 +356,7 @@ def productions(self): form P -> C1 C2 ... Cn. >>> t = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))") - >>> t.productions() + >>> t.productions() # doctest: +NORMALIZE_WHITESPACE [S -> NP VP, NP -> D N, D -> 'the', N -> 'dog', VP -> V NP, V -> 'chased', NP -> D N, D -> 'the', N -> 'cat'] @@ -479,7 +473,7 @@ def chomsky_normal_form( :param parentChar: A string used to separate the node representation from its vertical annotation :type parentChar: str """ - from nltk.treetransforms import chomsky_normal_form + from nltk.tree.transforms import chomsky_normal_form chomsky_normal_form(self, factor, horzMarkov, vertMarkov, childChar, parentChar) @@ -504,7 +498,7 @@ def un_chomsky_normal_form( :param unaryChar: A string joining two non-terminals in a unary production (default = "+") :type unaryChar: str """ - from nltk.treetransforms import un_chomsky_normal_form + from nltk.tree.transforms import un_chomsky_normal_form un_chomsky_normal_form(self, expandUnary, childChar, parentChar, unaryChar) @@ -527,7 +521,7 @@ def collapse_unary(self, collapsePOS=False, collapseRoot=False, joinChar="+"): :param joinChar: A string used to connect collapsed node values (default = "+") :type joinChar: str """ - from nltk.treetransforms import collapse_unary + from nltk.tree.transforms import collapse_unary collapse_unary(self, collapsePOS, collapseRoot, joinChar) @@ -564,6 +558,8 @@ def copy(self, deep=False): return type(self).convert(self) def _frozen_class(self): + from nltk.tree.immutable import ImmutableTree + return ImmutableTree def freeze(self, leaf_freezer=None): @@ -768,9 +764,9 @@ def pretty_print(self, sentence=None, highlight=(), stream=None, **kwargs): """ Pretty-print this tree as ASCII or Unicode art. For explanation of the arguments, see the documentation for - `nltk.treeprettyprinter.TreePrettyPrinter`. + `nltk.tree.prettyprinter.TreePrettyPrinter`. """ - from nltk.treeprettyprinter import TreePrettyPrinter + from nltk.tree.prettyprinter import TreePrettyPrinter print(TreePrettyPrinter(self, sentence, highlight).text(**kwargs), file=stream) @@ -782,61 +778,10 @@ def __repr__(self): childstr, ) - def _repr_png_(self): - """ - Draws and outputs in PNG for ipython. - PNG is used instead of PDF, since it can be displayed in the qt console and - has wider browser support. - """ - import base64 - import os - import subprocess - import tempfile - - from nltk.draw.tree import tree_to_treesegment - from nltk.draw.util import CanvasFrame - from nltk.internals import find_binary - - _canvas_frame = CanvasFrame() - widget = tree_to_treesegment(_canvas_frame.canvas(), self) - _canvas_frame.add_widget(widget) - x, y, w, h = widget.bbox() - # print_to_file uses scrollregion to set the width and height of the pdf. - _canvas_frame.canvas()["scrollregion"] = (0, 0, w, h) - with tempfile.NamedTemporaryFile() as file: - in_path = f"{file.name}.ps" - out_path = f"{file.name}.png" - _canvas_frame.print_to_file(in_path) - _canvas_frame.destroy_widget(widget) - try: - subprocess.call( - [ - find_binary( - "gs", - binary_names=["gswin32c.exe", "gswin64c.exe"], - env_vars=["PATH"], - verbose=False, - ) - ] - + "-q -dEPSCrop -sDEVICE=png16m -r90 -dTextAlphaBits=4 -dGraphicsAlphaBits=4 -dSAFER -dBATCH -dNOPAUSE -sOutputFile={} {}".format( - out_path, in_path - ).split() - ) - except LookupError as e: - pre_error_message = str( - "The Ghostscript executable isn't found.\n" - "See https://web.mit.edu/ghostscript/www/Install.htm\n" - "If you're using a Mac, you can try installing\n" - "https://docs.brew.sh/Installation then `brew install ghostscript`" - ) - print(pre_error_message, file=sys.stderr) - raise LookupError from e + def _repr_svg_(self): + from svgling import draw_tree - with open(out_path, "rb") as sr: - res = sr.read() - os.remove(in_path) - os.remove(out_path) - return base64.b64encode(res).decode() + return draw_tree(self)._repr_svg_() def __str__(self): return self.pformat() @@ -945,718 +890,6 @@ def _pformat_flat(self, nodesep, parens, quotes): ) -class ImmutableTree(Tree): - def __init__(self, node, children=None): - super().__init__(node, children) - # Precompute our hash value. This ensures that we're really - # immutable. It also means we only have to calculate it once. - try: - self._hash = hash((self._label, tuple(self))) - except (TypeError, ValueError) as e: - raise ValueError( - "%s: node value and children " "must be immutable" % type(self).__name__ - ) from e - - def __setitem__(self, index, value): - raise ValueError("%s may not be modified" % type(self).__name__) - - def __setslice__(self, i, j, value): - raise ValueError("%s may not be modified" % type(self).__name__) - - def __delitem__(self, index): - raise ValueError("%s may not be modified" % type(self).__name__) - - def __delslice__(self, i, j): - raise ValueError("%s may not be modified" % type(self).__name__) - - def __iadd__(self, other): - raise ValueError("%s may not be modified" % type(self).__name__) - - def __imul__(self, other): - raise ValueError("%s may not be modified" % type(self).__name__) - - def append(self, v): - raise ValueError("%s may not be modified" % type(self).__name__) - - def extend(self, v): - raise ValueError("%s may not be modified" % type(self).__name__) - - def pop(self, v=None): - raise ValueError("%s may not be modified" % type(self).__name__) - - def remove(self, v): - raise ValueError("%s may not be modified" % type(self).__name__) - - def reverse(self): - raise ValueError("%s may not be modified" % type(self).__name__) - - def sort(self): - raise ValueError("%s may not be modified" % type(self).__name__) - - def __hash__(self): - return self._hash - - def set_label(self, value): - """ - Set the node label. This will only succeed the first time the - node label is set, which should occur in ImmutableTree.__init__(). - """ - if hasattr(self, "_label"): - raise ValueError("%s may not be modified" % type(self).__name__) - self._label = value - - -###################################################################### -## Parented trees -###################################################################### -class AbstractParentedTree(Tree, metaclass=ABCMeta): - """ - An abstract base class for a ``Tree`` that automatically maintains - pointers to parent nodes. These parent pointers are updated - whenever any change is made to a tree's structure. Two subclasses - are currently defined: - - - ``ParentedTree`` is used for tree structures where each subtree - has at most one parent. This class should be used in cases - where there is no"sharing" of subtrees. - - - ``MultiParentedTree`` is used for tree structures where a - subtree may have zero or more parents. This class should be - used in cases where subtrees may be shared. - - Subclassing - =========== - The ``AbstractParentedTree`` class redefines all operations that - modify a tree's structure to call two methods, which are used by - subclasses to update parent information: - - - ``_setparent()`` is called whenever a new child is added. - - ``_delparent()`` is called whenever a child is removed. - """ - - def __init__(self, node, children=None): - super().__init__(node, children) - # If children is None, the tree is read from node, and - # all parents will be set during parsing. - if children is not None: - # Otherwise we have to set the parent of the children. - # Iterate over self, and *not* children, because children - # might be an iterator. - for i, child in enumerate(self): - if isinstance(child, Tree): - self._setparent(child, i, dry_run=True) - for i, child in enumerate(self): - if isinstance(child, Tree): - self._setparent(child, i) - - # //////////////////////////////////////////////////////////// - # Parent management - # //////////////////////////////////////////////////////////// - @abstractmethod - def _setparent(self, child, index, dry_run=False): - """ - Update the parent pointer of ``child`` to point to ``self``. This - method is only called if the type of ``child`` is ``Tree``; - i.e., it is not called when adding a leaf to a tree. This method - is always called before the child is actually added to the - child list of ``self``. - - :type child: Tree - :type index: int - :param index: The index of ``child`` in ``self``. - :raise TypeError: If ``child`` is a tree with an impropriate - type. Typically, if ``child`` is a tree, then its type needs - to match the type of ``self``. This prevents mixing of - different tree types (single-parented, multi-parented, and - non-parented). - :param dry_run: If true, the don't actually set the child's - parent pointer; just check for any error conditions, and - raise an exception if one is found. - """ - - @abstractmethod - def _delparent(self, child, index): - """ - Update the parent pointer of ``child`` to not point to self. This - method is only called if the type of ``child`` is ``Tree``; i.e., it - is not called when removing a leaf from a tree. This method - is always called before the child is actually removed from the - child list of ``self``. - - :type child: Tree - :type index: int - :param index: The index of ``child`` in ``self``. - """ - - # //////////////////////////////////////////////////////////// - # Methods that add/remove children - # //////////////////////////////////////////////////////////// - # Every method that adds or removes a child must make - # appropriate calls to _setparent() and _delparent(). - - def __delitem__(self, index): - # del ptree[start:stop] - if isinstance(index, slice): - start, stop, step = slice_bounds(self, index, allow_step=True) - # Clear all the children pointers. - for i in range(start, stop, step): - if isinstance(self[i], Tree): - self._delparent(self[i], i) - # Delete the children from our child list. - super().__delitem__(index) - - # del ptree[i] - elif isinstance(index, int): - if index < 0: - index += len(self) - if index < 0: - raise IndexError("index out of range") - # Clear the child's parent pointer. - if isinstance(self[index], Tree): - self._delparent(self[index], index) - # Remove the child from our child list. - super().__delitem__(index) - - elif isinstance(index, (list, tuple)): - # del ptree[()] - if len(index) == 0: - raise IndexError("The tree position () may not be deleted.") - # del ptree[(i,)] - elif len(index) == 1: - del self[index[0]] - # del ptree[i1, i2, i3] - else: - del self[index[0]][index[1:]] - - else: - raise TypeError( - "%s indices must be integers, not %s" - % (type(self).__name__, type(index).__name__) - ) - - def __setitem__(self, index, value): - # ptree[start:stop] = value - if isinstance(index, slice): - start, stop, step = slice_bounds(self, index, allow_step=True) - # make a copy of value, in case it's an iterator - if not isinstance(value, (list, tuple)): - value = list(value) - # Check for any error conditions, so we can avoid ending - # up in an inconsistent state if an error does occur. - for i, child in enumerate(value): - if isinstance(child, Tree): - self._setparent(child, start + i * step, dry_run=True) - # clear the child pointers of all parents we're removing - for i in range(start, stop, step): - if isinstance(self[i], Tree): - self._delparent(self[i], i) - # set the child pointers of the new children. We do this - # after clearing *all* child pointers, in case we're e.g. - # reversing the elements in a tree. - for i, child in enumerate(value): - if isinstance(child, Tree): - self._setparent(child, start + i * step) - # finally, update the content of the child list itself. - super().__setitem__(index, value) - - # ptree[i] = value - elif isinstance(index, int): - if index < 0: - index += len(self) - if index < 0: - raise IndexError("index out of range") - # if the value is not changing, do nothing. - if value is self[index]: - return - # Set the new child's parent pointer. - if isinstance(value, Tree): - self._setparent(value, index) - # Remove the old child's parent pointer - if isinstance(self[index], Tree): - self._delparent(self[index], index) - # Update our child list. - super().__setitem__(index, value) - - elif isinstance(index, (list, tuple)): - # ptree[()] = value - if len(index) == 0: - raise IndexError("The tree position () may not be assigned to.") - # ptree[(i,)] = value - elif len(index) == 1: - self[index[0]] = value - # ptree[i1, i2, i3] = value - else: - self[index[0]][index[1:]] = value - - else: - raise TypeError( - "%s indices must be integers, not %s" - % (type(self).__name__, type(index).__name__) - ) - - def append(self, child): - if isinstance(child, Tree): - self._setparent(child, len(self)) - super().append(child) - - def extend(self, children): - for child in children: - if isinstance(child, Tree): - self._setparent(child, len(self)) - super().append(child) - - def insert(self, index, child): - # Handle negative indexes. Note that if index < -len(self), - # we do *not* raise an IndexError, unlike __getitem__. This - # is done for consistency with list.__getitem__ and list.index. - if index < 0: - index += len(self) - if index < 0: - index = 0 - # Set the child's parent, and update our child list. - if isinstance(child, Tree): - self._setparent(child, index) - super().insert(index, child) - - def pop(self, index=-1): - if index < 0: - index += len(self) - if index < 0: - raise IndexError("index out of range") - if isinstance(self[index], Tree): - self._delparent(self[index], index) - return super().pop(index) - - # n.b.: like `list`, this is done by equality, not identity! - # To remove a specific child, use del ptree[i]. - def remove(self, child): - index = self.index(child) - if isinstance(self[index], Tree): - self._delparent(self[index], index) - super().remove(child) - - # We need to implement __getslice__ and friends, even though - # they're deprecated, because otherwise list.__getslice__ will get - # called (since we're subclassing from list). Just delegate to - # __getitem__ etc., but use max(0, start) and max(0, stop) because - # because negative indices are already handled *before* - # __getslice__ is called; and we don't want to double-count them. - if hasattr(list, "__getslice__"): - - def __getslice__(self, start, stop): - return self.__getitem__(slice(max(0, start), max(0, stop))) - - def __delslice__(self, start, stop): - return self.__delitem__(slice(max(0, start), max(0, stop))) - - def __setslice__(self, start, stop, value): - return self.__setitem__(slice(max(0, start), max(0, stop)), value) - - -class ParentedTree(AbstractParentedTree): - """ - A ``Tree`` that automatically maintains parent pointers for - single-parented trees. The following are methods for querying - the structure of a parented tree: ``parent``, ``parent_index``, - ``left_sibling``, ``right_sibling``, ``root``, ``treeposition``. - - Each ``ParentedTree`` may have at most one parent. In - particular, subtrees may not be shared. Any attempt to reuse a - single ``ParentedTree`` as a child of more than one parent (or - as multiple children of the same parent) will cause a - ``ValueError`` exception to be raised. - - ``ParentedTrees`` should never be used in the same tree as ``Trees`` - or ``MultiParentedTrees``. Mixing tree implementations may result - in incorrect parent pointers and in ``TypeError`` exceptions. - """ - - def __init__(self, node, children=None): - self._parent = None - """The parent of this Tree, or None if it has no parent.""" - super().__init__(node, children) - if children is None: - # If children is None, the tree is read from node. - # After parsing, the parent of the immediate children - # will point to an intermediate tree, not self. - # We fix this by brute force: - for i, child in enumerate(self): - if isinstance(child, Tree): - child._parent = None - self._setparent(child, i) - - def _frozen_class(self): - return ImmutableParentedTree - - # ///////////////////////////////////////////////////////////////// - # Methods - # ///////////////////////////////////////////////////////////////// - - def parent(self): - """The parent of this tree, or None if it has no parent.""" - return self._parent - - def parent_index(self): - """ - The index of this tree in its parent. I.e., - ``ptree.parent()[ptree.parent_index()] is ptree``. Note that - ``ptree.parent_index()`` is not necessarily equal to - ``ptree.parent.index(ptree)``, since the ``index()`` method - returns the first child that is equal to its argument. - """ - if self._parent is None: - return None - for i, child in enumerate(self._parent): - if child is self: - return i - assert False, "expected to find self in self._parent!" - - def left_sibling(self): - """The left sibling of this tree, or None if it has none.""" - parent_index = self.parent_index() - if self._parent and parent_index > 0: - return self._parent[parent_index - 1] - return None # no left sibling - - def right_sibling(self): - """The right sibling of this tree, or None if it has none.""" - parent_index = self.parent_index() - if self._parent and parent_index < (len(self._parent) - 1): - return self._parent[parent_index + 1] - return None # no right sibling - - def root(self): - """ - The root of this tree. I.e., the unique ancestor of this tree - whose parent is None. If ``ptree.parent()`` is None, then - ``ptree`` is its own root. - """ - root = self - while root.parent() is not None: - root = root.parent() - return root - - def treeposition(self): - """ - The tree position of this tree, relative to the root of the - tree. I.e., ``ptree.root[ptree.treeposition] is ptree``. - """ - if self.parent() is None: - return () - else: - return self.parent().treeposition() + (self.parent_index(),) - - # ///////////////////////////////////////////////////////////////// - # Parent Management - # ///////////////////////////////////////////////////////////////// - - def _delparent(self, child, index): - # Sanity checks - assert isinstance(child, ParentedTree) - assert self[index] is child - assert child._parent is self - - # Delete child's parent pointer. - child._parent = None - - def _setparent(self, child, index, dry_run=False): - # If the child's type is incorrect, then complain. - if not isinstance(child, ParentedTree): - raise TypeError( - "Can not insert a non-ParentedTree " + "into a ParentedTree" - ) - - # If child already has a parent, then complain. - if child._parent is not None: - raise ValueError("Can not insert a subtree that already " "has a parent.") - - # Set child's parent pointer & index. - if not dry_run: - child._parent = self - - -class MultiParentedTree(AbstractParentedTree): - """ - A ``Tree`` that automatically maintains parent pointers for - multi-parented trees. The following are methods for querying the - structure of a multi-parented tree: ``parents()``, ``parent_indices()``, - ``left_siblings()``, ``right_siblings()``, ``roots``, ``treepositions``. - - Each ``MultiParentedTree`` may have zero or more parents. In - particular, subtrees may be shared. If a single - ``MultiParentedTree`` is used as multiple children of the same - parent, then that parent will appear multiple times in its - ``parents()`` method. - - ``MultiParentedTrees`` should never be used in the same tree as - ``Trees`` or ``ParentedTrees``. Mixing tree implementations may - result in incorrect parent pointers and in ``TypeError`` exceptions. - """ - - def __init__(self, node, children=None): - self._parents = [] - """A list of this tree's parents. This list should not - contain duplicates, even if a parent contains this tree - multiple times.""" - super().__init__(node, children) - if children is None: - # If children is None, the tree is read from node. - # After parsing, the parent(s) of the immediate children - # will point to an intermediate tree, not self. - # We fix this by brute force: - for i, child in enumerate(self): - if isinstance(child, Tree): - child._parents = [] - self._setparent(child, i) - - def _frozen_class(self): - return ImmutableMultiParentedTree - - # ///////////////////////////////////////////////////////////////// - # Methods - # ///////////////////////////////////////////////////////////////// - - def parents(self): - """ - The set of parents of this tree. If this tree has no parents, - then ``parents`` is the empty set. To check if a tree is used - as multiple children of the same parent, use the - ``parent_indices()`` method. - - :type: list(MultiParentedTree) - """ - return list(self._parents) - - def left_siblings(self): - """ - A list of all left siblings of this tree, in any of its parent - trees. A tree may be its own left sibling if it is used as - multiple contiguous children of the same parent. A tree may - appear multiple times in this list if it is the left sibling - of this tree with respect to multiple parents. - - :type: list(MultiParentedTree) - """ - return [ - parent[index - 1] - for (parent, index) in self._get_parent_indices() - if index > 0 - ] - - def right_siblings(self): - """ - A list of all right siblings of this tree, in any of its parent - trees. A tree may be its own right sibling if it is used as - multiple contiguous children of the same parent. A tree may - appear multiple times in this list if it is the right sibling - of this tree with respect to multiple parents. - - :type: list(MultiParentedTree) - """ - return [ - parent[index + 1] - for (parent, index) in self._get_parent_indices() - if index < (len(parent) - 1) - ] - - def _get_parent_indices(self): - return [ - (parent, index) - for parent in self._parents - for index, child in enumerate(parent) - if child is self - ] - - def roots(self): - """ - The set of all roots of this tree. This set is formed by - tracing all possible parent paths until trees with no parents - are found. - - :type: list(MultiParentedTree) - """ - return list(self._get_roots_helper({}).values()) - - def _get_roots_helper(self, result): - if self._parents: - for parent in self._parents: - parent._get_roots_helper(result) - else: - result[id(self)] = self - return result - - def parent_indices(self, parent): - """ - Return a list of the indices where this tree occurs as a child - of ``parent``. If this child does not occur as a child of - ``parent``, then the empty list is returned. The following is - always true:: - - for parent_index in ptree.parent_indices(parent): - parent[parent_index] is ptree - """ - if parent not in self._parents: - return [] - else: - return [index for (index, child) in enumerate(parent) if child is self] - - def treepositions(self, root): - """ - Return a list of all tree positions that can be used to reach - this multi-parented tree starting from ``root``. I.e., the - following is always true:: - - for treepos in ptree.treepositions(root): - root[treepos] is ptree - """ - if self is root: - return [()] - else: - return [ - treepos + (index,) - for parent in self._parents - for treepos in parent.treepositions(root) - for (index, child) in enumerate(parent) - if child is self - ] - - # ///////////////////////////////////////////////////////////////// - # Parent Management - # ///////////////////////////////////////////////////////////////// - - def _delparent(self, child, index): - # Sanity checks - assert isinstance(child, MultiParentedTree) - assert self[index] is child - assert len([p for p in child._parents if p is self]) == 1 - - # If the only copy of child in self is at index, then delete - # self from child's parent list. - for i, c in enumerate(self): - if c is child and i != index: - break - else: - child._parents.remove(self) - - def _setparent(self, child, index, dry_run=False): - # If the child's type is incorrect, then complain. - if not isinstance(child, MultiParentedTree): - raise TypeError( - "Can not insert a non-MultiParentedTree " + "into a MultiParentedTree" - ) - - # Add self as a parent pointer if it's not already listed. - if not dry_run: - for parent in child._parents: - if parent is self: - break - else: - child._parents.append(self) - - -class ImmutableParentedTree(ImmutableTree, ParentedTree): - pass - - -class ImmutableMultiParentedTree(ImmutableTree, MultiParentedTree): - pass - - -###################################################################### -## Probabilistic trees -###################################################################### - - -class ProbabilisticTree(Tree, ProbabilisticMixIn): - def __init__(self, node, children=None, **prob_kwargs): - Tree.__init__(self, node, children) - ProbabilisticMixIn.__init__(self, **prob_kwargs) - - # We have to patch up these methods to make them work right: - def _frozen_class(self): - return ImmutableProbabilisticTree - - def __repr__(self): - return f"{Tree.__repr__(self)} (p={self.prob()!r})" - - def __str__(self): - return f"{self.pformat(margin=60)} (p={self.prob():.6g})" - - def copy(self, deep=False): - if not deep: - return type(self)(self._label, self, prob=self.prob()) - else: - return type(self).convert(self) - - @classmethod - def convert(cls, val): - if isinstance(val, Tree): - children = [cls.convert(child) for child in val] - if isinstance(val, ProbabilisticMixIn): - return cls(val._label, children, prob=val.prob()) - else: - return cls(val._label, children, prob=1.0) - else: - return val - - def __eq__(self, other): - return ( - self.__class__ is other.__class__ - and ( - self._label, - list(self), - self.prob(), - ) - == (other._label, list(other), other.prob()) - ) - - def __lt__(self, other): - if not isinstance(other, Tree): - raise_unorderable_types("<", self, other) - if self.__class__ is other.__class__: - return (self._label, list(self), self.prob()) < ( - other._label, - list(other), - other.prob(), - ) - else: - return self.__class__.__name__ < other.__class__.__name__ - - -class ImmutableProbabilisticTree(ImmutableTree, ProbabilisticMixIn): - def __init__(self, node, children=None, **prob_kwargs): - ImmutableTree.__init__(self, node, children) - ProbabilisticMixIn.__init__(self, **prob_kwargs) - self._hash = hash((self._label, tuple(self), self.prob())) - - # We have to patch up these methods to make them work right: - def _frozen_class(self): - return ImmutableProbabilisticTree - - def __repr__(self): - return f"{Tree.__repr__(self)} [{self.prob()}]" - - def __str__(self): - return f"{self.pformat(margin=60)} [{self.prob()}]" - - def copy(self, deep=False): - if not deep: - return type(self)(self._label, self, prob=self.prob()) - else: - return type(self).convert(self) - - @classmethod - def convert(cls, val): - if isinstance(val, Tree): - children = [cls.convert(child) for child in val] - if isinstance(val, ProbabilisticMixIn): - return cls(val._label, children, prob=val.prob()) - else: - return cls(val._label, children, prob=1.0) - else: - return val - - def _child_names(tree): names = [] for child in tree: @@ -1667,55 +900,6 @@ def _child_names(tree): return names -###################################################################### -## Parsing -###################################################################### - - -def bracket_parse(s): - """ - Use Tree.read(s, remove_empty_top_bracketing=True) instead. - """ - raise NameError("Use Tree.read(s, remove_empty_top_bracketing=True) instead.") - - -def sinica_parse(s): - """ - Parse a Sinica Treebank string and return a tree. Trees are represented as nested brackettings, - as shown in the following example (X represents a Chinese character): - S(goal:NP(Head:Nep:XX)|theme:NP(Head:Nhaa:X)|quantity:Dab:X|Head:VL2:X)#0(PERIODCATEGORY) - - :return: A tree corresponding to the string representation. - :rtype: Tree - :param s: The string to be converted - :type s: str - """ - tokens = re.split(r"([()| ])", s) - for i in range(len(tokens)): - if tokens[i] == "(": - tokens[i - 1], tokens[i] = ( - tokens[i], - tokens[i - 1], - ) # pull nonterminal inside parens - elif ":" in tokens[i]: - fields = tokens[i].split(":") - if len(fields) == 2: # non-terminal - tokens[i] = fields[1] - else: - tokens[i] = "(" + fields[-2] + " " + fields[-1] + ")" - elif tokens[i] == "|": - tokens[i] = "" - - treebank_string = " ".join(tokens) - return Tree.fromstring(treebank_string, remove_empty_top_bracketing=True) - - -# s = re.sub(r'^#[^\s]*\s', '', s) # remove leading identifier -# s = re.sub(r'\w+:', '', s) # remove role tags - -# return s - - ###################################################################### ## Demonstration ###################################################################### @@ -1794,15 +978,5 @@ def demo(): __all__ = [ - "ImmutableProbabilisticTree", - "ImmutableTree", - "ProbabilisticMixIn", - "ProbabilisticTree", "Tree", - "bracket_parse", - "sinica_parse", - "ParentedTree", - "MultiParentedTree", - "ImmutableParentedTree", - "ImmutableMultiParentedTree", ] diff --git a/nltk/treeprettyprinter.py b/nltk/treeprettyprinter.py index defdf5e9d4..22686d75a1 100644 --- a/nltk/treeprettyprinter.py +++ b/nltk/treeprettyprinter.py @@ -17,612 +17,12 @@ https://jgaa.info/accepted/2006/EschbachGuentherBecker2006.10.2.pdf """ -import re +from nltk.internals import Deprecated +from nltk.tree.prettyprinter import TreePrettyPrinter as TPP -try: - from html import escape -except ImportError: - from cgi import escape -from collections import defaultdict -from operator import itemgetter - -from nltk.tree import Tree -from nltk.util import OrderedDict - -ANSICOLOR = { - "black": 30, - "red": 31, - "green": 32, - "yellow": 33, - "blue": 34, - "magenta": 35, - "cyan": 36, - "white": 37, -} - - -class TreePrettyPrinter: - """ - Pretty-print a tree in text format, either as ASCII or Unicode. - The tree can be a normal tree, or discontinuous. - - ``TreePrettyPrinter(tree, sentence=None, highlight=())`` - creates an object from which different visualizations can be created. - - :param tree: a Tree object. - :param sentence: a list of words (strings). If `sentence` is given, - `tree` must contain integers as leaves, which are taken as indices - in `sentence`. Using this you can display a discontinuous tree. - :param highlight: Optionally, a sequence of Tree objects in `tree` which - should be highlighted. Has the effect of only applying colors to nodes - in this sequence (nodes should be given as Tree objects, terminals as - indices). - - >>> from nltk.tree import Tree - >>> tree = Tree.fromstring('(S (NP Mary) (VP walks))') - >>> print(TreePrettyPrinter(tree).text()) - ... # doctest: +NORMALIZE_WHITESPACE - S - ____|____ - NP VP - | | - Mary walks - """ - - def __init__(self, tree, sentence=None, highlight=()): - if sentence is None: - leaves = tree.leaves() - if ( - leaves - and not any(len(a) == 0 for a in tree.subtrees()) - and all(isinstance(a, int) for a in leaves) - ): - sentence = [str(a) for a in leaves] - else: - # this deals with empty nodes (frontier non-terminals) - # and multiple/mixed terminals under non-terminals. - tree = tree.copy(True) - sentence = [] - for a in tree.subtrees(): - if len(a) == 0: - a.append(len(sentence)) - sentence.append(None) - elif any(not isinstance(b, Tree) for b in a): - for n, b in enumerate(a): - if not isinstance(b, Tree): - a[n] = len(sentence) - if type(b) == tuple: - b = "/".join(b) - sentence.append("%s" % b) - self.nodes, self.coords, self.edges, self.highlight = self.nodecoords( - tree, sentence, highlight - ) - - def __str__(self): - return self.text() - - def __repr__(self): - return "" % len(self.nodes) - - @staticmethod - def nodecoords(tree, sentence, highlight): - """ - Produce coordinates of nodes on a grid. - - Objective: - - - Produce coordinates for a non-overlapping placement of nodes and - horizontal lines. - - Order edges so that crossing edges cross a minimal number of previous - horizontal lines (never vertical lines). - - Approach: - - - bottom up level order traversal (start at terminals) - - at each level, identify nodes which cannot be on the same row - - identify nodes which cannot be in the same column - - place nodes into a grid at (row, column) - - order child-parent edges with crossing edges last - - Coordinates are (row, column); the origin (0, 0) is at the top left; - the root node is on row 0. Coordinates do not consider the size of a - node (which depends on font, &c), so the width of a column of the grid - should be automatically determined by the element with the greatest - width in that column. Alternatively, the integer coordinates could be - converted to coordinates in which the distances between adjacent nodes - are non-uniform. - - Produces tuple (nodes, coords, edges, highlighted) where: - - - nodes[id]: Tree object for the node with this integer id - - coords[id]: (n, m) coordinate where to draw node with id in the grid - - edges[id]: parent id of node with this id (ordered dictionary) - - highlighted: set of ids that should be highlighted - """ - - def findcell(m, matrix, startoflevel, children): - """ - Find vacant row, column index for node ``m``. - Iterate over current rows for this level (try lowest first) - and look for cell between first and last child of this node, - add new row to level if no free row available. - """ - candidates = [a for _, a in children[m]] - minidx, maxidx = min(candidates), max(candidates) - leaves = tree[m].leaves() - center = scale * sum(leaves) // len(leaves) # center of gravity - if minidx < maxidx and not minidx < center < maxidx: - center = sum(candidates) // len(candidates) - if max(candidates) - min(candidates) > 2 * scale: - center -= center % scale # round to unscaled coordinate - if minidx < maxidx and not minidx < center < maxidx: - center += scale - if ids[m] == 0: - startoflevel = len(matrix) - for rowidx in range(startoflevel, len(matrix) + 1): - if rowidx == len(matrix): # need to add a new row - matrix.append( - [ - vertline if a not in (corner, None) else None - for a in matrix[-1] - ] - ) - row = matrix[rowidx] - i = j = center - if len(children[m]) == 1: # place unaries directly above child - return rowidx, next(iter(children[m]))[1] - elif all( - a is None or a == vertline - for a in row[min(candidates) : max(candidates) + 1] - ): - # find free column - for n in range(scale): - i = j = center + n - while j > minidx or i < maxidx: - if i < maxidx and ( - matrix[rowidx][i] is None or i in candidates - ): - return rowidx, i - elif j > minidx and ( - matrix[rowidx][j] is None or j in candidates - ): - return rowidx, j - i += scale - j -= scale - raise ValueError( - "could not find a free cell for:\n%s\n%s" - "min=%d; max=%d" % (tree[m], minidx, maxidx, dumpmatrix()) - ) - - def dumpmatrix(): - """Dump matrix contents for debugging purposes.""" - return "\n".join( - "%2d: %s" % (n, " ".join(("%2r" % i)[:2] for i in row)) - for n, row in enumerate(matrix) - ) - - leaves = tree.leaves() - if not all(isinstance(n, int) for n in leaves): - raise ValueError("All leaves must be integer indices.") - if len(leaves) != len(set(leaves)): - raise ValueError("Indices must occur at most once.") - if not all(0 <= n < len(sentence) for n in leaves): - raise ValueError( - "All leaves must be in the interval 0..n " - "with n=len(sentence)\ntokens: %d indices: " - "%r\nsentence: %s" % (len(sentence), tree.leaves(), sentence) - ) - vertline, corner = -1, -2 # constants - tree = tree.copy(True) - for a in tree.subtrees(): - a.sort(key=lambda n: min(n.leaves()) if isinstance(n, Tree) else n) - scale = 2 - crossed = set() - # internal nodes and lexical nodes (no frontiers) - positions = tree.treepositions() - maxdepth = max(map(len, positions)) + 1 - childcols = defaultdict(set) - matrix = [[None] * (len(sentence) * scale)] - nodes = {} - ids = {a: n for n, a in enumerate(positions)} - highlighted_nodes = { - n for a, n in ids.items() if not highlight or tree[a] in highlight - } - levels = {n: [] for n in range(maxdepth - 1)} - terminals = [] - for a in positions: - node = tree[a] - if isinstance(node, Tree): - levels[maxdepth - node.height()].append(a) - else: - terminals.append(a) - - for n in levels: - levels[n].sort(key=lambda n: max(tree[n].leaves()) - min(tree[n].leaves())) - terminals.sort() - positions = set(positions) - - for m in terminals: - i = int(tree[m]) * scale - assert matrix[0][i] is None, (matrix[0][i], m, i) - matrix[0][i] = ids[m] - nodes[ids[m]] = sentence[tree[m]] - if nodes[ids[m]] is None: - nodes[ids[m]] = "..." - highlighted_nodes.discard(ids[m]) - positions.remove(m) - childcols[m[:-1]].add((0, i)) - - # add other nodes centered on their children, - # if the center is already taken, back off - # to the left and right alternately, until an empty cell is found. - for n in sorted(levels, reverse=True): - nodesatdepth = levels[n] - startoflevel = len(matrix) - matrix.append( - [vertline if a not in (corner, None) else None for a in matrix[-1]] - ) - for m in nodesatdepth: # [::-1]: - if n < maxdepth - 1 and childcols[m]: - _, pivot = min(childcols[m], key=itemgetter(1)) - if { - a[:-1] - for row in matrix[:-1] - for a in row[:pivot] - if isinstance(a, tuple) - } & { - a[:-1] - for row in matrix[:-1] - for a in row[pivot:] - if isinstance(a, tuple) - }: - crossed.add(m) - - rowidx, i = findcell(m, matrix, startoflevel, childcols) - positions.remove(m) - - # block positions where children of this node branch out - for _, x in childcols[m]: - matrix[rowidx][x] = corner - # assert m == () or matrix[rowidx][i] in (None, corner), ( - # matrix[rowidx][i], m, str(tree), ' '.join(sentence)) - # node itself - matrix[rowidx][i] = ids[m] - nodes[ids[m]] = tree[m] - # add column to the set of children for its parent - if m != (): - childcols[m[:-1]].add((rowidx, i)) - assert len(positions) == 0 - - # remove unused columns, right to left - for m in range(scale * len(sentence) - 1, -1, -1): - if not any(isinstance(row[m], (Tree, int)) for row in matrix): - for row in matrix: - del row[m] - - # remove unused rows, reverse - matrix = [ - row - for row in reversed(matrix) - if not all(a is None or a == vertline for a in row) - ] - - # collect coordinates of nodes - coords = {} - for n, _ in enumerate(matrix): - for m, i in enumerate(matrix[n]): - if isinstance(i, int) and i >= 0: - coords[i] = n, m - - # move crossed edges last - positions = sorted( - (a for level in levels.values() for a in level), - key=lambda a: a[:-1] in crossed, - ) - - # collect edges from node to node - edges = OrderedDict() - for i in reversed(positions): - for j, _ in enumerate(tree[i]): - edges[ids[i + (j,)]] = ids[i] - - return nodes, coords, edges, highlighted_nodes - - def text( - self, - nodedist=1, - unicodelines=False, - html=False, - ansi=False, - nodecolor="blue", - leafcolor="red", - funccolor="green", - abbreviate=None, - maxwidth=16, - ): - """ - :return: ASCII art for a discontinuous tree. - - :param unicodelines: whether to use Unicode line drawing characters - instead of plain (7-bit) ASCII. - :param html: whether to wrap output in html code (default plain text). - :param ansi: whether to produce colors with ANSI escape sequences - (only effective when html==False). - :param leafcolor, nodecolor: specify colors of leaves and phrasal - nodes; effective when either html or ansi is True. - :param abbreviate: if True, abbreviate labels longer than 5 characters. - If integer, abbreviate labels longer than `abbr` characters. - :param maxwidth: maximum number of characters before a label starts to - wrap; pass None to disable. - """ - if abbreviate == True: - abbreviate = 5 - if unicodelines: - horzline = "\u2500" - leftcorner = "\u250c" - rightcorner = "\u2510" - vertline = " \u2502 " - tee = horzline + "\u252C" + horzline - bottom = horzline + "\u2534" + horzline - cross = horzline + "\u253c" + horzline - ellipsis = "\u2026" - else: - horzline = "_" - leftcorner = rightcorner = " " - vertline = " | " - tee = 3 * horzline - cross = bottom = "_|_" - ellipsis = "." - - def crosscell(cur, x=vertline): - """Overwrite center of this cell with a vertical branch.""" - splitl = len(cur) - len(cur) // 2 - len(x) // 2 - 1 - lst = list(cur) - lst[splitl : splitl + len(x)] = list(x) - return "".join(lst) - - result = [] - matrix = defaultdict(dict) - maxnodewith = defaultdict(lambda: 3) - maxnodeheight = defaultdict(lambda: 1) - maxcol = 0 - minchildcol = {} - maxchildcol = {} - childcols = defaultdict(set) - labels = {} - wrapre = re.compile( - "(.{%d,%d}\\b\\W*|.{%d})" % (maxwidth - 4, maxwidth, maxwidth) - ) - # collect labels and coordinates - for a in self.nodes: - row, column = self.coords[a] - matrix[row][column] = a - maxcol = max(maxcol, column) - label = ( - self.nodes[a].label() - if isinstance(self.nodes[a], Tree) - else self.nodes[a] - ) - if abbreviate and len(label) > abbreviate: - label = label[:abbreviate] + ellipsis - if maxwidth and len(label) > maxwidth: - label = wrapre.sub(r"\1\n", label).strip() - label = label.split("\n") - maxnodeheight[row] = max(maxnodeheight[row], len(label)) - maxnodewith[column] = max(maxnodewith[column], max(map(len, label))) - labels[a] = label - if a not in self.edges: - continue # e.g., root - parent = self.edges[a] - childcols[parent].add((row, column)) - minchildcol[parent] = min(minchildcol.get(parent, column), column) - maxchildcol[parent] = max(maxchildcol.get(parent, column), column) - # bottom up level order traversal - for row in sorted(matrix, reverse=True): - noderows = [ - ["".center(maxnodewith[col]) for col in range(maxcol + 1)] - for _ in range(maxnodeheight[row]) - ] - branchrow = ["".center(maxnodewith[col]) for col in range(maxcol + 1)] - for col in matrix[row]: - n = matrix[row][col] - node = self.nodes[n] - text = labels[n] - if isinstance(node, Tree): - # draw horizontal branch towards children for this node - if n in minchildcol and minchildcol[n] < maxchildcol[n]: - i, j = minchildcol[n], maxchildcol[n] - a, b = (maxnodewith[i] + 1) // 2 - 1, maxnodewith[j] // 2 - branchrow[i] = ((" " * a) + leftcorner).ljust( - maxnodewith[i], horzline - ) - branchrow[j] = (rightcorner + (" " * b)).rjust( - maxnodewith[j], horzline - ) - for i in range(minchildcol[n] + 1, maxchildcol[n]): - if i == col and any(a == i for _, a in childcols[n]): - line = cross - elif i == col: - line = bottom - elif any(a == i for _, a in childcols[n]): - line = tee - else: - line = horzline - branchrow[i] = line.center(maxnodewith[i], horzline) - else: # if n and n in minchildcol: - branchrow[col] = crosscell(branchrow[col]) - text = [a.center(maxnodewith[col]) for a in text] - color = nodecolor if isinstance(node, Tree) else leafcolor - if isinstance(node, Tree) and node.label().startswith("-"): - color = funccolor - if html: - text = [escape(a, quote=False) for a in text] - if n in self.highlight: - text = [f"{a}" for a in text] - elif ansi and n in self.highlight: - text = ["\x1b[%d;1m%s\x1b[0m" % (ANSICOLOR[color], a) for a in text] - for x in range(maxnodeheight[row]): - # draw vertical lines in partially filled multiline node - # labels, but only if it's not a frontier node. - noderows[x][col] = ( - text[x] - if x < len(text) - else (vertline if childcols[n] else " ").center( - maxnodewith[col], " " - ) - ) - # for each column, if there is a node below us which has a parent - # above us, draw a vertical branch in that column. - if row != max(matrix): - for n, (childrow, col) in self.coords.items(): - if n > 0 and self.coords[self.edges[n]][0] < row < childrow: - branchrow[col] = crosscell(branchrow[col]) - if col not in matrix[row]: - for noderow in noderows: - noderow[col] = crosscell(noderow[col]) - branchrow = [ - a + ((a[-1] if a[-1] != " " else b[0]) * nodedist) - for a, b in zip(branchrow, branchrow[1:] + [" "]) - ] - result.append("".join(branchrow)) - result.extend( - (" " * nodedist).join(noderow) for noderow in reversed(noderows) - ) - return "\n".join(reversed(result)) + "\n" - - def svg(self, nodecolor="blue", leafcolor="red", funccolor="green"): - """ - :return: SVG representation of a tree. - """ - fontsize = 12 - hscale = 40 - vscale = 25 - hstart = vstart = 20 - width = max(col for _, col in self.coords.values()) - height = max(row for row, _ in self.coords.values()) - result = [ - '' - % ( - width * 3, - height * 2.5, - -hstart, - -vstart, - width * hscale + 3 * hstart, - height * vscale + 3 * vstart, - ) - ] - - children = defaultdict(set) - for n in self.nodes: - if n: - children[self.edges[n]].add(n) - - # horizontal branches from nodes to children - for node in self.nodes: - if not children[node]: - continue - y, x = self.coords[node] - x *= hscale - y *= vscale - x += hstart - y += vstart + fontsize // 2 - childx = [self.coords[c][1] for c in children[node]] - xmin = hstart + hscale * min(childx) - xmax = hstart + hscale * max(childx) - result.append( - '\t' % (xmin, y, xmax, y) - ) - result.append( - '\t' % (x, y, x, y - fontsize // 3) - ) - - # vertical branches from children to parents - for child, parent in self.edges.items(): - y, _ = self.coords[parent] - y *= vscale - y += vstart + fontsize // 2 - childy, childx = self.coords[child] - childx *= hscale - childy *= vscale - childx += hstart - childy += vstart - fontsize - result += [ - '\t' % (childx, childy, childx, y + 5), - '\t' % (childx, childy, childx, y), - ] - - # write nodes with coordinates - for n, (row, column) in self.coords.items(): - node = self.nodes[n] - x = column * hscale + hstart - y = row * vscale + vstart - if n in self.highlight: - color = nodecolor if isinstance(node, Tree) else leafcolor - if isinstance(node, Tree) and node.label().startswith("-"): - color = funccolor - else: - color = "black" - result += [ - '\t%s' - % ( - color, - fontsize, - x, - y, - escape( - node.label() if isinstance(node, Tree) else node, quote=False - ), - ) - ] - - result += [""] - return "\n".join(result) - - -def test(): - """Do some tree drawing tests.""" - - def print_tree(n, tree, sentence=None, ansi=True, **xargs): - print() - print('{}: "{}"'.format(n, " ".join(sentence or tree.leaves()))) - print(tree) - print() - drawtree = TreePrettyPrinter(tree, sentence) - try: - print(drawtree.text(unicodelines=ansi, ansi=ansi, **xargs)) - except (UnicodeDecodeError, UnicodeEncodeError): - print(drawtree.text(unicodelines=False, ansi=False, **xargs)) - - from nltk.corpus import treebank - - for n in [0, 1440, 1591, 2771, 2170]: - tree = treebank.parsed_sents()[n] - print_tree(n, tree, nodedist=2, maxwidth=8) - print() - print("ASCII version:") - print(TreePrettyPrinter(tree).text(nodedist=2)) - - tree = Tree.fromstring( - "(top (punct 8) (smain (noun 0) (verb 1) (inf (verb 5) (inf (verb 6) " - "(conj (inf (pp (prep 2) (np (det 3) (noun 4))) (verb 7)) (inf (verb 9)) " - "(vg 10) (inf (verb 11)))))) (punct 12))", - read_leaf=int, - ) - sentence = ( - "Ze had met haar moeder kunnen gaan winkelen ," - " zwemmen of terrassen .".split() - ) - print_tree("Discontinuous tree", tree, sentence, nodedist=2) +class TreePrettyPrinter(Deprecated, TPP): + """Import `TreePrettyPrinter` using `from nltk.tree import TreePrettyPrinter` instead.""" __all__ = ["TreePrettyPrinter"] - -if __name__ == "__main__": - test() diff --git a/nltk/treetransforms.py b/nltk/treetransforms.py index 5b89d1a017..d1360618de 100644 --- a/nltk/treetransforms.py +++ b/nltk/treetransforms.py @@ -107,232 +107,20 @@ """ -from nltk.tree import Tree +from nltk.internals import deprecated +from nltk.tree.transforms import chomsky_normal_form as cnf +from nltk.tree.transforms import collapse_unary as cu +from nltk.tree.transforms import un_chomsky_normal_form as ucnf + +chomsky_normal_form = deprecated( + "Import using `from nltk.tree import chomsky_normal_form` instead." +)(cnf) +un_chomsky_normal_form = deprecated( + "Import using `from nltk.tree import un_chomsky_normal_form` instead." +)(ucnf) +collapse_unary = deprecated( + "Import using `from nltk.tree import collapse_unary` instead." +)(cu) -def chomsky_normal_form( - tree, factor="right", horzMarkov=None, vertMarkov=0, childChar="|", parentChar="^" -): - # assume all subtrees have homogeneous children - # assume all terminals have no siblings - - # A semi-hack to have elegant looking code below. As a result, - # any subtree with a branching factor greater than 999 will be incorrectly truncated. - if horzMarkov is None: - horzMarkov = 999 - - # Traverse the tree depth-first keeping a list of ancestor nodes to the root. - # I chose not to use the tree.treepositions() method since it requires - # two traversals of the tree (one to get the positions, one to iterate - # over them) and node access time is proportional to the height of the node. - # This method is 7x faster which helps when parsing 40,000 sentences. - - nodeList = [(tree, [tree.label()])] - while nodeList != []: - node, parent = nodeList.pop() - if isinstance(node, Tree): - - # parent annotation - parentString = "" - originalNode = node.label() - if vertMarkov != 0 and node != tree and isinstance(node[0], Tree): - parentString = "{}<{}>".format(parentChar, "-".join(parent)) - node.set_label(node.label() + parentString) - parent = [originalNode] + parent[: vertMarkov - 1] - - # add children to the agenda before we mess with them - for child in node: - nodeList.append((child, parent)) - - # chomsky normal form factorization - if len(node) > 2: - childNodes = [child.label() for child in node] - nodeCopy = node.copy() - node[0:] = [] # delete the children - - curNode = node - numChildren = len(nodeCopy) - for i in range(1, numChildren - 1): - if factor == "right": - newHead = "{}{}<{}>{}".format( - originalNode, - childChar, - "-".join( - childNodes[i : min([i + horzMarkov, numChildren])] - ), - parentString, - ) # create new head - newNode = Tree(newHead, []) - curNode[0:] = [nodeCopy.pop(0), newNode] - else: - newHead = "{}{}<{}>{}".format( - originalNode, - childChar, - "-".join( - childNodes[max([numChildren - i - horzMarkov, 0]) : -i] - ), - parentString, - ) - newNode = Tree(newHead, []) - curNode[0:] = [newNode, nodeCopy.pop()] - - curNode = newNode - - curNode[0:] = [child for child in nodeCopy] - - -def un_chomsky_normal_form( - tree, expandUnary=True, childChar="|", parentChar="^", unaryChar="+" -): - # Traverse the tree-depth first keeping a pointer to the parent for modification purposes. - nodeList = [(tree, [])] - while nodeList != []: - node, parent = nodeList.pop() - if isinstance(node, Tree): - # if the node contains the 'childChar' character it means that - # it is an artificial node and can be removed, although we still need - # to move its children to its parent - childIndex = node.label().find(childChar) - if childIndex != -1: - nodeIndex = parent.index(node) - parent.remove(parent[nodeIndex]) - # Generated node was on the left if the nodeIndex is 0 which - # means the grammar was left factored. We must insert the children - # at the beginning of the parent's children - if nodeIndex == 0: - parent.insert(0, node[0]) - parent.insert(1, node[1]) - else: - parent.extend([node[0], node[1]]) - - # parent is now the current node so the children of parent will be added to the agenda - node = parent - else: - parentIndex = node.label().find(parentChar) - if parentIndex != -1: - # strip the node name of the parent annotation - node.set_label(node.label()[:parentIndex]) - - # expand collapsed unary productions - if expandUnary == True: - unaryIndex = node.label().find(unaryChar) - if unaryIndex != -1: - newNode = Tree( - node.label()[unaryIndex + 1 :], [i for i in node] - ) - node.set_label(node.label()[:unaryIndex]) - node[0:] = [newNode] - - for child in node: - nodeList.append((child, node)) - - -def collapse_unary(tree, collapsePOS=False, collapseRoot=False, joinChar="+"): - """ - Collapse subtrees with a single child (ie. unary productions) - into a new non-terminal (Tree node) joined by 'joinChar'. - This is useful when working with algorithms that do not allow - unary productions, and completely removing the unary productions - would require loss of useful information. The Tree is modified - directly (since it is passed by reference) and no value is returned. - - :param tree: The Tree to be collapsed - :type tree: Tree - :param collapsePOS: 'False' (default) will not collapse the parent of leaf nodes (ie. - Part-of-Speech tags) since they are always unary productions - :type collapsePOS: bool - :param collapseRoot: 'False' (default) will not modify the root production - if it is unary. For the Penn WSJ treebank corpus, this corresponds - to the TOP -> productions. - :type collapseRoot: bool - :param joinChar: A string used to connect collapsed node values (default = "+") - :type joinChar: str - """ - - if collapseRoot == False and isinstance(tree, Tree) and len(tree) == 1: - nodeList = [tree[0]] - else: - nodeList = [tree] - - # depth-first traversal of tree - while nodeList != []: - node = nodeList.pop() - if isinstance(node, Tree): - if ( - len(node) == 1 - and isinstance(node[0], Tree) - and (collapsePOS == True or isinstance(node[0, 0], Tree)) - ): - node.set_label(node.label() + joinChar + node[0].label()) - node[0:] = [child for child in node[0]] - # since we assigned the child's children to the current node, - # evaluate the current node again - nodeList.append(node) - else: - for child in node: - nodeList.append(child) - - -################################################################# -# Demonstration -################################################################# - - -def demo(): - """ - A demonstration showing how each tree transform can be used. - """ - - from copy import deepcopy - - from nltk import tree, treetransforms - from nltk.draw.tree import draw_trees - - # original tree from WSJ bracketed text - sentence = """(TOP - (S - (S - (VP - (VBN Turned) - (ADVP (RB loose)) - (PP - (IN in) - (NP - (NP (NNP Shane) (NNP Longman) (POS 's)) - (NN trading) - (NN room))))) - (, ,) - (NP (DT the) (NN yuppie) (NNS dealers)) - (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right)))) - (. .)))""" - t = tree.Tree.fromstring(sentence, remove_empty_top_bracketing=True) - - # collapse subtrees with only one child - collapsedTree = deepcopy(t) - treetransforms.collapse_unary(collapsedTree) - - # convert the tree to CNF - cnfTree = deepcopy(collapsedTree) - treetransforms.chomsky_normal_form(cnfTree) - - # convert the tree to CNF with parent annotation (one level) and horizontal smoothing of order two - parentTree = deepcopy(collapsedTree) - treetransforms.chomsky_normal_form(parentTree, horzMarkov=2, vertMarkov=1) - - # convert the tree back to its original form (used to make CYK results comparable) - original = deepcopy(parentTree) - treetransforms.un_chomsky_normal_form(original) - - # convert tree back to bracketed text - sentence2 = original.pprint() - print(sentence) - print(sentence2) - print("Sentences the same? ", sentence == sentence2) - - draw_trees(t, collapsedTree, cnfTree, parentTree, original) - - -if __name__ == "__main__": - demo() - __all__ = ["chomsky_normal_form", "un_chomsky_normal_form", "collapse_unary"]