From c30bd05009f74cbe6646571c2849cead29189290 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 30 Oct 2022 17:55:29 +0200 Subject: [PATCH 1/6] Fix tests for newest astroid --- asttokens/mark_tokens.py | 11 +++-------- asttokens/util.py | 17 ++++++++++++++++- setup.cfg | 2 +- tests/test_astroid.py | 4 ++-- tests/test_mark_tokens.py | 11 ++++++++++- 5 files changed, 32 insertions(+), 13 deletions(-) diff --git a/asttokens/mark_tokens.py b/asttokens/mark_tokens.py index 0f935c0..139920c 100644 --- a/asttokens/mark_tokens.py +++ b/asttokens/mark_tokens.py @@ -23,13 +23,7 @@ from . import util from .asttokens import ASTTokens -from .util import AstConstant - -try: - import astroid.node_classes as nc -except Exception: - # This is only used for type checking, we don't need it if astroid isn't installed. - nc = None +from .util import AstConstant, astroid_node_classes as nc if TYPE_CHECKING: from .util import AstNode @@ -88,7 +82,8 @@ def _visit_after_children(self, node, parent_token, token): first = token last = None for child in cast(Callable, self._iter_children)(node): - if not first or child.first_token.index < first.index: + # astroid slices have especially wrong positions, we don't want them to corrupt their parents. + if not first or child.first_token.index < first.index and not util.is_astroid_slice(child): first = child.first_token if not last or child.last_token.index > last.index: last = child.last_token diff --git a/asttokens/util.py b/asttokens/util.py index 96fa931..c4299b5 100644 --- a/asttokens/util.py +++ b/asttokens/util.py @@ -24,8 +24,18 @@ from six import iteritems +try: + from astroid import nodes as astroid_node_classes + getattr(astroid_node_classes, "NodeNG") +except Exception: + try: + from astroid import node_classes as astroid_node_classes + except Exception: + astroid_node_classes = None + + if TYPE_CHECKING: # pragma: no cover - from astroid.node_classes import NodeNG + NodeNG = astroid_node_classes.NodeNG # Type class used to expand out the definition of AST to include fields added by this library # It's not actually used for anything other than type checking though! @@ -218,6 +228,11 @@ def is_slice(node): ) +def is_astroid_slice(node): + # type: (AstNode) -> bool + return is_slice(node) and not isinstance(node, ast.AST) + + # Sentinel value used by visit_tree(). _PREVISIT = object() diff --git a/setup.cfg b/setup.cfg index a2bde84..f506500 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,7 +39,7 @@ install_requires = setup_requires = setuptools>=44; setuptools_scm[toml]>=3.4.3 [options.extras_require] -test = astroid<=2.5.3; pytest +test = astroid; pytest [options.package_data] asttokens = py.typed diff --git a/tests/test_astroid.py b/tests/test_astroid.py index 1608359..30ac6d6 100644 --- a/tests/test_astroid.py +++ b/tests/test_astroid.py @@ -2,9 +2,9 @@ from __future__ import unicode_literals, print_function import astroid -from astroid.node_classes import NodeNG from asttokens import ASTTokens +from asttokens.util import astroid_node_classes from . import test_mark_tokens @@ -13,7 +13,7 @@ class TestAstroid(test_mark_tokens.TestMarkTokens): is_astroid_test = True module = astroid - nodes_classes = NodeNG + nodes_classes = astroid_node_classes.NodeNG context_classes = [ (astroid.Name, astroid.DelName, astroid.AssignName), (astroid.Attribute, astroid.DelAttr, astroid.AssignAttr), diff --git a/tests/test_mark_tokens.py b/tests/test_mark_tokens.py index cebb226..dbcc52f 100644 --- a/tests/test_mark_tokens.py +++ b/tests/test_mark_tokens.py @@ -19,6 +19,11 @@ from . import tools +try: + from astroid.nodes.utils import Position as AstroidPosition +except Exception: + AstroidPosition = () + class TestMarkTokens(unittest.TestCase): maxDiff = None @@ -243,7 +248,7 @@ def test_slices(self): # important, so we skip them here. self.assertEqual({n for n in m.view_nodes_at(1, 56) if 'Slice:' not in n}, { "Subscript:foo[:]", "Name:foo" }) - self.assertEqual({n for n in m.view_nodes_at(1, 64) if 'Slice:' not in n}, + self.assertEqual({n for n in m.view_nodes_at(1, 64) if 'Slice:' not in n and 'Tuple:' not in n}, { "Subscript:bar[::2, :]", "Name:bar" }) def test_adjacent_strings(self): @@ -814,6 +819,10 @@ def assert_nodes_equal(self, t1, t2): else: self.assertEqual(type(t1), type(t2)) + if isinstance(t1, AstroidPosition): + # Ignore the lineno/col_offset etc. from astroid + return + if isinstance(t1, (list, tuple)): self.assertEqual(len(t1), len(t2)) for vc1, vc2 in zip(t1, t2): From 5c7e360f93b55553b6898f14936d9873461eb3a9 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 30 Oct 2022 18:17:43 +0200 Subject: [PATCH 2/6] Specifically ignore empty slices --- asttokens/mark_tokens.py | 4 +++- asttokens/util.py | 8 ++++++-- tests/test_mark_tokens.py | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/asttokens/mark_tokens.py b/asttokens/mark_tokens.py index 139920c..61f7017 100644 --- a/asttokens/mark_tokens.py +++ b/asttokens/mark_tokens.py @@ -83,7 +83,9 @@ def _visit_after_children(self, node, parent_token, token): last = None for child in cast(Callable, self._iter_children)(node): # astroid slices have especially wrong positions, we don't want them to corrupt their parents. - if not first or child.first_token.index < first.index and not util.is_astroid_slice(child): + if util.is_empty_astroid_slice(child): + continue + if not first or child.first_token.index < first.index: first = child.first_token if not last or child.last_token.index > last.index: last = child.last_token diff --git a/asttokens/util.py b/asttokens/util.py index c4299b5..b515cf0 100644 --- a/asttokens/util.py +++ b/asttokens/util.py @@ -228,9 +228,13 @@ def is_slice(node): ) -def is_astroid_slice(node): +def is_empty_astroid_slice(node): # type: (AstNode) -> bool - return is_slice(node) and not isinstance(node, ast.AST) + return ( + node.__class__.__name__ == "Slice" + and not isinstance(node, ast.AST) + and node.lower is node.upper is node.step + ) # Sentinel value used by visit_tree(). diff --git a/tests/test_mark_tokens.py b/tests/test_mark_tokens.py index dbcc52f..5aba077 100644 --- a/tests/test_mark_tokens.py +++ b/tests/test_mark_tokens.py @@ -235,7 +235,7 @@ def test_deep_recursion(self): def test_slices(self): # Make sure we don't fail on parsing slices of the form `foo[4:]`. - source = "(foo.Area_Code, str(foo.Phone)[:3], str(foo.Phone)[3:], foo[:], bar[::2, :], [a[:]][::-1])" + source = "(foo.Area_Code, str(foo.Phone)[:3], str(foo.Phone)[3:], foo[:], bar[::2, :], bar2[:, ::2], [a[:]][::-1])" m = self.create_mark_checker(source) self.assertIn("Tuple:" + source, m.view_nodes_at(1, 0)) self.assertEqual(m.view_nodes_at(1, 1), From 32b476b0f9f86c876b94dc53ffc319756705bcb8 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 30 Oct 2022 18:23:03 +0200 Subject: [PATCH 3/6] comment on astroid_node_classes --- asttokens/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/asttokens/util.py b/asttokens/util.py index b515cf0..e29ef3b 100644 --- a/asttokens/util.py +++ b/asttokens/util.py @@ -26,6 +26,7 @@ try: from astroid import nodes as astroid_node_classes + # astroid_node_classes should be whichever module has the NodeNG class getattr(astroid_node_classes, "NodeNG") except Exception: try: From 8a01fbca4518582eb16c81cdf64424bba9a750a9 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 30 Oct 2022 19:12:54 +0200 Subject: [PATCH 4/6] mypy --- asttokens/astroid_compat.py | 14 ++++++++++++++ asttokens/mark_tokens.py | 3 ++- asttokens/util.py | 12 +----------- pyproject.toml | 4 ++-- tests/test_astroid.py | 2 +- 5 files changed, 20 insertions(+), 15 deletions(-) create mode 100644 asttokens/astroid_compat.py diff --git a/asttokens/astroid_compat.py b/asttokens/astroid_compat.py new file mode 100644 index 0000000..32eada9 --- /dev/null +++ b/asttokens/astroid_compat.py @@ -0,0 +1,14 @@ +try: + from astroid import nodes as astroid_node_classes + + # astroid_node_classes should be whichever module has the NodeNG class + from astroid.nodes import NodeNG +except Exception: + try: + from astroid import node_classes as astroid_node_classes + from astroid.node_classes import NodeNG + except Exception: + astroid_node_classes = None + NodeNG = None + +__all__ = ["astroid_node_classes", "NodeNG"] diff --git a/asttokens/mark_tokens.py b/asttokens/mark_tokens.py index 61f7017..0aa497f 100644 --- a/asttokens/mark_tokens.py +++ b/asttokens/mark_tokens.py @@ -23,7 +23,8 @@ from . import util from .asttokens import ASTTokens -from .util import AstConstant, astroid_node_classes as nc +from .util import AstConstant +from .astroid_compat import astroid_node_classes as nc if TYPE_CHECKING: from .util import AstNode diff --git a/asttokens/util.py b/asttokens/util.py index e29ef3b..6230f7e 100644 --- a/asttokens/util.py +++ b/asttokens/util.py @@ -24,20 +24,10 @@ from six import iteritems -try: - from astroid import nodes as astroid_node_classes - # astroid_node_classes should be whichever module has the NodeNG class - getattr(astroid_node_classes, "NodeNG") -except Exception: - try: - from astroid import node_classes as astroid_node_classes - except Exception: - astroid_node_classes = None +from .astroid_compat import NodeNG if TYPE_CHECKING: # pragma: no cover - NodeNG = astroid_node_classes.NodeNG - # Type class used to expand out the definition of AST to include fields added by this library # It's not actually used for anything other than type checking though! class EnhancedAST(AST): diff --git a/pyproject.toml b/pyproject.toml index ea6e65f..2543e24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,5 +20,5 @@ disallow_untyped_calls=false ignore_missing_imports=true [[tool.mypy.overrides]] -module = ["astroid", "astroid.node_classes"] -ignore_missing_imports = true \ No newline at end of file +module = ["astroid", "astroid.node_classes", "astroid.nodes", "astroid.nodes.utils"] +ignore_missing_imports = true diff --git a/tests/test_astroid.py b/tests/test_astroid.py index 30ac6d6..a5cc6d7 100644 --- a/tests/test_astroid.py +++ b/tests/test_astroid.py @@ -4,7 +4,7 @@ import astroid from asttokens import ASTTokens -from asttokens.util import astroid_node_classes +from asttokens.astroid_compat import astroid_node_classes from . import test_mark_tokens From 586ca787cc71a7483d78f9efb4ce50e2c97cb53b Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 30 Oct 2022 19:15:44 +0200 Subject: [PATCH 5/6] coverage --- asttokens/astroid_compat.py | 2 +- asttokens/util.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/asttokens/astroid_compat.py b/asttokens/astroid_compat.py index 32eada9..3ba5e8d 100644 --- a/asttokens/astroid_compat.py +++ b/asttokens/astroid_compat.py @@ -7,7 +7,7 @@ try: from astroid import node_classes as astroid_node_classes from astroid.node_classes import NodeNG - except Exception: + except Exception: # pragma: no cover astroid_node_classes = None NodeNG = None diff --git a/asttokens/util.py b/asttokens/util.py index 6230f7e..f3dc114 100644 --- a/asttokens/util.py +++ b/asttokens/util.py @@ -24,10 +24,10 @@ from six import iteritems -from .astroid_compat import NodeNG - if TYPE_CHECKING: # pragma: no cover + from .astroid_compat import NodeNG + # Type class used to expand out the definition of AST to include fields added by this library # It's not actually used for anything other than type checking though! class EnhancedAST(AST): From 17fb607f8de7f9e487b3ad23c0ff9d8c1c5266fe Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Fri, 25 Nov 2022 11:52:31 +0200 Subject: [PATCH 6/6] fix is_empty_astroid_slice --- asttokens/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asttokens/util.py b/asttokens/util.py index f3dc114..4abc83e 100644 --- a/asttokens/util.py +++ b/asttokens/util.py @@ -224,7 +224,7 @@ def is_empty_astroid_slice(node): return ( node.__class__.__name__ == "Slice" and not isinstance(node, ast.AST) - and node.lower is node.upper is node.step + and node.lower is node.upper is node.step is None )