From f98902d169a1a49a69e27a600a4db5275ddbc1d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Mon, 25 Oct 2021 11:40:37 +0200 Subject: [PATCH] Changes to integrate new parser into packaging + test adjustments --- packaging/markers.py | 42 +++++++------------------------------- packaging/requirements.py | 25 ++++++++++++++--------- tests/test_markers.py | 3 ++- tests/test_requirements.py | 34 ++++++++++++++++++++++-------- 4 files changed, 49 insertions(+), 55 deletions(-) diff --git a/packaging/markers.py b/packaging/markers.py index d3b062369..18c0bbce5 100644 --- a/packaging/markers.py +++ b/packaging/markers.py @@ -6,8 +6,10 @@ import os import platform import sys -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Callable, Dict, List, Optional, Union +from ._parser import MarkerAtom, MarkerList, Op, Variable, parse_quoted_marker +from ._tokenizer import ParseException, Tokenizer from .specifiers import InvalidSpecifier, Specifier __all__ = [ @@ -40,37 +42,8 @@ class UndefinedEnvironmentName(ValueError): """ -class Node: - def __init__(self, value: Any) -> None: - self.value = value - - def __str__(self) -> str: - return str(self.value) - - def __repr__(self) -> str: - return f"<{self.__class__.__name__}('{self}')>" - - def serialize(self) -> str: - raise NotImplementedError - - -class Variable(Node): - def serialize(self) -> str: - return str(self) - - -class Value(Node): - def serialize(self) -> str: - return f'"{self}"' - - -class Op(Node): - def serialize(self) -> str: - return str(self) - - def _format_marker( - marker: Union[List[str], Tuple[Node, ...], str], first: Optional[bool] = True + marker: Union[List[str], MarkerAtom, str], first: Optional[bool] = True ) -> str: assert isinstance(marker, (list, tuple, str)) @@ -143,7 +116,7 @@ def _get_env(environment: Dict[str, str], name: str) -> str: return value -def _evaluate_markers(markers: List[Any], environment: Dict[str, str]) -> bool: +def _evaluate_markers(markers: MarkerList, environment: Dict[str, str]) -> bool: groups: List[List[bool]] = [[]] for marker in markers: @@ -199,11 +172,11 @@ def default_environment() -> Dict[str, str]: class Marker: def __init__(self, marker: str) -> None: try: - self._markers = _coerce_parse_result(MARKER.parseString(marker)) + self._markers = parse_quoted_marker(Tokenizer(marker)) except ParseException as e: raise InvalidMarker( f"Invalid marker: {marker!r}, parse error at " - f"{marker[e.loc : e.loc + 8]!r}" + f"{marker[e.position : e.position + 8]!r}" ) def __str__(self) -> str: @@ -224,5 +197,4 @@ def evaluate(self, environment: Optional[Dict[str, str]] = None) -> bool: current_environment = default_environment() if environment is not None: current_environment.update(environment) - return _evaluate_markers(self._markers, current_environment) diff --git a/packaging/requirements.py b/packaging/requirements.py index 7d128f5ed..ce4638b95 100644 --- a/packaging/requirements.py +++ b/packaging/requirements.py @@ -2,13 +2,14 @@ # 2.0, and the BSD License. See the LICENSE file in the root of this repository # for complete details. -import re -import string import urllib.parse +from collections import namedtuple from typing import List, Optional as TOptional, Set -from .markers import Marker -from .specifiers import LegacySpecifier, Specifier, SpecifierSet +from ._parser import parse_named_requirement +from ._tokenizer import ParseException +from .markers import InvalidMarker, Marker +from .specifiers import SpecifierSet class InvalidRequirement(ValueError): @@ -31,12 +32,13 @@ class Requirement: # TODO: Can we normalize the name and extra name? def __init__(self, requirement_string: str) -> None: + _RequirementTuple = namedtuple( + "_RequirementTuple", ["name", "url", "extras", "specifier", "marker"] + ) try: - req = REQUIREMENT.parseString(requirement_string) + req = _RequirementTuple(*parse_named_requirement(requirement_string)) except ParseException as e: - raise InvalidRequirement( - f'Parse error at "{ requirement_string[e.loc : e.loc + 8]!r}": {e.msg}' - ) + raise InvalidRequirement(str(e)) self.name: str = req.name if req.url: @@ -51,9 +53,12 @@ def __init__(self, requirement_string: str) -> None: self.url: TOptional[str] = req.url else: self.url = None - self.extras: Set[str] = set(req.extras.asList() if req.extras else []) + self.extras: Set[str] = set(req.extras if req.extras else []) self.specifier: SpecifierSet = SpecifierSet(req.specifier) - self.marker: TOptional[Marker] = req.marker if req.marker else None + try: + self.marker: TOptional[Marker] = Marker(req.marker) if req.marker else None + except InvalidMarker as e: + raise InvalidRequirement(str(e)) def __str__(self) -> str: parts: List[str] = [self.name] diff --git a/tests/test_markers.py b/tests/test_markers.py index 64573a901..d73569cdd 100644 --- a/tests/test_markers.py +++ b/tests/test_markers.py @@ -10,10 +10,10 @@ import pytest +from packaging._parser import Node from packaging.markers import ( InvalidMarker, Marker, - Node, UndefinedComparison, UndefinedEnvironmentName, default_environment, @@ -161,6 +161,7 @@ def test_parses_valid(self, marker_string): "python_version", "(python_version)", "python_version >= 1.0 and (python_version)", + '(python_version == "2.7" and os_name == "linux"', ], ) def test_parses_invalid(self, marker_string): diff --git a/tests/test_requirements.py b/tests/test_requirements.py index 68ad57c3b..daab527c5 100644 --- a/tests/test_requirements.py +++ b/tests/test_requirements.py @@ -5,7 +5,7 @@ import pytest from packaging.markers import Marker -from packaging.requirements import URL, URL_AND_MARKER, InvalidRequirement, Requirement +from packaging.requirements import InvalidRequirement, Requirement from packaging.specifiers import SpecifierSet @@ -59,10 +59,20 @@ def test_name_with_version(self): req = Requirement("name>=3") self._assert_requirement(req, "name", specifier=">=3") + def test_name_with_missing_version(self): + with pytest.raises(InvalidRequirement) as e: + Requirement("name>=") + assert "Missing version" in str(e) + def test_version_with_parens_and_whitespace(self): req = Requirement("name (==4)") self._assert_requirement(req, "name", specifier="==4") + def test_version_with_missing_closing_paren(self): + with pytest.raises(InvalidRequirement) as e: + Requirement("name(==4") + assert "Closing right parenthesis is missing" in str(e) + def test_name_with_multiple_versions(self): req = Requirement("name>=3,<2") self._assert_requirement(req, "name", specifier="<2,>=3") @@ -79,21 +89,27 @@ def test_empty_extras(self): req = Requirement("foo[]") self._assert_requirement(req, "foo") + def test_unclosed_extras(self): + with pytest.raises(InvalidRequirement) as e: + Requirement("foo[") + assert "Closing square bracket is missing" in str(e) + def test_url(self): - url_section = "@ http://example.com" - parsed = URL.parseString(url_section) - assert parsed.url == "http://example.com" + url_section = "test @ http://example.com" + req = Requirement(url_section) + self._assert_requirement(req, "test", "http://example.com", extras=[]) def test_url_and_marker(self): - instring = "@ http://example.com ; os_name=='a'" - parsed = URL_AND_MARKER.parseString(instring) - assert parsed.url == "http://example.com" - assert str(parsed.marker) == 'os_name == "a"' + instring = "test @ http://example.com ; os_name=='a'" + req = Requirement(instring) + self._assert_requirement( + req, "test", "http://example.com", extras=[], marker='os_name == "a"' + ) def test_invalid_url(self): with pytest.raises(InvalidRequirement) as e: Requirement("name @ gopher:/foo/com") - assert "Invalid URL: " in str(e.value) + assert "Invalid URL: " in str(e) assert "gopher:/foo/com" in str(e.value) def test_file_url(self):