From bc5a3ccfec2c1caabe6e466c9730cfb3fd876929 Mon Sep 17 00:00:00 2001 From: Jon Duckworth Date: Tue, 22 Feb 2022 06:41:08 -0500 Subject: [PATCH 01/10] Update pystac.utils docs --- docs/api/utils.rst | 1 - docs/conf.py | 1 + pystac/utils.py | 170 +++++++++++++++++++++++++++++++++++---------- 3 files changed, 133 insertions(+), 39 deletions(-) diff --git a/docs/api/utils.rst b/docs/api/utils.rst index 4ce4f8d35..7417438ee 100644 --- a/docs/api/utils.rst +++ b/docs/api/utils.rst @@ -4,4 +4,3 @@ pystac.utils .. automodule:: pystac.utils :members: :undoc-members: - :noindex: diff --git a/docs/conf.py b/docs/conf.py index 3bdd208e8..d3dca1b0b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -231,6 +231,7 @@ intersphinx_mapping = { "python": ("https://docs.python.org/3", None), + "dateutil": ("https://dateutil.readthedocs.io/en/stable", None) } # -- Substutition variables diff --git a/pystac/utils.py b/pystac/utils.py index f79493db8..cc8af9c33 100644 --- a/pystac/utils.py +++ b/pystac/utils.py @@ -13,10 +13,18 @@ def safe_urlparse(href: str) -> URLParseResult: - """Version of URL parse that takes into account windows paths. + """Wrapper around :func:`urllib.parse.urlparse` that returns consistent results for + both Windows and UNIX file paths. - A windows absolute path will be parsed with a scheme from urllib.parse.urlparse. - This method will take this into account. + For Windows paths, this function will include the drive prefix (e.g. ``"D:\\"``) as + part of the ``path`` of the :class:`urllib.parse.ParseResult` rather than as the + ``scheme`` for consistency with handling of UNIX/LINUX file paths. + + Args: + href (str) : The HREF to parse. May be a local file path or URL. + + Returns: + urllib.parse.ParseResult : The named tuple representing the parsed HREF. """ parsed = urlparse(href) if parsed.scheme != "" and href.lower().startswith("{}:\\".format(parsed.scheme)): @@ -38,19 +46,23 @@ def safe_urlparse(href: str) -> URLParseResult: class StringEnum(str, Enum): - """Base Enum class for string enums that will serialize as the string value.""" + """Base :class:`enum.Enum` class for string enums that will serialize as the string value.""" def __str__(self) -> str: return cast(str, self.value) class JoinType(StringEnum): - """Allowed join types for the :func:`_join` function.""" + """Allowed join types for :func:`~pystac.utils.join_path_or_url`.""" @staticmethod def from_parsed_uri(parsed_uri: URLParseResult) -> "JoinType": """Determines the appropriate join type based on the scheme of the parsed - result.""" + result. + + Args: + parsed_uri (urllib.parse.ParseResult) : A named tuple representing the parsed URI. + """ if parsed_uri.scheme == "": return JoinType.PATH else: @@ -61,8 +73,19 @@ def from_parsed_uri(parsed_uri: URLParseResult) -> "JoinType": def join_path_or_url(join_type: JoinType, *args: str) -> str: - """Version of os.path.join that takes into account whether or not we are working - with a URL.""" + """Functions similarly to :func:`os.path.join`, but can be used to join either a + local file path or a URL. + + Args: + join_type (JoinType) : One of ``JoinType.PATH`` or ``JoinType.URL``. If + ``JoinType.PATH``, then :func:`os.path.join` is used for the join. + If ``JoinType.URL``, then :func:`posixpath.join` is used. + *args (str): Additional positional string arguments to be joined. + + Returns: + str : The joined path + + """ if join_type == JoinType.PATH: return _pathlib.join(*args) @@ -127,19 +150,21 @@ def _make_relative_href_path( def make_relative_href( source_href: str, start_href: str, start_is_dir: bool = False ) -> str: - """Makes a given HREF relative to the given starting HREF. + """Returns a new string that represents the ``source_href`` as a path relative to + ``start_href``. If ``source_href`` and ``start_href`` do not share a common parent, + then ``source_href`` is returned unchanged. + + May be used on either local file paths or URLs. Args: source_href : The HREF to make relative. - start_href : The HREF that the resulting HREF will be relative with - respect to. - start_is_dir : If True, the start_href is treated as a directory. - Otherwise, the start_href is considered to be a file HREF. - Defaults to False. + start_href : The HREF that the resulting HREF will be relative to. + start_is_dir : If ``True``, ``start_href`` is treated as a directory. + Otherwise, ``start_href`` is considered to be a path to a file. Defaults to + ``False``. Returns: - str: The relative HREF. If the source_href and start_href do not share a common - parent, then source_href will be returned unchanged. + str: The relative HREF. """ parsed_source = safe_urlparse(source_href) @@ -219,20 +244,24 @@ def _make_absolute_href_path( def make_absolute_href( source_href: str, start_href: Optional[str] = None, start_is_dir: bool = False ) -> str: - """Makes a given HREF absolute based on the given starting HREF. + """Returns a new string that represents ``source_href`` as an absolute path. If + ``source_href`` is already absolute it is returned unchanged. If ``source_href`` + is relative, the absolute HREF is constructed by joining ``source_href`` to + ``start_href``. + + May be used on either local file paths or URLs. Args: source_href : The HREF to make absolute. - start_href : The HREF that will be used as the basis for which to resolve - relative paths, if source_href is a relative path. Defaults to the - current working directory. - start_is_dir : If True, the start_href is treated as a directory. - Otherwise, the start_href is considered to be a file HREF. - Defaults to False. + start_href : The HREF that will be used as the basis for resolving relative + paths, if ``source_href`` is a relative path. Defaults to the current + working directory. + start_is_dir : If ``True``, ``start_href`` is treated as a directory. + Otherwise, ``start_href`` is considered to be a path to a file. Defaults to + ``False``. Returns: - str: The absolute HREF. If the source_href is already an absolute href, - then it will be returned unchanged. + str: The absolute HREF. """ if start_href is None: start_href = os.getcwd() @@ -253,24 +282,29 @@ def make_absolute_href( def is_absolute_href(href: str) -> bool: """Determines if an HREF is absolute or not. + May be used on either local file paths or URLs. + Args: href : The HREF to consider. Returns: - bool: True if the given HREF is absolute, False if it is relative. + bool: ``True`` if the given HREF is absolute, ``False`` if it is relative. """ parsed = safe_urlparse(href) return parsed.scheme != "" or _pathlib.isabs(parsed.path) def datetime_to_str(dt: datetime) -> str: - """Convert a python datetime to an ISO8601 string + """Converts a :class:`datetime.datetime` instance to an ISO8601 string in the + `RFC 3339, section 5.6 `__ + format required by the :stac-spec:`STAC Spec + `. Args: dt : The datetime to convert. Returns: - str: The ISO8601 formatted string representing the datetime. + str: The ISO8601 (RFC 3339) formatted string representing the datetime. """ if dt.tzinfo is None: dt = dt.replace(tzinfo=timezone.utc) @@ -284,6 +318,13 @@ def datetime_to_str(dt: datetime) -> str: def str_to_datetime(s: str) -> datetime: + """Converts a string timestamp to a :class:`datetime.datetime` instance using + :meth:`dateutil.parser.parse` under the hood. The input string may be in any + format :std:doc:`supported by the parser `. + + Args: + s (str) : The string to conver to :class:`datetime.datetime`. + """ return dateutil.parser.parse(s) @@ -337,23 +378,62 @@ def extract_coords(coords: List[Union[List[float], List[List[Any]]]]) -> None: def map_opt(fn: Callable[[T], U], v: Optional[T]) -> Optional[U]: - """Maps the value of an option to another value, returning - None if the input option is None. + """Maps the value of an optional type to another value, returning + ``None`` if the input option is ``None``. + + Examples: + + Given an optional value like the following... + + .. code-block:: python + + maybe_item: Optional[pystac.Item] = ... + + ...you could replace... + + .. code-block:: python + + maybe_item_id: Optional[str] = None + if maybe_item is not None: + maybe_item_id = maybe_item.id + + ...with: + + .. code-block:: python + + maybe_item_id = map_opt(lambda item: item.id, maybe_item) """ return v if v is None else fn(v) def get_opt(option: Optional[T]) -> T: - """Retrieves the value of the Optional type. + """Retrieves the value of the ``Optional`` type, raising a :exc:`ValueError` if + the value is ``None``. - If the Optional is None, this will raise a value error. Use this to get a properly typed value from an optional - in contexts where you can be certain the value is not None. - If there is potential for a non-None value, it's best to handle - the None case of the optional instead of using this method. + in contexts where you can be certain the value is not ``None``. + If there is potential for a non-``None`` value, it's best to handle + the ``None`` case of the optional instead of using this method. + + Args: + option (Optional[T]) : Some ``Optional`` value Returns: The value of type T wrapped by the Optional[T] + + Examples: + + .. code-block:: python + + d = { + "some_key": "some_value" + } + + # This passes + val: str = map_opt(d.get("some_key")) + + # This raises a ValueError + val: str = map_opt(d.get("does_not_exist")) """ if option is None: raise ValueError("Cannot get value from None") @@ -361,9 +441,23 @@ def get_opt(option: Optional[T]) -> T: def get_required(option: Optional[T], obj: Union[str, Any], prop: str) -> T: - """Retrieves an optional value that comes from a required property. - If the option is None, throws an RequiredPropertyError with - the given obj and property + """Retrieves an ``Optional`` value that comes from a required property of some + object. If the option is ``None``, throws an :exc:`pystac.RequiredPropertyMissing` + with the given obj and property. + + This method is primarily used internally to retrieve properly typed required + properties from dictionaries. For an example usage, see the + :attr:`pystac.extensions.eo.Band.name` source code. + + Args: + option (Optional[T]) : The ``Optional`` value. + obj (str, Any) : The object from which the value is being retrieved. This will + be passed to the :exc:`~pystac.RequiredPropertyMissing` exception if + ``option`` is ``None``. + prop (str) : The name of the property being retrieved. + + Returns: + T : The properly typed, non-``None`` value. """ if option is None: raise RequiredPropertyMissing(obj, prop) From d458827ad283185a86e6497c7e4a082243badd5e Mon Sep 17 00:00:00 2001 From: Jon Duckworth Date: Tue, 22 Feb 2022 06:41:08 -0500 Subject: [PATCH 02/10] Fix lint errors --- docs/conf.py | 2 +- pystac/utils.py | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index d3dca1b0b..65f2f805c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -231,7 +231,7 @@ intersphinx_mapping = { "python": ("https://docs.python.org/3", None), - "dateutil": ("https://dateutil.readthedocs.io/en/stable", None) + "dateutil": ("https://dateutil.readthedocs.io/en/stable", None), } # -- Substutition variables diff --git a/pystac/utils.py b/pystac/utils.py index cc8af9c33..775e40993 100644 --- a/pystac/utils.py +++ b/pystac/utils.py @@ -46,7 +46,8 @@ def safe_urlparse(href: str) -> URLParseResult: class StringEnum(str, Enum): - """Base :class:`enum.Enum` class for string enums that will serialize as the string value.""" + """Base :class:`enum.Enum` class for string enums that will serialize as the string + value.""" def __str__(self) -> str: return cast(str, self.value) @@ -59,9 +60,10 @@ class JoinType(StringEnum): def from_parsed_uri(parsed_uri: URLParseResult) -> "JoinType": """Determines the appropriate join type based on the scheme of the parsed result. - + Args: - parsed_uri (urllib.parse.ParseResult) : A named tuple representing the parsed URI. + parsed_uri (urllib.parse.ParseResult) : A named tuple representing the + parsed URI. """ if parsed_uri.scheme == "": return JoinType.PATH @@ -75,7 +77,7 @@ def from_parsed_uri(parsed_uri: URLParseResult) -> "JoinType": def join_path_or_url(join_type: JoinType, *args: str) -> str: """Functions similarly to :func:`os.path.join`, but can be used to join either a local file path or a URL. - + Args: join_type (JoinType) : One of ``JoinType.PATH`` or ``JoinType.URL``. If ``JoinType.PATH``, then :func:`os.path.join` is used for the join. @@ -164,7 +166,7 @@ def make_relative_href( ``False``. Returns: - str: The relative HREF. + str: The relative HREF. """ parsed_source = safe_urlparse(source_href) @@ -295,10 +297,10 @@ def is_absolute_href(href: str) -> bool: def datetime_to_str(dt: datetime) -> str: - """Converts a :class:`datetime.datetime` instance to an ISO8601 string in the - `RFC 3339, section 5.6 `__ - format required by the :stac-spec:`STAC Spec - `. + """Converts a :class:`datetime.datetime` instance to an ISO8601 string in the + `RFC 3339, section 5.6 + `__ format required by + the :stac-spec:`STAC Spec `. Args: dt : The datetime to convert. @@ -323,7 +325,7 @@ def str_to_datetime(s: str) -> datetime: format :std:doc:`supported by the parser `. Args: - s (str) : The string to conver to :class:`datetime.datetime`. + s (str) : The string to convert to :class:`datetime.datetime`. """ return dateutil.parser.parse(s) @@ -396,7 +398,7 @@ def map_opt(fn: Callable[[T], U], v: Optional[T]) -> Optional[U]: maybe_item_id: Optional[str] = None if maybe_item is not None: maybe_item_id = maybe_item.id - + ...with: .. code-block:: python From f9d27f002292e4042a2baa5cabb42b68975d8b3f Mon Sep 17 00:00:00 2001 From: Jon Duckworth Date: Tue, 22 Feb 2022 06:41:54 -0500 Subject: [PATCH 03/10] Add CHANGELOG entry for #735 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a059ffecf..0801962cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Accept PathLike objects in `StacIO` I/O methods, `pystac.read_file` and `pystac.write_file` ([#728](https://github.com/stac-utils/pystac/pull/728)) - Support for Storage Extension ([#745](https://github.com/stac-utils/pystac/pull/745)) - Optional `StacIO` instance as argument to `Catalog.save`/`Catalog.normalize_and_save` ([#751](https://github.com/stac-utils/pystac/pull/751)) +- More thorough docstrings for `pystac.utils` functions and classes ([#735](https://github.com/stac-utils/pystac/pull/735)) ### Removed From 536e4d183bf703b2efce5116c1563705b4233b81 Mon Sep 17 00:00:00 2001 From: Jon Duckworth Date: Tue, 22 Feb 2022 06:48:31 -0500 Subject: [PATCH 04/10] Add missing docs sections --- pystac/utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pystac/utils.py b/pystac/utils.py index 775e40993..4de73034e 100644 --- a/pystac/utils.py +++ b/pystac/utils.py @@ -64,6 +64,9 @@ def from_parsed_uri(parsed_uri: URLParseResult) -> "JoinType": Args: parsed_uri (urllib.parse.ParseResult) : A named tuple representing the parsed URI. + + Returns: + JoinType : The join type for the URI. """ if parsed_uri.scheme == "": return JoinType.PATH @@ -383,6 +386,12 @@ def map_opt(fn: Callable[[T], U], v: Optional[T]) -> Optional[U]: """Maps the value of an optional type to another value, returning ``None`` if the input option is ``None``. + Args: + fn (Callable) : A function that maps the non-optional value of type ``T`` to + another value. This function will be called on non-``None`` values of + ``v``. + v (Optional[T]) : The optional value to map. + Examples: Given an optional value like the following... @@ -432,10 +441,10 @@ def get_opt(option: Optional[T]) -> T: } # This passes - val: str = map_opt(d.get("some_key")) + val: str = get_opt(d.get("some_key")) # This raises a ValueError - val: str = map_opt(d.get("does_not_exist")) + val: str = get_opt(d.get("does_not_exist")) """ if option is None: raise ValueError("Cannot get value from None") From 3162e452a61f173bca7013637a405b3891e7ef75 Mon Sep 17 00:00:00 2001 From: Jon Duckworth Date: Tue, 22 Feb 2022 06:52:14 -0500 Subject: [PATCH 05/10] Fix duplicate object warnings when building docs --- docs/api/serialization/identify.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/api/serialization/identify.rst b/docs/api/serialization/identify.rst index adbff0a9e..97451bf51 100644 --- a/docs/api/serialization/identify.rst +++ b/docs/api/serialization/identify.rst @@ -5,3 +5,4 @@ pystac.serialization.identify :members: :undoc-members: :show-inheritance: + :noindex: From c4617fc3ee95f57c7df8da384c645a60ccfdf3fe Mon Sep 17 00:00:00 2001 From: Jon Duckworth Date: Tue, 22 Feb 2022 07:05:22 -0500 Subject: [PATCH 06/10] Fix lint errors --- pystac/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pystac/utils.py b/pystac/utils.py index 4de73034e..8c1d4fd04 100644 --- a/pystac/utils.py +++ b/pystac/utils.py @@ -64,7 +64,7 @@ def from_parsed_uri(parsed_uri: URLParseResult) -> "JoinType": Args: parsed_uri (urllib.parse.ParseResult) : A named tuple representing the parsed URI. - + Returns: JoinType : The join type for the URI. """ From 14c075fe33ef4c9ad0b2e5072c1804f039ee05c3 Mon Sep 17 00:00:00 2001 From: Jon Duckworth Date: Tue, 22 Feb 2022 10:06:45 -0500 Subject: [PATCH 07/10] Replace hard-coded extension links with extlinks reference --- pystac/extensions/datacube.py | 5 +---- pystac/extensions/eo.py | 5 +---- pystac/extensions/file.py | 5 +---- pystac/extensions/item_assets.py | 5 +---- pystac/extensions/label.py | 5 +---- pystac/extensions/pointcloud.py | 5 +---- pystac/extensions/projection.py | 5 +---- pystac/extensions/raster.py | 13 +++++-------- pystac/extensions/sar.py | 5 +---- pystac/extensions/sat.py | 5 +---- pystac/extensions/scientific.py | 4 +--- pystac/extensions/table.py | 5 +---- pystac/extensions/timestamps.py | 5 +---- pystac/extensions/version.py | 9 +++------ pystac/extensions/view.py | 5 +---- 15 files changed, 21 insertions(+), 65 deletions(-) diff --git a/pystac/extensions/datacube.py b/pystac/extensions/datacube.py index 2758b2d4e..d02264a64 100644 --- a/pystac/extensions/datacube.py +++ b/pystac/extensions/datacube.py @@ -1,7 +1,4 @@ -"""Implements the Datacube extension. - -https://github.com/stac-extensions/datacube -""" +"""Implements the :stac-ext:`Datacube Extension `.""" from abc import ABC from typing import Any, Dict, Generic, List, Optional, TypeVar, Union, cast diff --git a/pystac/extensions/eo.py b/pystac/extensions/eo.py index 3ad48c96f..12699eaf7 100644 --- a/pystac/extensions/eo.py +++ b/pystac/extensions/eo.py @@ -1,7 +1,4 @@ -"""Implements the Electro-Optical (EO) extension. - -https://github.com/stac-extensions/eo -""" +"""Implements the :stac-ext:`Electro-Optical Extension `.""" from typing import ( Any, diff --git a/pystac/extensions/file.py b/pystac/extensions/file.py index 02d8b428b..63840cfd7 100644 --- a/pystac/extensions/file.py +++ b/pystac/extensions/file.py @@ -1,7 +1,4 @@ -"""Implements the File extension. - -https://github.com/stac-extensions/file -""" +"""Implements the :stac-ext:`File Info Extension `.""" from typing import Any, Dict, Iterable, List, Optional, Union diff --git a/pystac/extensions/item_assets.py b/pystac/extensions/item_assets.py index a37197e23..c92b1282f 100644 --- a/pystac/extensions/item_assets.py +++ b/pystac/extensions/item_assets.py @@ -1,7 +1,4 @@ -"""Implements the Item Assets Definition extension. - -https://github.com/stac-extensions/item-assets -""" +"""Implements the :stac-ext:`Item Assets Definition Extension `.""" from copy import deepcopy from typing import Any, Dict, List, Optional diff --git a/pystac/extensions/label.py b/pystac/extensions/label.py index cdaed7b47..578257e72 100644 --- a/pystac/extensions/label.py +++ b/pystac/extensions/label.py @@ -1,7 +1,4 @@ -"""Implements the Label extension. - -https://github.com/stac-extensions/label -""" +"""Implements the :stac-ext:`Label Extension