Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify timestamp parsing #3063

Open
wants to merge 3 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 0 additions & 13 deletions botocore/compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@

from botocore.vendored import six
from botocore.exceptions import MD5UnavailableError
from dateutil.tz import tzlocal
from urllib3 import exceptions

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -273,18 +272,6 @@ def _windows_shell_split(s):
return components


def get_tzinfo_options():
# Due to dateutil/dateutil#197, Windows may fail to parse times in the past
# with the system clock. We can alternatively fallback to tzwininfo when
# this happens, which will get time info from the Windows registry.
if sys.platform == 'win32':
from dateutil.tz import tzwinlocal

return (tzlocal, tzwinlocal)
else:
return (tzlocal,)


# Detect if CRT is available for use
try:
import awscrt.auth
Expand Down
84 changes: 27 additions & 57 deletions botocore/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@
UNSAFE_URL_CHARS,
OrderedDict,
get_md5,
get_tzinfo_options,
json,
quote,
urlparse,
Expand Down Expand Up @@ -919,39 +918,27 @@ def percent_encode(input_str, safe=SAFE_CHARS):
return quote(input_str, safe=safe)


def _epoch_seconds_to_datetime(value, tzinfo):
_EPOCH_ZERO = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc())


def _epoch_seconds_to_datetime(value):
"""Parse numerical epoch timestamps (seconds since 1970) into a
``datetime.datetime`` in UTC using ``datetime.timedelta``. This is intended
as fallback when ``fromtimestamp`` raises ``OverflowError`` or ``OSError``.

:type value: float or int
:param value: The Unix timestamps as number.

:type tzinfo: callable
:param tzinfo: A ``datetime.tzinfo`` class or compatible callable.
"""
epoch_zero = datetime.datetime(1970, 1, 1, 0, 0, 0, tzinfo=tzutc())
epoch_zero_localized = epoch_zero.astimezone(tzinfo())
return epoch_zero_localized + datetime.timedelta(seconds=value)


def _parse_timestamp_with_tzinfo(value, tzinfo):
"""Parse timestamp with pluggable tzinfo options."""
if isinstance(value, (int, float)):
# Possibly an epoch time.
return datetime.datetime.fromtimestamp(value, tzinfo())
else:
try:
return datetime.datetime.fromtimestamp(float(value), tzinfo())
except (TypeError, ValueError):
pass
try:
# In certain cases, a timestamp marked with GMT can be parsed into a
# different time zone, so here we provide a context which will
# enforce that GMT == UTC.
return dateutil.parser.parse(value, tzinfos={'GMT': tzutc()})
except (TypeError, ValueError) as e:
raise ValueError(f'Invalid timestamp "{value}": {e}')
return datetime.datetime.fromtimestamp(value, tz=tzutc())
except (OverflowError, OSError):
# For numeric values attempt fallback to using fromtimestamp-free method.
# From Python's ``datetime.datetime.fromtimestamp`` documentation: "This
# may raise ``OverflowError``, if the timestamp is out of the range of
# values supported by the platform C localtime() function, and ``OSError``
# on localtime() failure. It's common for this to be restricted to years
# from 1970 through 2038."
return _EPOCH_ZERO + datetime.timedelta(seconds=value)


def parse_timestamp(value):
Expand All @@ -966,40 +953,23 @@ def parse_timestamp(value):
This will return a ``datetime.datetime`` object.

"""
tzinfo_options = get_tzinfo_options()
for tzinfo in tzinfo_options:
try:
return _parse_timestamp_with_tzinfo(value, tzinfo)
except (OSError, OverflowError) as e:
logger.debug(
'Unable to parse timestamp with "%s" timezone info.',
tzinfo.__name__,
exc_info=e,
)
# For numeric values attempt fallback to using fromtimestamp-free method.
# From Python's ``datetime.datetime.fromtimestamp`` documentation: "This
# may raise ``OverflowError``, if the timestamp is out of the range of
# values supported by the platform C localtime() function, and ``OSError``
# on localtime() failure. It's common for this to be restricted to years
# from 1970 through 2038."
if isinstance(value, (int, float)):
# Possibly an epoch time.
return _epoch_seconds_to_datetime(value)

# Possibly something we can cast to an epoch time and convert.
try:
numeric_value = float(value)
return _epoch_seconds_to_datetime(float(value))
except (TypeError, ValueError):
pass
else:
try:
for tzinfo in tzinfo_options:
return _epoch_seconds_to_datetime(numeric_value, tzinfo=tzinfo)
except (OSError, OverflowError) as e:
logger.debug(
'Unable to parse timestamp using fallback method with "%s" '
'timezone info.',
tzinfo.__name__,
exc_info=e,
)
raise RuntimeError(
'Unable to calculate correct timezone offset for "%s"' % value
)

try:
# In certain cases, a timestamp marked with GMT can be parsed into a
# different time zone, so here we provide a context which will
# enforce that GMT == UTC.
return dateutil.parser.parse(value, tzinfos={'GMT': tzutc()})
except (TypeError, ValueError) as e:
raise ValueError(f'Invalid timestamp "{value}": {e}')


def parse_to_aware_datetime(value):
Expand Down
10 changes: 0 additions & 10 deletions tests/unit/test_compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
compat_shell_split,
ensure_bytes,
get_md5,
get_tzinfo_options,
total_seconds,
unquote_str,
)
Expand Down Expand Up @@ -208,15 +207,6 @@ def assert_raises(self, s, exception_cls, platform):
compat_shell_split(s, platform)


class TestTimezoneOperations(unittest.TestCase):
def test_get_tzinfo_options(self):
options = get_tzinfo_options()
self.assertTrue(len(options) > 0)

for tzinfo in options:
self.assertIsInstance(tzinfo(), datetime.tzinfo)


class TestCRTIntegration(unittest.TestCase):
def test_has_crt_global(self):
try:
Expand Down
12 changes: 0 additions & 12 deletions tests/unit/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -466,18 +466,6 @@ def test_parse_invalid_timestamp(self):
with self.assertRaises(ValueError):
parse_timestamp('invalid date')

def test_parse_timestamp_fails_with_bad_tzinfo(self):
mock_tzinfo = mock.Mock()
mock_tzinfo.__name__ = 'tzinfo'
mock_tzinfo.side_effect = OSError()
mock_get_tzinfo_options = mock.MagicMock(return_value=(mock_tzinfo,))

with mock.patch(
'botocore.utils.get_tzinfo_options', mock_get_tzinfo_options
):
with self.assertRaises(RuntimeError):
parse_timestamp(0)

@contextmanager
def mocked_fromtimestamp_that_raises(self, exception_type):
class MockDatetime(datetime.datetime):
Expand Down