From 04a69953c9afe3e80d77507001f1bb02dc942118 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 15 Dec 2021 09:50:15 -0600 Subject: [PATCH 01/10] fix: pass usedforsecurity to hashlib.md5 to fix error on FIPS-enabled systems --- starlette/_crypto.py | 14 ++++++++++++++ starlette/responses.py | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 starlette/_crypto.py diff --git a/starlette/_crypto.py b/starlette/_crypto.py new file mode 100644 index 000000000..a9a36a1b2 --- /dev/null +++ b/starlette/_crypto.py @@ -0,0 +1,14 @@ +import hashlib +import inspect + + +# TODO: Remove when dropping support for PY38. inspect.signature() is used to +# detect whether the usedforsecurity argument is available as this fix may also +# have been applied by downstream package maintainers to other versions in +# their repositories. +if "usedforsecurity" in inspect.signature(hashlib.md5).parameters: + def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: + return hashlib.md5(data, usedforsecurity=usedforsecurity).hexdigest() +else: + def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: + return hashlib.md5(data).hexdigest() diff --git a/starlette/responses.py b/starlette/responses.py index 1f9c43a21..b12cea738 100644 --- a/starlette/responses.py +++ b/starlette/responses.py @@ -1,4 +1,3 @@ -import hashlib import http.cookies import json import os @@ -12,6 +11,7 @@ import anyio +from starlette._crypto import md5_hexdigest from starlette.background import BackgroundTask from starlette.concurrency import iterate_in_threadpool from starlette.datastructures import URL, MutableHeaders @@ -287,7 +287,7 @@ def set_stat_headers(self, stat_result: os.stat_result) -> None: content_length = str(stat_result.st_size) last_modified = formatdate(stat_result.st_mtime, usegmt=True) etag_base = str(stat_result.st_mtime) + "-" + str(stat_result.st_size) - etag = hashlib.md5(etag_base.encode()).hexdigest() + etag = md5_hexdigest(etag_base.encode(), usedforsecurity=False) self.headers.setdefault("content-length", content_length) self.headers.setdefault("last-modified", last_modified) From 05b0819e65ee650283bbae073d8c001bb07b6e17 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 15 Dec 2021 10:01:27 -0600 Subject: [PATCH 02/10] rename module to _compat --- starlette/_crypto.py | 14 -------------- starlette/responses.py | 2 +- 2 files changed, 1 insertion(+), 15 deletions(-) delete mode 100644 starlette/_crypto.py diff --git a/starlette/_crypto.py b/starlette/_crypto.py deleted file mode 100644 index a9a36a1b2..000000000 --- a/starlette/_crypto.py +++ /dev/null @@ -1,14 +0,0 @@ -import hashlib -import inspect - - -# TODO: Remove when dropping support for PY38. inspect.signature() is used to -# detect whether the usedforsecurity argument is available as this fix may also -# have been applied by downstream package maintainers to other versions in -# their repositories. -if "usedforsecurity" in inspect.signature(hashlib.md5).parameters: - def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: - return hashlib.md5(data, usedforsecurity=usedforsecurity).hexdigest() -else: - def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: - return hashlib.md5(data).hexdigest() diff --git a/starlette/responses.py b/starlette/responses.py index b12cea738..ffde4b97d 100644 --- a/starlette/responses.py +++ b/starlette/responses.py @@ -11,7 +11,7 @@ import anyio -from starlette._crypto import md5_hexdigest +from starlette._compat import md5_hexdigest from starlette.background import BackgroundTask from starlette.concurrency import iterate_in_threadpool from starlette.datastructures import URL, MutableHeaders From c9568778fb9088c6691021678feeb1f29befbb2c Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 15 Dec 2021 10:03:07 -0600 Subject: [PATCH 03/10] add missing file --- starlette/_compat.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 starlette/_compat.py diff --git a/starlette/_compat.py b/starlette/_compat.py new file mode 100644 index 000000000..a9a36a1b2 --- /dev/null +++ b/starlette/_compat.py @@ -0,0 +1,14 @@ +import hashlib +import inspect + + +# TODO: Remove when dropping support for PY38. inspect.signature() is used to +# detect whether the usedforsecurity argument is available as this fix may also +# have been applied by downstream package maintainers to other versions in +# their repositories. +if "usedforsecurity" in inspect.signature(hashlib.md5).parameters: + def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: + return hashlib.md5(data, usedforsecurity=usedforsecurity).hexdigest() +else: + def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: + return hashlib.md5(data).hexdigest() From ece8cd1014b3d949f50d1b6c7992dbbf65756539 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 15 Dec 2021 10:45:33 -0600 Subject: [PATCH 04/10] lint --- starlette/_compat.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/starlette/_compat.py b/starlette/_compat.py index a9a36a1b2..1e0b2996f 100644 --- a/starlette/_compat.py +++ b/starlette/_compat.py @@ -1,14 +1,17 @@ import hashlib import inspect - # TODO: Remove when dropping support for PY38. inspect.signature() is used to # detect whether the usedforsecurity argument is available as this fix may also # have been applied by downstream package maintainers to other versions in # their repositories. if "usedforsecurity" in inspect.signature(hashlib.md5).parameters: + def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: return hashlib.md5(data, usedforsecurity=usedforsecurity).hexdigest() + + else: + def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: return hashlib.md5(data).hexdigest() From 700deef6e4bf3cb4983c2207be551ce265f4dd0a Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 15 Dec 2021 11:09:55 -0600 Subject: [PATCH 05/10] add type ignore for mypy --- starlette/_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/starlette/_compat.py b/starlette/_compat.py index 1e0b2996f..e03955c31 100644 --- a/starlette/_compat.py +++ b/starlette/_compat.py @@ -8,7 +8,7 @@ if "usedforsecurity" in inspect.signature(hashlib.md5).parameters: def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: - return hashlib.md5(data, usedforsecurity=usedforsecurity).hexdigest() + return hashlib.md5(data, usedforsecurity=usedforsecurity).hexdigest() # type: ignore[call-arg] else: From 0f9e5adc8d18878d27d42660f9653afa90efe7b4 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 15 Dec 2021 11:25:24 -0600 Subject: [PATCH 06/10] wrap line --- starlette/_compat.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/starlette/_compat.py b/starlette/_compat.py index e03955c31..e85a0c36c 100644 --- a/starlette/_compat.py +++ b/starlette/_compat.py @@ -8,7 +8,9 @@ if "usedforsecurity" in inspect.signature(hashlib.md5).parameters: def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: - return hashlib.md5(data, usedforsecurity=usedforsecurity).hexdigest() # type: ignore[call-arg] + return hashlib.md5( # type: ignore[call-arg] + data, usedforsecurity=usedforsecurity + ).hexdigest() else: From fbf37452308ce9a4ffea7dbdbee4a4f238a23d81 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 15 Dec 2021 11:32:19 -0600 Subject: [PATCH 07/10] work around openssl lacking signature --- starlette/_compat.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/starlette/_compat.py b/starlette/_compat.py index e85a0c36c..29d738b44 100644 --- a/starlette/_compat.py +++ b/starlette/_compat.py @@ -1,11 +1,12 @@ import hashlib -import inspect -# TODO: Remove when dropping support for PY38. inspect.signature() is used to +# TODO: Remove when dropping support for PY38. a try/except is used to # detect whether the usedforsecurity argument is available as this fix may also # have been applied by downstream package maintainers to other versions in # their repositories. -if "usedforsecurity" in inspect.signature(hashlib.md5).parameters: +try: + + hashlib.md5(b"data", usedforsecurity=True) # type: ignore[call-arg] def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: return hashlib.md5( # type: ignore[call-arg] @@ -13,7 +14,7 @@ def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: ).hexdigest() -else: +except TypeError: def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: return hashlib.md5(data).hexdigest() From 1fd2fccdff33e23314262d91ba2ca3628a163260 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 15 Dec 2021 11:36:52 -0600 Subject: [PATCH 08/10] add pragma: no cover --- starlette/_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/starlette/_compat.py b/starlette/_compat.py index 29d738b44..22086ec3b 100644 --- a/starlette/_compat.py +++ b/starlette/_compat.py @@ -14,7 +14,7 @@ def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: ).hexdigest() -except TypeError: +except TypeError: # pragma: no cover def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: return hashlib.md5(data).hexdigest() From 266dbaf65416f721765bca6108b9e2f244ebba97 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 15 Dec 2021 11:46:49 -0600 Subject: [PATCH 09/10] more pragmas --- starlette/_compat.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/starlette/_compat.py b/starlette/_compat.py index 22086ec3b..c89ced5f5 100644 --- a/starlette/_compat.py +++ b/starlette/_compat.py @@ -8,7 +8,9 @@ hashlib.md5(b"data", usedforsecurity=True) # type: ignore[call-arg] - def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: + def md5_hexdigest( + data: bytes, *, usedforsecurity: bool = True + ) -> str: # pragma: no cover return hashlib.md5( # type: ignore[call-arg] data, usedforsecurity=usedforsecurity ).hexdigest() From 7fb28ccf2583fb4b8976e7a5f19a48fe126548e1 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 16 Dec 2021 10:12:23 -0600 Subject: [PATCH 10/10] incorporate feedback on comment --- starlette/_compat.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/starlette/_compat.py b/starlette/_compat.py index c89ced5f5..82aa72f38 100644 --- a/starlette/_compat.py +++ b/starlette/_compat.py @@ -1,9 +1,14 @@ import hashlib -# TODO: Remove when dropping support for PY38. a try/except is used to -# detect whether the usedforsecurity argument is available as this fix may also -# have been applied by downstream package maintainers to other versions in -# their repositories. +# Compat wrapper to always include the `usedforsecurity=...` parameter, +# which is only added from Python 3.9 onwards. +# We use this flag to indicate that we use `md5` hashes only for non-security +# cases (our ETag checksums). +# If we don't indicate that we're using MD5 for non-security related reasons, +# then attempting to use this function will raise an error when used +# environments which enable a strict "FIPs mode". +# +# See issue: https://github.com/encode/starlette/issues/1365 try: hashlib.md5(b"data", usedforsecurity=True) # type: ignore[call-arg] @@ -15,7 +20,6 @@ def md5_hexdigest( data, usedforsecurity=usedforsecurity ).hexdigest() - except TypeError: # pragma: no cover def md5_hexdigest(data: bytes, *, usedforsecurity: bool = True) -> str: