Merge branch 'main' into conti/fix-grpc-pin

.gitlab/benchmarks.yml

@@ -103,10 +103,11 @@ appsec-iast-propagation:
   variables:
     SCENARIO: "appsec_iast_propagation"
 
-benchmark-encoder:
-  extends: .benchmarks
-  variables:
-    SCENARIO: "encoder"
+# Flaky benchmark test. Timeout errors
+#benchmark-encoder:
+#  extends: .benchmarks
+#  variables:
+#    SCENARIO: "encoder"
 
 benchmark-http-propagation-extract:
   extends: .benchmarks

.riot/requirements/60a424f.txt → .riot/requirements/1c9c221.txt

@@ -2,53 +2,55 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --no-annotate .riot/requirements/60a424f.in
+#    pip-compile --no-annotate .riot/requirements/1c9c221.in
 #
 aiosqlite==0.17.0
+annotated-types==0.6.0
 attrs==23.2.0
-blinker==1.7.0
+blinker==1.8.2
 bytecode==0.15.1
 cattrs==22.2.0
 certifi==2024.2.2
 charset-normalizer==3.3.2
 click==8.1.7
-coverage[toml]==7.4.4
+coverage[toml]==7.5.1
 ddsketch==3.0.1
 deprecated==1.2.14
 envier==0.5.1
 flask==3.0.3
 greenlet==3.0.3
 hypothesis==6.45.0
-idna==3.6
+idna==3.7
 importlib-metadata==7.0.0
 iniconfig==2.0.0
 iso8601==1.1.0
-itsdangerous==2.1.2
-jinja2==3.1.3
+itsdangerous==2.2.0
+jinja2==3.1.4
 markupsafe==2.1.5
 mock==5.1.0
 opentelemetry-api==1.24.0
 opentracing==2.4.0
 packaging==24.0
-peewee==3.17.1
-pluggy==1.4.0
+peewee==3.17.3
+pluggy==1.5.0
 pony==0.7.17
 protobuf==5.26.1
 pycryptodome==3.20.0
+pydantic==2.7.1
+pydantic-core==2.18.2
 pypika-tortoise==0.1.6
-pytest==8.1.1
+pytest==8.2.0
 pytest-cov==5.0.0
 pytest-mock==3.14.0
 pytz==2024.1
 requests==2.31.0
 six==1.16.0
 sortedcontainers==2.4.0
-sqlalchemy==2.0.29
-sqlparse==0.4.4
-tortoise-orm==0.20.0
+sqlalchemy==2.0.30
+tortoise-orm==0.20.1
 typing-extensions==4.11.0
 urllib3==2.2.1
-werkzeug==3.0.2
+werkzeug==3.0.3
 wrapt==1.16.0
 xmltodict==0.13.0
 zipp==3.18.1

ddtrace/appsec/_handlers.py

@@ -1,3 +1,4 @@
+from collections.abc import MutableMapping
 import functools
 import io
 import json
@@ -19,6 +20,13 @@
 from ddtrace.vendor.wrapt import wrap_function_wrapper as _w
 
 
+MessageMapContainer = None
+try:
+    from google._upb._message import MessageMapContainer  # type: ignore[no-redef]
+except ImportError:
+    pass
+
+
 log = get_logger(__name__)
 _BODY_METHODS = {"POST", "PUT", "DELETE", "PATCH"}
 
@@ -359,10 +367,6 @@ def _on_django_patch():
 
 
 def _custom_protobuf_getattribute(self, name):
-    from collections.abc import MutableMapping
-
-    from google._upb._message import MessageMapContainer
-
     from ddtrace.appsec._iast._taint_tracking import taint_pyobject
     from ddtrace.appsec._iast._taint_tracking._native.taint_tracking import OriginType
     from ddtrace.appsec._iast._taint_utils import taint_structure
@@ -375,7 +379,7 @@ def _custom_protobuf_getattribute(self, name):
             source_value=ret,
             source_origin=OriginType.GRPC_BODY,
         )
-    elif isinstance(ret, MutableMapping):
+    elif MessageMapContainer is not None and isinstance(ret, MutableMapping):
         if isinstance(ret, MessageMapContainer) and len(ret):
             # Patch the message-values class
             first_key = next(iter(ret))
@@ -398,10 +402,14 @@ def _patch_protobuf_class(cls):
         return
 
     if not hasattr(getattr_method, "__datadog_custom"):
-        # Replace the class __getattribute__ method with our custom one
-        # (replacement is done at the class level because it would incur on a recursive loop with the instance)
-        cls.__saved_getattr = getattr_method
-        cls.__getattribute__ = _custom_protobuf_getattribute
+        try:
+            # Replace the class __getattribute__ method with our custom one
+            # (replacement is done at the class level because it would incur on a recursive loop with the instance)
+            cls.__saved_getattr = getattr_method
+            cls.__getattribute__ = _custom_protobuf_getattribute
+        except TypeError:
+            # Avoid failing on Python 3.12 while patching immutable types
+            pass
 
 
 def _on_grpc_response(response):

ddtrace/appsec/_iast/__init__.py

@@ -22,7 +22,6 @@ class MyVulnerability(VulnerabilityBase):
 
 @WeakHash.wrap
 def wrapped_function(wrapped, instance, args, kwargs):
-    # type: (Callable, str, Any, Any, Any) -> Any
     WeakHash.report(
         evidence_value=evidence,
     )

ddtrace/appsec/_iast/_ast/__init__.py

@@ -1 +0,0 @@
-#!/usr/bin/env python3

ddtrace/appsec/_iast/_ast/ast_patching.py

@@ -6,13 +6,11 @@
 import re
 from sys import builtin_module_names
 from types import ModuleType
-from typing import TYPE_CHECKING  # noqa:F401
+from typing import Optional
+from typing import Set
+from typing import Text
 from typing import Tuple
 
-
-if TYPE_CHECKING:
-    from typing import Optional  # noqa:F401
-
 from ddtrace.appsec._constants import IAST
 from ddtrace.appsec._python_info.stdlib import _stdlib_for_python_version
 from ddtrace.internal.logger import get_logger
@@ -22,8 +20,8 @@
 
 
 # Prefixes for modules where IAST patching is allowed
-IAST_ALLOWLIST = ("tests.appsec.iast",)  # type: tuple[str, ...]
-IAST_DENYLIST = (
+IAST_ALLOWLIST: Tuple[Text, ...] = ("tests.appsec.iast",)
+IAST_DENYLIST: Tuple[Text, ...] = (
     "ddtrace",
     "pkg_resources",
     "encodings",  # this package is used to load encodings when a module is imported, propagation is not needed
@@ -32,7 +30,7 @@
     "Crypto",  # This module is patched by the IAST patch methods, propagation is not needed
     "api_pb2",  # Patching crashes with these auto-generated modules, propagation is not needed
     "api_pb2_grpc",  # ditto
-)  # type: tuple[str, ...]
+)
 
 
 if IAST.PATCH_MODULES in os.environ:
@@ -47,7 +45,7 @@
 log = get_logger(__name__)
 
 
-def get_encoding(module_path):  # type: (str) -> str
+def get_encoding(module_path: Text) -> Text:
     """
     First tries to detect the encoding for the file,
     otherwise, returns global encoding default
@@ -67,7 +65,7 @@ def get_encoding(module_path):  # type: (str) -> str
     import importlib_metadata as il_md  # type: ignore[no-redef]
 
 
-def _build_installed_package_names_list():  # type: (...) -> set[str]
+def _build_installed_package_names_list() -> Set[Text]:
     return {
         ilmd_d.metadata["name"] for ilmd_d in il_md.distributions() if ilmd_d is not None and ilmd_d.files is not None
     }
@@ -78,11 +76,11 @@ def _build_installed_package_names_list():  # type: (...) -> set[str]
 )
 
 
-def _in_python_stdlib_or_third_party(module_name):  # type: (str) -> bool
+def _in_python_stdlib_or_third_party(module_name: str) -> bool:
     return module_name.split(".")[0].lower() in [x.lower() for x in _NOT_PATCH_MODULE_NAMES]
 
 
-def _should_iast_patch(module_name):  # type: (str) -> bool
+def _should_iast_patch(module_name: Text) -> bool:
     """
     select if module_name should be patch from the longuest prefix that match in allow or deny list.
     if a prefix is in both list, deny is selected.
@@ -94,10 +92,10 @@ def _should_iast_patch(module_name):  # type: (str) -> bool
 
 
 def visit_ast(
-    source_text,  # type: str
-    module_path,  # type: str
-    module_name="",  # type: str
-):  # type: (...) -> Optional[str]
+    source_text: Text,
+    module_path: Text,
+    module_name: Text = "",
+) -> Optional[str]:
     parsed_ast = ast.parse(source_text, module_path)
 
     visitor = AstVisitor(
@@ -116,7 +114,7 @@ def visit_ast(
 _FLASK_INSTANCE_REGEXP = re.compile(r"(\S*)\s*=.*Flask\(.*")
 
 
-def _remove_flask_run(text):  # type (str) -> str
+def _remove_flask_run(text: Text) -> Text:
     """
     Find and remove flask app.run() call. This is used for patching
     the app.py file and exec'ing to replace the module without creating