Merge remote-tracking branch 'upstream/main' into feature/unit-test-w…

…ith-k8s-skaffold

* upstream/main:
  bump python versions and use docker slim versions (elastic#1661)
  use github actions for pre-commit checks (elastic#1658)
  Use typing.TypeVar on decorators' type hints (elastic#1655)
  Use minimum of 5 seconds for interval between central config calls (elastic#1652)
  fix errors in pymongo tests introduced in elastic#1639 (elastic#1648)
  only use content-type and content-encoding headers for POSTing events (elastic#1651)
  fix starlette 0.21 tests (elastic#1653)

.ci/.jenkins_framework.yml

@@ -43,7 +43,7 @@ FRAMEWORK:
   - aiopg-newest
   - asyncpg-newest
   - tornado-newest
-  # - starlette-newest # disabled for now, see https://github.com/elastic/apm-agent-python/issues/1172
+  - starlette-newest
   - pymemcache-newest
   - graphene-2
   - httpx-newest

.github/workflows/pre-commit.yml

@@ -0,0 +1,14 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v3
+    - uses: pre-commit/action@v3.0.0

.pre-commit-config.yaml

@@ -1,15 +1,16 @@
 repos:
 -   repo: https://github.com/pycqa/isort
-    rev: 5.7.0
+    rev: 5.10.1
     hooks:
     -   id: isort
 -   repo: https://github.com/ambv/black
-    rev: 22.3.0
+    rev: 22.8.0
     hooks:
     -   id: black
         language_version: python3
--   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+        exclude: elasticapm\/utils\/wrapt
+-   repo: https://github.com/PyCQA/flake8
+    rev: 5.0.4
     hooks:
     -   id: flake8
         exclude: elasticapm\/utils\/wrapt|build|src|tests|dist|conftest.py|setup.py

Jenkinsfile

@@ -67,35 +67,6 @@ pipeline {
             }
           }
         }
-        stage('Sanity checks') {
-          when {
-            beforeAgent true
-            allOf {
-              expression { return env.ONLY_DOCS == "false" }
-              anyOf {
-                not { changeRequest() }
-                expression { return params.Run_As_Main_Branch }
-              }
-            }
-          }
-          environment {
-            PATH = "${env.WORKSPACE}/.local/bin:${env.WORKSPACE}/bin:${env.PATH}"
-          }
-          steps {
-            withGithubNotify(context: 'Sanity checks', tab: 'tests') {
-              deleteDir()
-              unstash 'source'
-              script {
-                docker.image('python:3.7-stretch').inside(){
-                  dir("${BASE_DIR}"){
-                    // registry: '' will help to disable the docker login
-                    preCommit(commit: "${GIT_BASE_COMMIT}", junit: true, registry: '')
-                  }
-                }
-              }
-            }
-          }
-        }
         /**
         Execute unit tests.
         */

elasticapm/base.py

@@ -140,8 +140,6 @@ def __init__(self, config=None, **inline):
                 logging.setLogRecordFactory(new_factory)
 
         headers = {
-            "Content-Type": "application/x-ndjson",
-            "Content-Encoding": "gzip",
             "User-Agent": self.get_user_agent(),
         }
 

elasticapm/contrib/asyncio/traces.py

@@ -30,15 +30,17 @@
 
 import functools
 from types import TracebackType
-from typing import Optional, Type
+from typing import Optional, Type, TypeVar
 
 from elasticapm.conf.constants import LABEL_RE
 from elasticapm.traces import SpanType, capture_span, execution_context
 from elasticapm.utils import get_name_from_func
 
+_AnnotatedFunctionT = TypeVar("_AnnotatedFunctionT")
+
 
 class async_capture_span(capture_span):
-    def __call__(self, func):
+    def __call__(self, func: _AnnotatedFunctionT) -> _AnnotatedFunctionT:
         self.name = self.name or get_name_from_func(func)
 
         @functools.wraps(func)

elasticapm/contrib/serverless/aws.py

@@ -35,7 +35,7 @@
 import os
 import platform
 import time
-from typing import Optional
+from typing import Optional, TypeVar
 from urllib.parse import urlencode
 
 import elasticapm
@@ -51,6 +51,8 @@
 
 COLD_START = True
 
+_AnnotatedFunctionT = TypeVar("_AnnotatedFunctionT")
+
 
 class capture_serverless(object):
     """
@@ -89,7 +91,7 @@ def __init__(self, name: Optional[str] = None, elasticapm_client: Optional[Clien
 
         self.client_kwargs = kwargs
 
-    def __call__(self, func):
+    def __call__(self, func: _AnnotatedFunctionT) -> _AnnotatedFunctionT:
         self.name = self.name or get_name_from_func(func)
 
         @functools.wraps(func)

elasticapm/instrumentation/packages/pymongo.py

@@ -132,7 +132,8 @@ def call(self, module, method, wrapped, instance, args, kwargs):
             span_type="db",
             span_subtype="mongodb",
             span_action="query",
-            extra={"destination": {}},
+            extra=context,
+            leaf=True,
         ) as span:
             response = wrapped(*args, **kwargs)
             if span.context and instance.address:

elasticapm/traces.py

@@ -39,7 +39,7 @@
 from collections import defaultdict
 from datetime import timedelta
 from types import TracebackType
-from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type, Union
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Type, TypeVar, Union
 
 import elasticapm
 from elasticapm.conf import constants
@@ -62,6 +62,7 @@
 execution_context = init_execution_context()
 
 SpanType = Union["Span", "DroppedSpan"]
+_AnnotatedFunctionT = TypeVar("_AnnotatedFunctionT")
 
 
 class ChildDuration(object):
@@ -1056,7 +1057,7 @@ def __init__(
         self.sync = sync
         self.links = links
 
-    def __call__(self, func: Callable) -> Callable:
+    def __call__(self, func: _AnnotatedFunctionT) -> _AnnotatedFunctionT:
         self.name = self.name or get_name_from_func(func)
 
         @functools.wraps(func)

elasticapm/transport/http.py

@@ -75,6 +75,12 @@ def send(self, data, forced_flush=False):
 
         headers = self._headers.copy() if self._headers else {}
         headers.update(self.auth_headers)
+        headers.update(
+            {
+                b"Content-Type": b"application/x-ndjson",
+                b"Content-Encoding": b"gzip",
+            }
+        )
 
         url = self._url
         if forced_flush:
@@ -146,7 +152,6 @@ def get_config(self, current_version=None, keys=None):
         data = json_encoder.dumps(keys).encode("utf-8")
         headers = self._headers.copy()
         headers[b"Content-Type"] = "application/json"
-        headers.pop(b"Content-Encoding", None)  # remove gzip content-encoding header
         headers.update(self.auth_headers)
         max_age = 300
         if current_version:
@@ -159,11 +164,9 @@ def get_config(self, current_version=None, keys=None):
             logger.debug("HTTP error while fetching remote config: %s", str(e))
             return current_version, None, max_age
         body = response.read()
-        if "Cache-Control" in response.headers:
-            try:
-                max_age = int(next(re.finditer(r"max-age=(\d+)", response.headers["Cache-Control"])).groups()[0])
-            except StopIteration:
-                logger.debug("Could not parse Cache-Control header: %s", response.headers["Cache-Control"])
+
+        max_age = self._get_cache_control_max_age(response.headers) or max_age
+
         if response.status == 304:
             # config is unchanged, return
             logger.debug("Configuration unchanged")
@@ -182,6 +185,22 @@ def get_config(self, current_version=None, keys=None):
             logger.warning("Failed decoding APM Server response as JSON: %s", body)
             return current_version, None, max_age
 
+    def _get_cache_control_max_age(self, response_headers):
+        max_age = None
+        if "Cache-Control" in response_headers:
+            try:
+                cc_max_age = int(next(re.finditer(r"max-age=(\d+)", response_headers["Cache-Control"])).groups()[0])
+                if cc_max_age <= 0:
+                    # max_age remains at default value
+                    pass
+                elif cc_max_age < 5:
+                    max_age = 5
+                else:
+                    max_age = cc_max_age
+            except StopIteration:
+                logger.debug("Could not parse Cache-Control header: %s", response_headers["Cache-Control"])
+        return max_age
+
     def _process_queue(self):
         if not self.client.server_version:
             self.fetch_server_info()
@@ -190,7 +209,7 @@ def _process_queue(self):
     def fetch_server_info(self):
         headers = self._headers.copy() if self._headers else {}
         headers.update(self.auth_headers)
-        headers["accept"] = "text/plain"
+        headers[b"accept"] = b"text/plain"
         try:
             response = self.http.urlopen("GET", self._server_info_url, headers=headers, timeout=self._timeout)
             body = response.data

elasticapm/utils/compat.py

@@ -33,9 +33,12 @@
 import atexit
 import functools
 import platform
+from typing import TypeVar
 
+_AnnotatedFunctionT = TypeVar("_AnnotatedFunctionT")
 
-def noop_decorator(func):
+
+def noop_decorator(func: _AnnotatedFunctionT) -> _AnnotatedFunctionT:
     @functools.wraps(func)
     def wrapped(*args, **kwargs):
         return func(*args, **kwargs)

elasticapm/utils/deprecation.py

@@ -30,6 +30,9 @@
 
 import functools
 import warnings
+from typing import TypeVar
+
+_AnnotatedFunctionT = TypeVar("_AnnotatedFunctionT")
 
 # https://wiki.python.org/moin/PythonDecoratorLibrary#Smart_deprecation_warnings_.28with_valid_filenames.2C_line_numbers.2C_etc..29
 
@@ -39,7 +42,7 @@ def deprecated(alternative=None):
     as deprecated. It will result in a warning being emitted
     when the function is used."""
 
-    def real_decorator(func):
+    def real_decorator(func: _AnnotatedFunctionT) -> _AnnotatedFunctionT:
         @functools.wraps(func)
         def new_func(*args, **kwargs):
             msg = "Call to deprecated function {0}.".format(func.__name__)

tests/Dockerfile

@@ -1,15 +1,20 @@
 ARG PYTHON_IMAGE
-FROM ${PYTHON_IMAGE}
+FROM ${PYTHON_IMAGE}-slim
 
 RUN apt-get -qq update && apt-get -qq -y --no-install-recommends install \
+    build-essential \
     ca-certificates \
     curl \
+    gnupg2 \
+    libmariadb-dev \
+    libpq-dev \
+    make \
     netcat \
     odbc-postgresql \
     unixodbc-dev \
     freetds-dev \
-    libmemcached-dev &&\
-    rm -rf /var/lib/apt/lists/*
+    libmemcached-dev \
+    && rm -rf /var/lib/apt/lists/*
 
 # connection to ha.pool.sks-keyservers.net fails sometimes, so let's retry with couple different servers
 RUN for server in $(shuf -e ha.pool.sks-keyservers.net \