Cache the custom secrets backend so the same instance gets re-used

Fixes apache#25555

This uses `functools.lru_cache` to re-use the same secrets backend
instance between the `conf` global when it loads configuration from
secrets and uses outside the `configuration` module like variables and
connections. Previously, each fetch of a configuration value from
secrets would use its own secrets backend instance.

airflow/configuration.py

@@ -1535,6 +1535,11 @@ def get_custom_secret_backend() -> Optional[BaseSecretsBackend]:
     """Get Secret Backend if defined in airflow.cfg"""
     secrets_backend_cls = conf.getimport(section='secrets', key='backend')
 
+    return _custom_secrets_backend(secrets_backend_cls)
+
+
+@functools.lru_cache(maxsize=2)
+def _custom_secrets_backend(secrets_backend_cls) -> Optional[BaseSecretsBackend]:
     if secrets_backend_cls:
         try:
             backends: Any = conf.get(section='secrets', key='backend_kwargs', fallback='{}')

tests/core/test_configuration.py

@@ -33,6 +33,7 @@
 from airflow.configuration import (
     AirflowConfigException,
     AirflowConfigParser,
+    _custom_secrets_backend,
     conf,
     expand_env_var,
     get_airflow_config,
@@ -296,6 +297,7 @@ def test_hidding_of_sensitive_config_values(self):
     def test_config_raise_exception_from_secret_backend_connection_error(self, mock_hvac):
         """Get Config Value from a Secret Backend"""
 
+        _custom_secrets_backend.cache_clear()  # so secrets backend gets reloaded
         mock_client = mock.MagicMock()
         # mock_client.side_effect = AirflowConfigException
         mock_hvac.Client.return_value = mock_client