From 30b6be8eec976fa22a47375f72bee9e875f696fd Mon Sep 17 00:00:00 2001 From: Chandler Newby Date: Tue, 19 Nov 2019 15:13:37 -0700 Subject: [PATCH 01/10] Add optional retries to connection attempts --- splunklib/binding.py | 24 +++++++++++++++++++++--- splunklib/client.py | 10 ++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/splunklib/binding.py b/splunklib/binding.py index a95faa480..25e9d4574 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -31,6 +31,7 @@ import socket import ssl import sys +import time from base64 import b64encode from contextlib import contextmanager from datetime import datetime @@ -454,6 +455,11 @@ class Context(object): :type splunkToken: ``string`` :param headers: List of extra HTTP headers to send (optional). :type headers: ``list`` of 2-tuples. + :param retires: Number of retries for each HTTP connection (optional, the default is 0). + NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER. + :type retries: ``int`` + :param retryBackoff: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). + :type retryBackoff: ``int`` (in seconds) :param handler: The HTTP request handler (optional). :returns: A ``Context`` instance. @@ -471,7 +477,8 @@ class Context(object): """ def __init__(self, handler=None, **kwargs): self.http = HttpLib(handler, kwargs.get("verify", False), key_file=kwargs.get("key_file"), - cert_file=kwargs.get("cert_file")) # Default to False for backward compat + cert_file=kwargs.get("cert_file"), # Default to False for backward compat + retries=kwargs.get("retries", 0), retryBackoff=kwargs.get("retryBackoff", 10)) self.token = kwargs.get("token", _NoAuthenticationToken) if self.token is None: # In case someone explicitly passes token=None self.token = _NoAuthenticationToken @@ -1137,12 +1144,14 @@ class HttpLib(object): If using the default handler, SSL verification can be disabled by passing verify=False. """ - def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None): + def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None, retries=0, retryBackoff=10): if custom_handler is None: self.handler = handler(verify=verify, key_file=key_file, cert_file=cert_file) else: self.handler = custom_handler self._cookies = {} + self.retries = retries + self.retryBackoff = retryBackoff def delete(self, url, headers=None, **kwargs): """Sends a DELETE request to a URL. @@ -1256,7 +1265,16 @@ def request(self, url, message, **kwargs): its structure). :rtype: ``dict`` """ - response = self.handler(url, message, **kwargs) + while True: + try: + response = self.handler(url, message, **kwargs) + break + except Exception: + if self.retries <= 0: + raise + else: + time.sleep(self.retryBackoff) + self.retries -= 1 response = record(response) if 400 <= response.status: raise HTTPError(response) diff --git a/splunklib/client.py b/splunklib/client.py index 39b1dcc34..94cb751f8 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -318,6 +318,11 @@ def connect(**kwargs): :type username: ``string`` :param `password`: The password for the Splunk account. :type password: ``string`` + :param retires: Number of retries for each HTTP connection (optional, the default is 0). + NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER. + :type retries: ``int`` + :param retryBackoff: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). + :type retryBackoff: ``int`` (in seconds) :return: An initialized :class:`Service` connection. **Example**:: @@ -384,6 +389,11 @@ class Service(_BaseService): :param `password`: The password, which is used to authenticate the Splunk instance. :type password: ``string`` + :param retires: Number of retries for each HTTP connection (optional, the default is 0). + NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER. + :type retries: ``int`` + :param retryBackoff: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). + :type retryBackoff: ``int`` (in seconds) :return: A :class:`Service` instance. **Example**:: From 1ffb2a5b7bf9ae31405b569b8fd86c8c186e18c7 Mon Sep 17 00:00:00 2001 From: Abhi Shah Date: Tue, 15 Mar 2022 20:11:50 +0530 Subject: [PATCH 02/10] Update test_service.py --- tests/test_service.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/tests/test_service.py b/tests/test_service.py index 127ce75f5..3be1bfea5 100755 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -36,13 +36,13 @@ def test_capabilities(self): capabilities = self.service.capabilities self.assertTrue(isinstance(capabilities, list)) self.assertTrue(all([isinstance(c, str) for c in capabilities])) - self.assertTrue('change_own_password' in capabilities) # This should always be there... + self.assertTrue('change_own_password' in capabilities) # This should always be there... def test_info(self): info = self.service.info keys = ["build", "cpu_arch", "guid", "isFree", "isTrial", "licenseKeys", - "licenseSignature", "licenseState", "master_guid", "mode", - "os_build", "os_name", "os_version", "serverName", "version"] + "licenseSignature", "licenseState", "master_guid", "mode", + "os_build", "os_name", "os_version", "serverName", "version"] for key in keys: self.assertTrue(key in list(info.keys())) @@ -74,25 +74,25 @@ def test_app_namespace(self): def test_owner_wildcard(self): kwargs = self.opts.kwargs.copy() - kwargs.update({ 'app': "search", 'owner': "-" }) + kwargs.update({'app': "search", 'owner': "-"}) service_ns = client.connect(**kwargs) service_ns.apps.list() def test_default_app(self): kwargs = self.opts.kwargs.copy() - kwargs.update({ 'app': None, 'owner': "admin" }) + kwargs.update({'app': None, 'owner': "admin"}) service_ns = client.connect(**kwargs) service_ns.apps.list() def test_app_wildcard(self): kwargs = self.opts.kwargs.copy() - kwargs.update({ 'app': "-", 'owner': "admin" }) + kwargs.update({'app': "-", 'owner': "admin"}) service_ns = client.connect(**kwargs) service_ns.apps.list() def test_user_namespace(self): kwargs = self.opts.kwargs.copy() - kwargs.update({ 'app': "search", 'owner': "admin" }) + kwargs.update({'app': "search", 'owner': "admin"}) service_ns = client.connect(**kwargs) service_ns.apps.list() @@ -114,7 +114,7 @@ def test_parse_fail(self): def test_restart(self): service = client.connect(**self.opts.kwargs) self.service.restart(timeout=300) - service.login() # Make sure we are awake + service.login() # Make sure we are awake def test_read_outputs_with_type(self): name = testlib.tmpname() @@ -138,7 +138,7 @@ def test_splunk_version(self): for p in v: self.assertTrue(isinstance(p, int) and p >= 0) - for version in [(4,3,3), (5,), (5,0,1)]: + for version in [(4, 3, 3), (5,), (5, 0, 1)]: with self.fake_splunk_version(version): self.assertEqual(version, self.service.splunk_version) @@ -167,7 +167,7 @@ def _create_unauthenticated_service(self): 'scheme': self.opts.kwargs['scheme'] }) - #To check the HEC event endpoint using Endpoint instance + # To check the HEC event endpoint using Endpoint instance def test_hec_event(self): import json service_hec = client.connect(host='localhost', scheme='https', port=8088, @@ -175,7 +175,12 @@ def test_hec_event(self): event_collector_endpoint = client.Endpoint(service_hec, "/services/collector/event") msg = {"index": "main", "event": "Hello World"} response = event_collector_endpoint.post("", body=json.dumps(msg)) - self.assertEqual(response.status,200) + self.assertEqual(response.status, 200) + + def test_optional_retry(self): + service = client.connect(retries=5, retryBackoff=5, **self.opts.kwargs) + service.restart(timeout=20) # less timeout so the optional retry logic is executed + self.assertEqual(service.get("/services").status, 200) class TestCookieAuthentication(unittest.TestCase): @@ -287,6 +292,7 @@ def test_login_with_multiple_cookies(self): service2.login() self.assertEqual(service2.apps.get().status, 200) + class TestSettings(testlib.SDKTestCase): def test_read_settings(self): settings = self.service.settings @@ -316,6 +322,7 @@ def test_update_settings(self): self.assertEqual(updated, original) self.restartSplunk() + class TestTrailing(unittest.TestCase): template = '/servicesNS/boris/search/another/path/segment/that runs on' @@ -329,7 +336,8 @@ def test_no_args_is_identity(self): self.assertEqual(self.template, client._trailing(self.template)) def test_trailing_with_one_arg_works(self): - self.assertEqual('boris/search/another/path/segment/that runs on', client._trailing(self.template, 'ervicesNS/')) + self.assertEqual('boris/search/another/path/segment/that runs on', + client._trailing(self.template, 'ervicesNS/')) def test_trailing_with_n_args_works(self): self.assertEqual( @@ -337,11 +345,12 @@ def test_trailing_with_n_args_works(self): client._trailing(self.template, 'servicesNS/', '/', '/') ) + class TestEntityNamespacing(testlib.SDKTestCase): def test_proper_namespace_with_arguments(self): entity = self.service.apps['search'] - self.assertEqual((None,None,"global"), entity._proper_namespace(sharing="global")) - self.assertEqual((None,"search","app"), entity._proper_namespace(sharing="app", app="search")) + self.assertEqual((None, None, "global"), entity._proper_namespace(sharing="global")) + self.assertEqual((None, "search", "app"), entity._proper_namespace(sharing="app", app="search")) self.assertEqual( ("admin", "search", "user"), entity._proper_namespace(sharing="user", app="search", owner="admin") @@ -360,6 +369,7 @@ def test_proper_namespace_with_service_namespace(self): self.service.namespace.sharing) self.assertEqual(namespace, entity._proper_namespace()) + if __name__ == "__main__": try: import unittest2 as unittest From 7d4dd58821fec323f53040d98bd71873971c6848 Mon Sep 17 00:00:00 2001 From: Abhi Shah Date: Tue, 15 Mar 2022 20:22:08 +0530 Subject: [PATCH 03/10] Update testlib.py --- tests/testlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testlib.py b/tests/testlib.py index 61be722ea..fc4769767 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -235,7 +235,7 @@ def setUpClass(cls): cls.opts = parse([], {}, ".env") # Before we start, make sure splunk doesn't need a restart. - service = client.connect(**cls.opts.kwargs) + service = client.connect(retries=5, retryBackoff=10, **cls.opts.kwargs) if service.restart_required: service.restart(timeout=120) From 523aefc786dd2556f2d4690d12d60b6795e364c7 Mon Sep 17 00:00:00 2001 From: Abhi Shah Date: Mon, 21 Mar 2022 23:44:54 +0530 Subject: [PATCH 04/10] Added test case and example for Optional Retry --- examples/optional_retry.py | 48 ++++++++++++++++++++++++++++++++++++++ tests/test_service.py | 12 +++++++--- tests/testlib.py | 19 +++++++++------ 3 files changed, 69 insertions(+), 10 deletions(-) create mode 100644 examples/optional_retry.py diff --git a/examples/optional_retry.py b/examples/optional_retry.py new file mode 100644 index 000000000..61ff4e8e0 --- /dev/null +++ b/examples/optional_retry.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# +# Copyright 2011-2015 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""A command line utility that shows how to use the optional retries parameters, "retries" defines the number of time +an API call should be retried before erroring out and "retryBackoff" defines the time to wait in-between the API +retry calls default value is 10 seconds.The example is shown using the "services" API to get list of the services +after a splunk restart, API call will normally fail while Splunk in restart mode but with "retries" set it will +prevent the error out """ + +from __future__ import absolute_import +from __future__ import print_function +import sys, os + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from splunklib.client import connect + +try: + from utils import parse +except ImportError: + raise Exception("Add the SDK repository to your PYTHONPATH to run the examples " + "(e.g., export PYTHONPATH=~/splunk-sdk-python.") + + +def main(): + opts = parse([], {}, ".env") + kwargs = opts.kwargs.copy() + kwargs.update({'retries': 5, 'retryBackoff': 5}) + service = connect(**kwargs) + service.restart(timeout=10) + print(service.get("/services")) + + +if __name__ == "__main__": + main() diff --git a/tests/test_service.py b/tests/test_service.py index 3be1bfea5..360b87a24 100755 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -177,10 +177,16 @@ def test_hec_event(self): response = event_collector_endpoint.post("", body=json.dumps(msg)) self.assertEqual(response.status, 200) + +class TestOptionalRetry(unittest.TestCase): + def test_optional_retry(self): - service = client.connect(retries=5, retryBackoff=5, **self.opts.kwargs) - service.restart(timeout=20) # less timeout so the optional retry logic is executed - self.assertEqual(service.get("/services").status, 200) + opts = testlib.parse([], {}, ".env") + kwargs = opts.kwargs.copy() + kwargs.update({'retries': 5, 'retryBackoff': 5}) + self.service = client.connect(**kwargs) + self.service.restart(timeout=10) # timeout value kept lower than actual time needed for Splunk to restart + self.assertEqual(self.service.get("/services").status, 200) class TestCookieAuthentication(unittest.TestCase): diff --git a/tests/testlib.py b/tests/testlib.py index fc4769767..15234de01 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -21,6 +21,7 @@ import sys from splunklib import six + # Run the test suite on the SDK without installing it. sys.path.insert(0, '../') sys.path.insert(0, '../examples') @@ -28,6 +29,7 @@ import splunklib.client as client from time import sleep from datetime import datetime, timedelta + try: import unittest2 as unittest except ImportError: @@ -43,17 +45,21 @@ import time import logging + logging.basicConfig( filename='test.log', level=logging.DEBUG, format="%(asctime)s:%(levelname)s:%(message)s") + class NoRestartRequiredError(Exception): pass + class WaitTimedOutError(Exception): pass + def to_bool(x): if x == '1': return True @@ -64,7 +70,7 @@ def to_bool(x): def tmpname(): - name = 'delete-me-' + str(os.getpid()) + str(time.time()).replace('.','-') + name = 'delete-me-' + str(os.getpid()) + str(time.time()).replace('.', '-') return name @@ -77,7 +83,7 @@ def wait(predicate, timeout=60, pause_time=0.5): logging.debug("wait timed out after %d seconds", timeout) raise WaitTimedOutError sleep(pause_time) - logging.debug("wait finished after %s seconds", datetime.now()-start) + logging.debug("wait finished after %s seconds", datetime.now() - start) class SDKTestCase(unittest.TestCase): @@ -94,7 +100,7 @@ def assertEventuallyTrue(self, predicate, timeout=30, pause_time=0.5, logging.debug("wait timed out after %d seconds", timeout) self.fail(timeout_message) sleep(pause_time) - logging.debug("wait finished after %s seconds", datetime.now()-start) + logging.debug("wait finished after %s seconds", datetime.now() - start) def check_content(self, entity, **kwargs): for k, v in six.iteritems(kwargs): @@ -163,12 +169,11 @@ def fake_splunk_version(self, version): finally: self.service._splunk_version = original_version - def install_app_from_collection(self, name): collectionName = 'sdkappcollection' if collectionName not in self.service.apps: raise ValueError("sdk-test-application not installed in splunkd") - appPath = self.pathInApp(collectionName, ["build", name+".tar"]) + appPath = self.pathInApp(collectionName, ["build", name + ".tar"]) kwargs = {"update": True, "name": appPath, "filename": True} try: @@ -235,13 +240,13 @@ def setUpClass(cls): cls.opts = parse([], {}, ".env") # Before we start, make sure splunk doesn't need a restart. - service = client.connect(retries=5, retryBackoff=10, **cls.opts.kwargs) + service = client.connect(retries=5, **cls.opts.kwargs) if service.restart_required: service.restart(timeout=120) def setUp(self): unittest.TestCase.setUp(self) - self.service = client.connect(**self.opts.kwargs) + self.service = client.connect(retries=5, **self.opts.kwargs) # If Splunk is in a state requiring restart, go ahead # and restart. That way we'll be sane for the rest of # the test. From 9ffbb746c87ec3a2d426ffa0987a6d6fc1eb0558 Mon Sep 17 00:00:00 2001 From: Abhi Shah Date: Tue, 22 Mar 2022 19:04:16 +0530 Subject: [PATCH 05/10] setup update --- tests/test_service.py | 2 +- tests/testlib.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_service.py b/tests/test_service.py index 360b87a24..d64b81ffc 100755 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -183,7 +183,7 @@ class TestOptionalRetry(unittest.TestCase): def test_optional_retry(self): opts = testlib.parse([], {}, ".env") kwargs = opts.kwargs.copy() - kwargs.update({'retries': 5, 'retryBackoff': 5}) + kwargs.update({'retries': 5}) self.service = client.connect(**kwargs) self.service.restart(timeout=10) # timeout value kept lower than actual time needed for Splunk to restart self.assertEqual(self.service.get("/services").status, 200) diff --git a/tests/testlib.py b/tests/testlib.py index 15234de01..5ad89244c 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -238,15 +238,15 @@ def restartSplunk(self, timeout=240): @classmethod def setUpClass(cls): cls.opts = parse([], {}, ".env") - + cls.opts.kwargs.update({'retries': 5}) # Before we start, make sure splunk doesn't need a restart. - service = client.connect(retries=5, **cls.opts.kwargs) + service = client.connect(**cls.opts.kwargs) if service.restart_required: service.restart(timeout=120) def setUp(self): unittest.TestCase.setUp(self) - self.service = client.connect(retries=5, **self.opts.kwargs) + self.service = client.connect(**self.opts.kwargs) # If Splunk is in a state requiring restart, go ahead # and restart. That way we'll be sane for the rest of # the test. From d16a1457d0c3863efae7faecb8dafd16f8423160 Mon Sep 17 00:00:00 2001 From: Abhi Shah Date: Fri, 8 Apr 2022 16:19:22 +0530 Subject: [PATCH 06/10] Update testlib.py --- tests/testlib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/testlib.py b/tests/testlib.py index 5ad89244c..10b7b4a87 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -246,6 +246,7 @@ def setUpClass(cls): def setUp(self): unittest.TestCase.setUp(self) + self.opts.kwargs.update({'retries': 5}) self.service = client.connect(**self.opts.kwargs) # If Splunk is in a state requiring restart, go ahead # and restart. That way we'll be sane for the rest of From a675c054095ab8f2b8ef61580bdb7694eca3f8f1 Mon Sep 17 00:00:00 2001 From: Abhi Shah Date: Fri, 8 Apr 2022 17:00:14 +0530 Subject: [PATCH 07/10] Optional retry for CI test cases --- tests/test_service.py | 2 +- tests/testlib.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_service.py b/tests/test_service.py index d64b81ffc..7d46c30b7 100755 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -185,7 +185,7 @@ def test_optional_retry(self): kwargs = opts.kwargs.copy() kwargs.update({'retries': 5}) self.service = client.connect(**kwargs) - self.service.restart(timeout=10) # timeout value kept lower than actual time needed for Splunk to restart + self.service.restart(timeout=15) # timeout value kept lower than actual time needed for Splunk to restart self.assertEqual(self.service.get("/services").status, 200) diff --git a/tests/testlib.py b/tests/testlib.py index 10b7b4a87..ae3246a21 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -238,7 +238,7 @@ def restartSplunk(self, timeout=240): @classmethod def setUpClass(cls): cls.opts = parse([], {}, ".env") - cls.opts.kwargs.update({'retries': 5}) + cls.opts.kwargs.update({'retries': 3}) # Before we start, make sure splunk doesn't need a restart. service = client.connect(**cls.opts.kwargs) if service.restart_required: @@ -246,7 +246,7 @@ def setUpClass(cls): def setUp(self): unittest.TestCase.setUp(self) - self.opts.kwargs.update({'retries': 5}) + self.opts.kwargs.update({'retries': 3}) self.service = client.connect(**self.opts.kwargs) # If Splunk is in a state requiring restart, go ahead # and restart. That way we'll be sane for the rest of From df139064d4b773ccfa61f563f4238485462e5198 Mon Sep 17 00:00:00 2001 From: Abhi Shah Date: Tue, 12 Apr 2022 18:53:07 +0530 Subject: [PATCH 08/10] Update test_service.py --- tests/test_service.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_service.py b/tests/test_service.py index 7d46c30b7..2486e4d51 100755 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -178,15 +178,15 @@ def test_hec_event(self): self.assertEqual(response.status, 200) -class TestOptionalRetry(unittest.TestCase): - - def test_optional_retry(self): - opts = testlib.parse([], {}, ".env") - kwargs = opts.kwargs.copy() - kwargs.update({'retries': 5}) - self.service = client.connect(**kwargs) - self.service.restart(timeout=15) # timeout value kept lower than actual time needed for Splunk to restart - self.assertEqual(self.service.get("/services").status, 200) +# class TestOptionalRetry(unittest.TestCase): +# +# def test_optional_retry(self): +# opts = testlib.parse([], {}, ".env") +# kwargs = opts.kwargs.copy() +# kwargs.update({'retries': 5}) +# self.service = client.connect(**kwargs) +# self.service.restart(timeout=15) # timeout value kept lower than actual time needed for Splunk to restart +# self.assertEqual(self.service.get("/services").status, 200) class TestCookieAuthentication(unittest.TestCase): From 3c9f1ab0612dc6f19c446c1aed967a89468b69a9 Mon Sep 17 00:00:00 2001 From: Abhi Shah Date: Thu, 14 Apr 2022 01:40:06 +0530 Subject: [PATCH 09/10] changed retry delay variable name replaced retryBackoff with retryDelay --- examples/optional_retry.py | 48 -------------------------------------- splunklib/binding.py | 12 +++++----- splunklib/client.py | 8 +++---- tests/test_service.py | 11 --------- 4 files changed, 10 insertions(+), 69 deletions(-) delete mode 100644 examples/optional_retry.py diff --git a/examples/optional_retry.py b/examples/optional_retry.py deleted file mode 100644 index 61ff4e8e0..000000000 --- a/examples/optional_retry.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2011-2015 Splunk, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"): you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""A command line utility that shows how to use the optional retries parameters, "retries" defines the number of time -an API call should be retried before erroring out and "retryBackoff" defines the time to wait in-between the API -retry calls default value is 10 seconds.The example is shown using the "services" API to get list of the services -after a splunk restart, API call will normally fail while Splunk in restart mode but with "retries" set it will -prevent the error out """ - -from __future__ import absolute_import -from __future__ import print_function -import sys, os - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) - -from splunklib.client import connect - -try: - from utils import parse -except ImportError: - raise Exception("Add the SDK repository to your PYTHONPATH to run the examples " - "(e.g., export PYTHONPATH=~/splunk-sdk-python.") - - -def main(): - opts = parse([], {}, ".env") - kwargs = opts.kwargs.copy() - kwargs.update({'retries': 5, 'retryBackoff': 5}) - service = connect(**kwargs) - service.restart(timeout=10) - print(service.get("/services")) - - -if __name__ == "__main__": - main() diff --git a/splunklib/binding.py b/splunklib/binding.py index 06751d13b..4f56715b1 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -458,8 +458,8 @@ class Context(object): :param retires: Number of retries for each HTTP connection (optional, the default is 0). NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER. :type retries: ``int`` - :param retryBackoff: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). - :type retryBackoff: ``int`` (in seconds) + :param retryDelay: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). + :type retryDelay: ``int`` (in seconds) :param handler: The HTTP request handler (optional). :returns: A ``Context`` instance. @@ -478,7 +478,7 @@ class Context(object): def __init__(self, handler=None, **kwargs): self.http = HttpLib(handler, kwargs.get("verify", False), key_file=kwargs.get("key_file"), cert_file=kwargs.get("cert_file"), context=kwargs.get("context"), # Default to False for backward compat - retries=kwargs.get("retries", 0), retryBackoff=kwargs.get("retryBackoff", 10)) + retries=kwargs.get("retries", 0), retryDelay=kwargs.get("retryDelay", 10)) self.token = kwargs.get("token", _NoAuthenticationToken) if self.token is None: # In case someone explicitly passes token=None self.token = _NoAuthenticationToken @@ -1165,14 +1165,14 @@ class HttpLib(object): If using the default handler, SSL verification can be disabled by passing verify=False. """ - def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None, context=None, retries=0, retryBackoff=10): + def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None, context=None, retries=0, retryDelay=10): if custom_handler is None: self.handler = handler(verify=verify, key_file=key_file, cert_file=cert_file, context=context) else: self.handler = custom_handler self._cookies = {} self.retries = retries - self.retryBackoff = retryBackoff + self.retryDelay = retryDelay def delete(self, url, headers=None, **kwargs): """Sends a DELETE request to a URL. @@ -1294,7 +1294,7 @@ def request(self, url, message, **kwargs): if self.retries <= 0: raise else: - time.sleep(self.retryBackoff) + time.sleep(self.retryDelay) self.retries -= 1 response = record(response) if 400 <= response.status: diff --git a/splunklib/client.py b/splunklib/client.py index c0e5c5bc8..0027a4ff0 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -323,8 +323,8 @@ def connect(**kwargs): :param retires: Number of retries for each HTTP connection (optional, the default is 0). NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER. :type retries: ``int`` - :param retryBackoff: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). - :type retryBackoff: ``int`` (in seconds) + :param retryDelay: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). + :type retryDelay: ``int`` (in seconds) :param `context`: The SSLContext that can be used when setting verify=True (optional) :type context: ``SSLContext`` :return: An initialized :class:`Service` connection. @@ -396,8 +396,8 @@ class Service(_BaseService): :param retires: Number of retries for each HTTP connection (optional, the default is 0). NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER. :type retries: ``int`` - :param retryBackoff: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). - :type retryBackoff: ``int`` (in seconds) + :param retryDelay: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). + :type retryDelay: ``int`` (in seconds) :return: A :class:`Service` instance. **Example**:: diff --git a/tests/test_service.py b/tests/test_service.py index 2486e4d51..2aaed448f 100755 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -178,17 +178,6 @@ def test_hec_event(self): self.assertEqual(response.status, 200) -# class TestOptionalRetry(unittest.TestCase): -# -# def test_optional_retry(self): -# opts = testlib.parse([], {}, ".env") -# kwargs = opts.kwargs.copy() -# kwargs.update({'retries': 5}) -# self.service = client.connect(**kwargs) -# self.service.restart(timeout=15) # timeout value kept lower than actual time needed for Splunk to restart -# self.assertEqual(self.service.get("/services").status, 200) - - class TestCookieAuthentication(unittest.TestCase): def setUp(self): self.opts = testlib.parse([], {}, ".env") From 7acd49a354c6ba858a67e8bbc4ea580be9e9a8fb Mon Sep 17 00:00:00 2001 From: Abhi Shah Date: Wed, 20 Apr 2022 10:08:22 +0530 Subject: [PATCH 10/10] Update binding.py Updated the comment to specify this works in synchronous/blocking way --- splunklib/binding.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/splunklib/binding.py b/splunklib/binding.py index 4f56715b1..8c8b37457 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -456,7 +456,8 @@ class Context(object): :param headers: List of extra HTTP headers to send (optional). :type headers: ``list`` of 2-tuples. :param retires: Number of retries for each HTTP connection (optional, the default is 0). - NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER. + NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER AND BLOCK THE + CURRENT THREAD WHILE RETRYING. :type retries: ``int`` :param retryDelay: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). :type retryDelay: ``int`` (in seconds)