diff --git a/.env b/.env index 0d5fabf1..59f7df83 100644 --- a/.env +++ b/.env @@ -9,7 +9,7 @@ password=changed! # Access scheme (default: https) scheme=https # Your version of Splunk (default: 6.2) -version=8.0 +version=9.0 # Bearer token for authentication #bearerToken="" # Session key for authentication diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 42713a68..ba6e777e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: - ubuntu-latest python: [ 2.7, 3.7 ] splunk-version: - - "8.0" + - "8.2" - "latest" fail-fast: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 4215c05d..f01bb3fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Splunk Enterprise SDK for Python Changelog +## Version 1.7.0 + +### New features and APIs +* [#468](https://github.com/splunk/splunk-sdk-python/pull/468) SDK Support for splunkd search API changes + +### Bug fixes +* [#464](https://github.com/splunk/splunk-sdk-python/pull/464) updated checks for wildcards in StoragePasswords [[issue#458](https://github.com/splunk/splunk-sdk-python/issues/458)] + +### Minor changes +* [#463](https://github.com/splunk/splunk-sdk-python/pull/463) Preserve thirdparty cookies + ## Version 1.6.20 ### New features and APIs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6de8fccd..8b22561a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ If you're seeing some unexpected behavior with this project, please create an [i 1. Version of this project you're using (ex: 1.5.0) 2. Platform version (ex: Windows Server 2012 R2) 3. Framework version (ex: Python 3.7) -4. Splunk Enterprise version (ex: 8.0) +4. Splunk Enterprise version (ex: 9.0) 5. Other relevant information (ex: local/remote environment, Splunk network configuration, standalone or distributed deployment, are load balancers used) Alternatively, if you have a Splunk question please ask on [Splunk Answers](https://community.splunk.com/t5/Splunk-Development/ct-p/developer-tools). diff --git a/README.md b/README.md index b8e386d7..762844a5 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # The Splunk Enterprise Software Development Kit for Python -#### Version 1.6.20 +#### Version 1.7.0 The Splunk Enterprise Software Development Kit (SDK) for Python contains library code designed to enable developers to build applications using the Splunk platform. @@ -58,7 +58,7 @@ Install the sources you cloned from GitHub: You'll need `docker` and `docker-compose` to get up and running using this method. ``` -make up SPLUNK_VERSION=8.0 +make up SPLUNK_VERSION=9.0 make wait_up make test make down @@ -107,7 +107,7 @@ here is an example of .env file: # Access scheme (default: https) scheme=https # Your version of Splunk Enterprise - version=8.0 + version=9.0 # Bearer token for authentication #bearerToken= # Session key for authentication diff --git a/splunklib/__init__.py b/splunklib/__init__.py index 1f9fc688..487a76c3 100644 --- a/splunklib/__init__.py +++ b/splunklib/__init__.py @@ -31,5 +31,5 @@ def setup_logging(level, log_format=DEFAULT_LOG_FORMAT, date_format=DEFAULT_DATE format=log_format, datefmt=date_format) -__version_info__ = (1, 6, 20) +__version_info__ = (1, 7, 0) __version__ = ".".join(map(str, __version_info__)) diff --git a/splunklib/binding.py b/splunklib/binding.py index bb2771d9..919ad4dc 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -526,23 +526,27 @@ def _auth_headers(self): :returns: A list of 2-tuples containing key and value """ + header = [] if self.has_cookies(): return [("Cookie", _make_cookie_header(list(self.get_cookies().items())))] elif self.basic and (self.username and self.password): token = 'Basic %s' % b64encode(("%s:%s" % (self.username, self.password)).encode('utf-8')).decode('ascii') - return [("Authorization", token)] elif self.bearerToken: token = 'Bearer %s' % self.bearerToken - return [("Authorization", token)] elif self.token is _NoAuthenticationToken: - return [] + token = [] else: # Ensure the token is properly formatted if self.token.startswith('Splunk '): token = self.token else: token = 'Splunk %s' % self.token - return [("Authorization", token)] + if token: + header.append(("Authorization", token)) + if self.get_cookies().__len__() > 0: + header.append("Cookie", _make_cookie_header(self.get_cookies().items())) + + return header def connect(self): """Returns an open connection (socket) to the Splunk instance. @@ -1430,7 +1434,7 @@ def request(url, message, **kwargs): head = { "Content-Length": str(len(body)), "Host": host, - "User-Agent": "splunk-sdk-python/1.6.20", + "User-Agent": "splunk-sdk-python/1.7.0", "Accept": "*/*", "Connection": "Close", } # defaults diff --git a/splunklib/client.py b/splunklib/client.py index 35d9e4f7..14fe2977 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -62,6 +62,7 @@ import datetime import json import logging +import re import socket from datetime import datetime, timedelta from time import sleep @@ -99,6 +100,7 @@ PATH_INDEXES = "data/indexes/" PATH_INPUTS = "data/inputs/" PATH_JOBS = "search/jobs/" +PATH_JOBS_V2 = "search/v2/jobs/" PATH_LOGGER = "/services/server/logger/" PATH_MESSAGES = "messages/" PATH_MODULAR_INPUTS = "data/modular-inputs" @@ -570,6 +572,8 @@ def parse(self, query, **kwargs): :type kwargs: ``dict`` :return: A semantic map of the parsed search query. """ + if self.splunk_version >= (9,): + return self.post("search/v2/parser", q=query, **kwargs) return self.get("search/parser", q=query, **kwargs) def restart(self, timeout=None): @@ -741,6 +745,25 @@ def __init__(self, service, path): self.service = service self.path = path + def get_api_version(self, path): + """Return the API version of the service used in the provided path. + + Args: + path (str): A fully-qualified endpoint path (for example, "/services/search/jobs"). + + Returns: + int: Version of the API (for example, 1) + """ + # Default to v1 if undefined in the path + # For example, "/services/search/jobs" is using API v1 + api_version = 1 + + versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) + if versionSearch: + api_version = int(versionSearch.group(1)) + + return api_version + def get(self, path_segment="", owner=None, app=None, sharing=None, **query): """Performs a GET operation on the path segment relative to this endpoint. @@ -803,6 +826,22 @@ def get(self, path_segment="", owner=None, app=None, sharing=None, **query): app=app, sharing=sharing) # ^-- This was "%s%s" % (self.path, path_segment). # That doesn't work, because self.path may be UrlEncoded. + + # Get the API version from the path + api_version = self.get_api_version(path) + + # Search API v2+ fallback to v1: + # - In v2+, /results_preview, /events and /results do not support search params. + # - Fallback from v2+ to v1 if Splunk Version is < 9. + # if api_version >= 2 and ('search' in query and path.endswith(tuple(["results_preview", "events", "results"])) or self.service.splunk_version < (9,)): + # path = path.replace(PATH_JOBS_V2, PATH_JOBS) + + if api_version == 1: + if isinstance(path, UrlEncoded): + path = UrlEncoded(path.replace(PATH_JOBS_V2, PATH_JOBS), skip_encode=True) + else: + path = path.replace(PATH_JOBS_V2, PATH_JOBS) + return self.service.get(path, owner=owner, app=app, sharing=sharing, **query) @@ -855,13 +894,29 @@ def post(self, path_segment="", owner=None, app=None, sharing=None, **query): apps.get('nonexistant/path') # raises HTTPError s.logout() apps.get() # raises AuthenticationError - """ + """ if path_segment.startswith('/'): path = path_segment else: if not self.path.endswith('/') and path_segment != "": self.path = self.path + '/' path = self.service._abspath(self.path + path_segment, owner=owner, app=app, sharing=sharing) + + # Get the API version from the path + api_version = self.get_api_version(path) + + # Search API v2+ fallback to v1: + # - In v2+, /results_preview, /events and /results do not support search params. + # - Fallback from v2+ to v1 if Splunk Version is < 9. + # if api_version >= 2 and ('search' in query and path.endswith(tuple(["results_preview", "events", "results"])) or self.service.splunk_version < (9,)): + # path = path.replace(PATH_JOBS_V2, PATH_JOBS) + + if api_version == 1: + if isinstance(path, UrlEncoded): + path = UrlEncoded(path.replace(PATH_JOBS_V2, PATH_JOBS), skip_encode=True) + else: + path = path.replace(PATH_JOBS_V2, PATH_JOBS) + return self.service.post(path, owner=owner, app=app, sharing=sharing, **query) @@ -1846,8 +1901,6 @@ class StoragePasswords(Collection): instance. Retrieve this collection using :meth:`Service.storage_passwords`. """ def __init__(self, service): - if service.namespace.owner == '-' or service.namespace.app == '-': - raise ValueError("StoragePasswords cannot have wildcards in namespace.") super(StoragePasswords, self).__init__(service, PATH_STORAGE_PASSWORDS, item=StoragePassword) def create(self, password, username, realm=None): @@ -1865,6 +1918,9 @@ def create(self, password, username, realm=None): :return: The :class:`StoragePassword` object created. """ + if self.service.namespace.owner == '-' or self.service.namespace.app == '-': + raise ValueError("While creating StoragePasswords, namespace cannot have wildcards.") + if not isinstance(username, six.string_types): raise ValueError("Invalid name: %s" % repr(username)) @@ -1896,6 +1952,9 @@ def delete(self, username, realm=None): :return: The `StoragePassword` collection. :rtype: ``self`` """ + if self.service.namespace.owner == '-' or self.service.namespace.app == '-': + raise ValueError("app context must be specified when removing a password.") + if realm is None: # This case makes the username optional, so # the full name can be passed in as realm. @@ -2660,7 +2719,14 @@ def oneshot(self, path, **kwargs): class Job(Entity): """This class represents a search job.""" def __init__(self, service, sid, **kwargs): - path = PATH_JOBS + sid + # Default to v2 in Splunk Version 9+ + path = "{path}{sid}" + # Formatting path based on the Splunk Version + if service.splunk_version < (9,): + path = path.format(path=PATH_JOBS, sid=sid) + else: + path = path.format(path=PATH_JOBS_V2, sid=sid) + Entity.__init__(self, service, path, skip_refresh=True, **kwargs) self.sid = sid @@ -2714,7 +2780,11 @@ def events(self, **kwargs): :return: The ``InputStream`` IO handle to this job's events. """ kwargs['segmentation'] = kwargs.get('segmentation', 'none') - return self.get("events", **kwargs).body + + # Search API v1(GET) and v2(POST) + if self.service.splunk_version < (9,): + return self.get("events", **kwargs).body + return self.post("events", **kwargs).body def finalize(self): """Stops the job and provides intermediate results for retrieval. @@ -2802,7 +2872,11 @@ def results(self, **query_params): :return: The ``InputStream`` IO handle to this job's results. """ query_params['segmentation'] = query_params.get('segmentation', 'none') - return self.get("results", **query_params).body + + # Search API v1(GET) and v2(POST) + if self.service.splunk_version < (9,): + return self.get("results", **query_params).body + return self.post("results", **query_params).body def preview(self, **query_params): """Returns a streaming handle to this job's preview search results. @@ -2843,7 +2917,11 @@ def preview(self, **query_params): :return: The ``InputStream`` IO handle to this job's preview results. """ query_params['segmentation'] = query_params.get('segmentation', 'none') - return self.get("results_preview", **query_params).body + + # Search API v1(GET) and v2(POST) + if self.service.splunk_version < (9,): + return self.get("results_preview", **query_params).body + return self.post("results_preview", **query_params).body def searchlog(self, **kwargs): """Returns a streaming handle to this job's search log. @@ -2932,7 +3010,12 @@ class Jobs(Collection): """This class represents a collection of search jobs. Retrieve this collection using :meth:`Service.jobs`.""" def __init__(self, service): - Collection.__init__(self, service, PATH_JOBS, item=Job) + # Splunk 9 introduces the v2 endpoint + if service.splunk_version >= (9,): + path = PATH_JOBS_V2 + else: + path = PATH_JOBS + Collection.__init__(self, service, path, item=Job) # The count value to say list all the contents of this # Collection is 0, not -1 as it is on most. self.null_count = 0 @@ -3770,4 +3853,4 @@ def batch_save(self, *documents): data = json.dumps(documents) - return json.loads(self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) + return json.loads(self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) \ No newline at end of file diff --git a/splunkrc.spec b/splunkrc.spec deleted file mode 100644 index cd158fcf..00000000 --- a/splunkrc.spec +++ /dev/null @@ -1,12 +0,0 @@ -# Splunk Enterprise host (default: localhost) -host=localhost -# Splunk Enterprise admin port (default: 8089) -port=8089 -# Splunk Enterprise username -username=admin -# Splunk Enterprise password -password=changeme -# Access scheme (default: https) -scheme=https -# Your version of Splunk Enterprise -version=8.0 \ No newline at end of file diff --git a/tests/test_job.py b/tests/test_job.py index 19ec8900..e514c83c 100755 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -382,6 +382,29 @@ def test_search_invalid_query_as_json(self): except Exception as e: self.fail("Got some unexpected error. %s" % e.message) + def test_v1_job_fallback(self): + self.assertEventuallyTrue(self.job.is_done) + self.assertLessEqual(int(self.job['eventCount']), 3) + + preview_stream = self.job.preview(output_mode='json', search='| head 1') + preview_r = results.JSONResultsReader(preview_stream) + self.assertFalse(preview_r.is_preview) + + events_stream = self.job.events(output_mode='json', search='| head 1') + events_r = results.JSONResultsReader(events_stream) + + results_stream = self.job.results(output_mode='json', search='| head 1') + results_r = results.JSONResultsReader(results_stream) + + n_events = len([x for x in events_r if isinstance(x, dict)]) + n_preview = len([x for x in preview_r if isinstance(x, dict)]) + n_results = len([x for x in results_r if isinstance(x, dict)]) + + # Fallback test for Splunk Version 9+ + if self.service.splunk_version[0] >= 9: + self.assertGreaterEqual(9, self.service.splunk_version[0]) + self.assertEqual(n_events, n_preview, n_results) + class TestResultsReader(unittest.TestCase): def test_results_reader(self): diff --git a/tests/test_service.py b/tests/test_service.py index 34afef2c..e3ffef68 100755 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -102,6 +102,11 @@ def test_parse(self): # objectified form of the results, but for now there's # nothing to test but a good response code. response = self.service.parse('search * abc="def" | dedup abc') + + # Splunk Version 9+ using API v2: search/v2/parser + if self.service.splunk_version[0] >= 9: + self.assertGreaterEqual(9, self.service.splunk_version[0]) + self.assertEqual(response.status, 200) def test_parse_fail(self):