Merge pull request #443 from splunk/release/1.6.19

Release/1.6.19
splunk · Mar 29, 2022 · 19acb9a · 19acb9a
2 parents ef88e9d + adb6fcd
commit 19acb9a
Show file tree

Hide file tree

Showing 91 changed files with 955 additions and 395 deletions.
diff --git a/.env b/.env
@@ -0,0 +1,16 @@
+# Splunk host (default: localhost)
+host=localhost
+# Splunk admin port (default: 8089)
+port=8089
+# Splunk username
+username=admin
+# Splunk password
+password=changed!
+# Access scheme (default: https)
+scheme=https
+# Your version of Splunk (default: 6.2)
+version=8.0
+# Bearer token for authentication
+#bearerToken="<Bearer-token>"
+# Session key for authentication
+#sessionKey="<Session-Key>"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -3,7 +3,7 @@ on:
   push:
     branches:
       - master
-  workflow_dispatch: {}
+  workflow_dispatch: { }
 
 jobs:
   find_version:
@@ -21,6 +21,13 @@ jobs:
       - name: Get version
         id: get-version
         run: python -c 'import splunklib; print("::set-output name=version::%s" % splunklib.__version__)'
+      - name: Install tox
+        run: pip install tox
+      - name: Generate API docs
+        run: |
+          rm -rf ./docs/_build
+          tox -e docs
+          cd ./docs/_build/html && zip -r ../docs_html.zip . -x ".*" -x "__MACOSX"
   tag_version:
     needs: find_version
     name: Tag Version
@@ -32,7 +39,7 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           tag: ${{ needs.find_version.outputs.version }}
   release:
-    needs: [find_version, tag_version]
+    needs: [ find_version, tag_version ]
     name: Create Release
     runs-on: ubuntu-latest
     steps:
@@ -52,6 +59,11 @@ jobs:
             **TODO: Insert CHANGELOG.md contents here.**
           draft: false
           prerelease: false
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: apidocs
+          path: docs/_build/docs_html.zip
   publish:
     needs: release
     name: Deploy Release to PyPI

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -1,7 +1,7 @@
 name: Python CI
 
 on:
-  [push, pull_request]
+  [ push, pull_request ]
 
 jobs:
   build:
@@ -11,7 +11,7 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
-        python: [2.7, 3.7]
+        python: [ 2.7, 3.7 ]
         splunk-version:
           - "8.0"
           - "latest"
@@ -36,15 +36,6 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}
-      - name: Create .splunkrc file
-        run: |
-          cd ~
-          echo host=localhost > .splunkrc
-          echo port=8089 >> .splunkrc
-          echo username=admin >> .splunkrc
-          echo password=changed! >> .splunkrc
-          echo scheme=https >> .splunkrc
-          echo version=${{ matrix.splunk }} >> .splunkrc
       - name: Install tox
         run: pip install tox
       - name: Test Execution

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,42 @@
 # Splunk Enterprise SDK for Python Changelog
 
+## Version 1.6.19
+
+### New features and APIs
+* [#441](https://github.com/splunk/splunk-sdk-python/pull/441) JSONResultsReader added and deprecated ResultsReader
+  * Pre-requisite: Query parameter 'output_mode' must be set to 'json'
+  * Improves performance by approx ~80-90%
+  * ResultsReader is deprecated and will be removed in future releases (NOTE: Please migrate to JSONResultsReader)
+* [#437](https://github.com/splunk/splunk-sdk-python/pull/437) added setup_logging() method in splunklib for logging
+* [#426](https://github.com/splunk/splunk-sdk-python/pull/426) Added new github_commit modular input example
+* [#392](https://github.com/splunk/splunk-sdk-python/pull/392) Break out search argument to option parsing for v2 custom search commands
+* [#384](https://github.com/splunk/splunk-sdk-python/pull/384) Added Float parameter validator for custom search commands
+* [#371](https://github.com/splunk/splunk-sdk-python/pull/371) Modinput preserve 'app' context
+
+### Bug fixes
+* [#439](https://github.com/splunk/splunk-sdk-python/pull/439) Modified POST method debug log to not log sensitive body/data
+* [#431](https://github.com/splunk/splunk-sdk-python/pull/431) Add distsearch.conf to Stream Search Command examples [ [issue#418](https://github.com/splunk/splunk-sdk-python/issues/418) ]
+* [#419](https://github.com/splunk/splunk-sdk-python/pull/419) Hec endpoint issue[ [issue#345](https://github.com/splunk/splunk-sdk-python/issues/345) ]
+* [#416](https://github.com/splunk/splunk-sdk-python/pull/416) Removed strip() method in load_value() method from data.py file [ [issue#400](https://github.com/splunk/splunk-sdk-python/issues/400) ]
+* [#148](https://github.com/splunk/splunk-sdk-python/pull/148) Identical entity names will cause an infinite loop
+
+### Minor changes
+* [#440](https://github.com/splunk/splunk-sdk-python/pull/440) Github release workflow modified to generate docs
+* [#430](https://github.com/splunk/splunk-sdk-python/pull/430) Fix indentation in README
+* [#429](https://github.com/splunk/splunk-sdk-python/pull/429) documented how to access modular input metadata
+* [#427](https://github.com/splunk/splunk-sdk-python/pull/427) Replace .splunkrc with .env file in test and examples
+* [#424](https://github.com/splunk/splunk-sdk-python/pull/424) Float validator test fix
+* [#423](https://github.com/splunk/splunk-sdk-python/pull/423) Python3 compatibility for ResponseReader.__str__()
+* [#422](https://github.com/splunk/splunk-sdk-python/pull/422) ordereddict and all its reference removed
+* [#421](https://github.com/splunk/splunk-sdk-python/pull/421) Update README.md
+* [#387](https://github.com/splunk/splunk-sdk-python/pull/387) Update filter.py
+* [#331](https://github.com/splunk/splunk-sdk-python/pull/331) Fix a couple of warnings spotted when running python 2.7 tests
+* [#330](https://github.com/splunk/splunk-sdk-python/pull/330) client: use six.string_types instead of basestring
+* [#329](https://github.com/splunk/splunk-sdk-python/pull/329) client: remove outdated comment in Index.submit
+* [#262](https://github.com/splunk/splunk-sdk-python/pull/262) properly add parameters to request based on the method of the request
+* [#237](https://github.com/splunk/splunk-sdk-python/pull/237) Don't output close tags if you haven't written a start tag
+* [#149](https://github.com/splunk/splunk-sdk-python/pull/149) "handlers" stanza missing in examples/searchcommands_template/default/logging.conf
+
 ## Version 1.6.18
 
 ### Bug fixes

diff --git a/Makefile b/Makefile
@@ -56,16 +56,16 @@ test_smoke_no_app:
 	@echo "$(ATTN_COLOR)==> test_smoke_no_app $(NO_COLOR)"
 	@tox -e py27,py37 -- -m "smoke and not app"
 
-.PHONY: splunkrc
-splunkrc:
-	@echo "$(ATTN_COLOR)==> splunkrc $(NO_COLOR)"
-	@echo "To make a .splunkrc:"
-	@echo "  [SPLUNK_INSTANCE_JSON] | python scripts/build-splunkrc.py ~/.splunkrc"
-
-.PHONY: splunkrc_default
-splunkrc_default:
-	@echo "$(ATTN_COLOR)==> splunkrc_default $(NO_COLOR)"
-	@python scripts/build-splunkrc.py ~/.splunkrc
+.PHONY: env
+env:
+	@echo "$(ATTN_COLOR)==> env $(NO_COLOR)"
+	@echo "To make a .env:"
+	@echo "  [SPLUNK_INSTANCE_JSON] | python scripts/build-env.py"
+
+.PHONY: env_default
+env_default:
+	@echo "$(ATTN_COLOR)==> env_default $(NO_COLOR)"
+	@python scripts/build-env.py
 
 .PHONY: up
 up:

diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@
 
 # The Splunk Enterprise Software Development Kit for Python
 
-#### Version 1.6.18
+#### Version 1.6.19
 
 The Splunk Enterprise Software Development Kit (SDK) for Python contains library code and examples designed to enable developers to build applications using the Splunk platform.
 
@@ -60,7 +60,6 @@ You'll need `docker` and `docker-compose` to get up and running using this metho
 ```
 make up SPLUNK_VERSION=8.0
 make wait_up
-make splunkrc_default
 make test
 make down
 ```
@@ -75,7 +74,7 @@ The SDK command-line examples require a common set of arguments that specify the
 #### Using username/password
 ```python
 import splunklib.client as client
-    service = client.connect(host=<host_url>, username=<username>, password=<password>, autoLogin=True)
+service = client.connect(host=<host_url>, username=<username>, password=<password>, autologin=True)
 ```
 
 #### Using bearer token
@@ -91,13 +90,13 @@ service = client.connect(host=<host_url>, token=<session_key>, autologin=True)
 ```
 
 ###
-#### Create a .splunkrc convenience file
+#### Update a .env file
 
-To connect to Splunk Enterprise, many of the SDK examples and unit tests take command-line arguments that specify values for the host, port, and login credentials for Splunk Enterprise. For convenience during development, you can store these arguments as key-value pairs in a text file named **.splunkrc**. Then, the SDK examples and unit tests use the values from the **.splunkrc** file when you don't specify them.
+To connect to Splunk Enterprise, many of the SDK examples and unit tests take command-line arguments that specify values for the host, port, and login credentials for Splunk Enterprise. For convenience during development, you can store these arguments as key-value pairs in a **.env** file. Then, the SDK examples and unit tests use the values from the **.env** file when you don't specify them.
 
->**Note**: Storing login credentials in the **.splunkrc** file is only for convenience during development. This file isn't part of the Splunk platform and shouldn't be used for storing user credentials for production. And, if you're at all concerned about the security of your credentials, enter them at the command line rather than saving them in this file.
+>**Note**: Storing login credentials in the **.env** file is only for convenience during development. This file isn't part of the Splunk platform and shouldn't be used for storing user credentials for production. And, if you're at all concerned about the security of your credentials, enter them at the command line rather than saving them in this file.
 
-To use this convenience file, create a text file with the following format:
+here is an example of .env file:
 
     # Splunk Enterprise host (default: localhost)
     host=localhost
@@ -106,27 +105,15 @@ To use this convenience file, create a text file with the following format:
     # Splunk Enterprise username
     username=admin
     # Splunk Enterprise password
-    password=changeme
+    password=changed!
     # Access scheme (default: https)
     scheme=https
     # Your version of Splunk Enterprise
     version=8.0
-
-Save the file as **.splunkrc** in the current user's home directory.
-
-*   For example on OS X, save the file as:
-
-        ~/.splunkrc
-
-*   On Windows, save the file as:
-
-        C:\Users\currentusername\.splunkrc
-
-    You might get errors in Windows when you try to name the file because ".splunkrc" appears to be a nameless file with an extension. You can use the command line to create this file by going to the **C:\Users\\&lt;currentusername&gt;** directory and entering the following command:
-
-        Notepad.exe .splunkrc
-
-    Click **Yes**, then continue creating the file.
+    # Bearer token for authentication
+    #bearerToken=<Bearer-token>
+    # Session key for authentication
+    #sessionKey=<Session-Key>
 
 #### Run the examples
 
@@ -144,7 +131,7 @@ Using Session key
 
     python examplename.py --sessionKey="<value>"
 
-If you saved your login credentials in the **.splunkrc** file, you can omit those arguments:
+If you saved your login credentials in the **.env** file, you can omit those arguments:
 
     python examplename.py
 
@@ -212,19 +199,48 @@ class CustomStreamingCommand(StreamingCommand):
 Do
 ```python
 @Configuration()
-    class GeneratorTest(GeneratingCommand):
-        def generate(self):
-            yield self.gen_record(_time=time.time(), one=1)
-            yield self.gen_record(_time=time.time(), two=2)
+class GeneratorTest(GeneratingCommand):
+    def generate(self):
+        yield self.gen_record(_time=time.time(), one=1)
+        yield self.gen_record(_time=time.time(), two=2)
 ```
 
 Don't
 ```python
 @Configuration()
-    class GeneratorTest(GeneratingCommand):
-        def generate(self):
-            yield {'_time': time.time(), 'one': 1}
-            yield {'_time': time.time(), 'two': 2}
+class GeneratorTest(GeneratingCommand):
+    def generate(self):
+        yield {'_time': time.time(), 'one': 1}
+        yield {'_time': time.time(), 'two': 2}
+```
+
+### Access metadata of modular inputs app
+* In stream_events() method we can access modular input app metadata from InputDefinition object
+* See [GitHub Commit](https://github.com/splunk/splunk-sdk-python/blob/develop/examples/github_commits/bin/github_commits.py) Modular input App example for reference.
+```python
+    def stream_events(self, inputs, ew):
+        # other code
+
+        # access metadata (like server_host, server_uri, etc) of modular inputs app from InputDefinition object
+        # here inputs is a InputDefinition object
+        server_host = inputs.metadata["server_host"]
+        server_uri = inputs.metadata["server_uri"]
+
+        # Get the checkpoint directory out of the modular input's metadata
+        checkpoint_dir = inputs.metadata["checkpoint_dir"]
+```
+
+#### Optional:Set up logging for splunklib
++ The default level is WARNING, which means that only events of this level and above will be visible
++ To change a logging level we can call setup_logging() method and pass the logging level as an argument.
++ Optional: we can also pass log format and date format string as a method argument to modify default format
+
+```python
+import logging
+from splunklib import setup_logging
+
+# To see debug and above level logs
+setup_logging(logging.DEBUG)
 ```
 
 ### Changelog

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -14,6 +14,8 @@ services:
             - ./splunklib:/opt/splunk/etc/apps/github_forks/lib/splunklib
             - ./examples/random_numbers:/opt/splunk/etc/apps/random_numbers
             - ./splunklib:/opt/splunk/etc/apps/random_numbers/lib/splunklib
+            - ./examples/github_commits:/opt/splunk/etc/apps/github_commits
+            - ./splunklib:/opt/splunk/etc/apps/github_commits/lib/splunklib
             - ./examples/searchcommands_app/package:/opt/splunk/etc/apps/searchcommands_app
             - ./splunklib:/opt/splunk/etc/apps/searchcommands_app/lib/splunklib
             - ./examples/twitted/twitted:/opt/splunk/etc/apps/twitted

diff --git a/docs/searchcommands.rst b/docs/searchcommands.rst
@@ -88,6 +88,10 @@ splunklib.searchcommands
     :members:
     :inherited-members:
 
+.. autoclass:: Float
+    :members:
+    :inherited-members:
+
 .. autoclass:: RegularExpression
     :members:
     :inherited-members:

diff --git a/examples/analytics/input.py b/examples/analytics/input.py
@@ -102,7 +102,7 @@ def main():
 
     argv = sys.argv[1:]
 
-    splunk_opts = utils.parse(argv, {}, ".splunkrc", usage=usage)
+    splunk_opts = utils.parse(argv, {}, ".env", usage=usage)
     tracker = AnalyticsTracker("cli_app", splunk_opts.kwargs)
 
     #tracker.track("test_event", "abc123", foo="bar", bar="foo")

diff --git a/examples/analytics/output.py b/examples/analytics/output.py
@@ -152,7 +152,7 @@ def main():
 
     argv = sys.argv[1:]
 
-    opts = utils.parse(argv, {}, ".splunkrc", usage=usage)
+    opts = utils.parse(argv, {}, ".env", usage=usage)
     retriever = AnalyticsRetriever(opts.args[0], opts.kwargs)    
 
     #events = retriever.events()

diff --git a/examples/analytics/server.py b/examples/analytics/server.py
@@ -146,7 +146,7 @@ def application(name):
 def main():
     argv = sys.argv[1:]
 
-    opts = utils.parse(argv, {}, ".splunkrc")
+    opts = utils.parse(argv, {}, ".env")
     global splunk_opts
     splunk_opts = opts.kwargs
 

diff --git a/examples/async/async.py b/examples/async/async.py
@@ -51,7 +51,7 @@ def main(argv):
     usage = "async.py <sync | async>"
 
     # Parse the command line args.
-    opts = parse(argv, {}, ".splunkrc")
+    opts = parse(argv, {}, ".env")
 
     # We have to see if we got either the "sync" or
     # "async" command line arguments.

diff --git a/examples/binding1.py b/examples/binding1.py
@@ -52,7 +52,7 @@ def search(self, query, **kwargs):
         return self.context.post("search/jobs/export", search=query, **kwargs)
 
 def main(argv):
-    opts = parse(argv, {}, ".splunkrc")
+    opts = parse(argv, {}, ".env")
     context = connect(**opts.kwargs)
     service = Service(context)
     assert service.apps().status == 200

diff --git a/examples/conf.py b/examples/conf.py
@@ -151,7 +151,7 @@ def main():
     commands = ['create', 'delete', 'list']
 
     # parse args, connect and setup 
-    opts = parse(argv, {}, ".splunkrc", usage=usage)
+    opts = parse(argv, {}, ".env", usage=usage)
     service = connect(**opts.kwargs)
     program = Program(service)
 

diff --git a/examples/dashboard/feed.py b/examples/dashboard/feed.py
@@ -171,7 +171,7 @@ def iterate(job):
 
 def main(argv):
     # Parse the command line args.
-    opts = parse(argv, {}, ".splunkrc")
+    opts = parse(argv, {}, ".env")
 
     # Connect to Splunk
     service = client.connect(**opts.kwargs)

diff --git a/examples/event_types.py b/examples/event_types.py
@@ -30,7 +30,7 @@
                     "(e.g., export PYTHONPATH=~/splunk-sdk-python.")
 
 def main():
-    opts = parse(sys.argv[1:], {}, ".splunkrc")
+    opts = parse(sys.argv[1:], {}, ".env")
     service = connect(**opts.kwargs)
 
     for item in service.event_types: