From af44021f3a595637b4ef2c3a873d628b309905a1 Mon Sep 17 00:00:00 2001 From: Luca Cillario Date: Mon, 21 Nov 2022 11:03:19 +0100 Subject: [PATCH 1/5] Retry documentation. (#2166) (#2456) * Retry documentation. (#2166) * Fixed typo. * Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> * Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> * Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> * Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> * Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> * Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> * Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> * Update docs/retry.rst Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> Co-authored-by: Nermina Miller <102551568+nermiller@users.noreply.github.com> --- docs/backoff.rst | 2 ++ docs/exceptions.rst | 2 +- docs/retry.rst | 67 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/docs/backoff.rst b/docs/backoff.rst index e640b5682e..c5ab01ab03 100644 --- a/docs/backoff.rst +++ b/docs/backoff.rst @@ -1,3 +1,5 @@ +.. _backoff-label: + Backoff ############# diff --git a/docs/exceptions.rst b/docs/exceptions.rst index b8aeb33e49..8a9fe457fb 100644 --- a/docs/exceptions.rst +++ b/docs/exceptions.rst @@ -1,4 +1,4 @@ - +.. _exceptions-label: Exceptions ########## diff --git a/docs/retry.rst b/docs/retry.rst index 2b4f22c2f6..acf198ec94 100644 --- a/docs/retry.rst +++ b/docs/retry.rst @@ -2,4 +2,69 @@ Retry Helpers ############# .. automodule:: redis.retry - :members: \ No newline at end of file + :members: + + +Retry in Redis Standalone +************************** + +>>> from redis.backoff import ExponentialBackoff +>>> from redis.retry import Retry +>>> from redis.client import Redis +>>> from redis.exceptions import ( +>>> BusyLoadingError, +>>> ConnectionError, +>>> TimeoutError +>>> ) +>>> +>>> # Run 3 retries with exponential backoff strategy +>>> retry = Retry(ExponentialBackoff(), 3) +>>> # Redis client with retries on custom errors +>>> r = Redis(host='localhost', port=6379, retry=retry, retry_on_error=[BusyLoadingError, ConnectionError, TimeoutError]) +>>> # Redis client with retries on TimeoutError only +>>> r_only_timeout = Redis(host='localhost', port=6379, retry=retry, retry_on_timeout=True) + +As you can see from the example above, Redis client supports 3 parameters to configure the retry behaviour: + +* ``retry``: :class:`~.Retry` instance with a :ref:`backoff-label` strategy and the max number of retries +* ``retry_on_error``: list of :ref:`exceptions-label` to retry on +* ``retry_on_timeout``: if ``True``, retry on :class:`~.TimeoutError` only + +If either ``retry_on_error`` or ``retry_on_timeout`` are passed and no ``retry`` is given, +by default it uses a ``Retry(NoBackoff(), 1)`` (meaning 1 retry right after the first failure). + + +Retry in Redis Cluster +************************** + +>>> from redis.backoff import ExponentialBackoff +>>> from redis.retry import Retry +>>> from redis.cluster import RedisCluster +>>> +>>> # Run 3 retries with exponential backoff strategy +>>> retry = Retry(ExponentialBackoff(), 3) +>>> # Redis Cluster client with retries +>>> rc = RedisCluster(host='localhost', port=6379, retry=retry, cluster_error_retry_attempts=2) + +Retry behaviour in Redis Cluster is a little bit different from Standalone: + +* ``retry``: :class:`~.Retry` instance with a :ref:`backoff-label` strategy and the max number of retries, default value is ``Retry(NoBackoff(), 0)`` +* ``cluster_error_retry_attempts``: number of times to retry before raising an error when :class:`~.TimeoutError` or :class:`~.ConnectionError` or :class:`~.ClusterDownError` are encountered, default value is ``3`` + +Let's consider the following example: + +>>> from redis.backoff import ExponentialBackoff +>>> from redis.retry import Retry +>>> from redis.cluster import RedisCluster +>>> +>>> rc = RedisCluster(host='localhost', port=6379, retry=Retry(ExponentialBackoff(), 6), cluster_error_retry_attempts=1) +>>> rc.set('foo', 'bar') + +#. the client library calculates the hash slot for key 'foo'. +#. given the hash slot, it then determines which node to connect to, in order to execute the command. +#. during the connection, a :class:`~.ConnectionError` is raised. +#. because we set ``retry=Retry(ExponentialBackoff(), 6)``, the client tries to reconnect to the node up to 6 times, with an exponential backoff between each attempt. +#. even after 6 retries, the client is still unable to connect. +#. because we set ``cluster_error_retry_attempts=1``, before giving up, the client starts a cluster update, removes the failed node from the startup nodes, and re-initializes the cluster. +#. after the cluster has been re-initialized, it starts a new cycle of retries, up to 6 retries, with an exponential backoff. +#. if the client can connect, we're good. Otherwise, the exception is finally raised to the caller, because we've run out of attempts. \ No newline at end of file From 59c8e7d05a5cf79c933f0c7e0fef0dfee93b0ed4 Mon Sep 17 00:00:00 2001 From: Bar Shaul <88437685+barshaul@users.noreply.github.com> Date: Mon, 21 Nov 2022 12:03:53 +0200 Subject: [PATCH 2/5] Removed bad tokens from connection_examples.ipynb to fix the json output (#2455) --- docs/examples/connection_examples.ipynb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/examples/connection_examples.ipynb b/docs/examples/connection_examples.ipynb index ca8dd443c6..a15b4c6cc0 100644 --- a/docs/examples/connection_examples.ipynb +++ b/docs/examples/connection_examples.ipynb @@ -116,7 +116,6 @@ "user_connection.ping()" ], "metadata": {} - } }, { "cell_type": "markdown", @@ -124,7 +123,6 @@ "## Connecting to a redis instance with standard credential provider" ], "metadata": {} - } }, { "cell_type": "code", @@ -162,7 +160,6 @@ "user_connection.ping()" ], "metadata": {} - } }, { "cell_type": "markdown", @@ -170,7 +167,6 @@ "## Connecting to a redis instance first with an initial credential set and then calling the credential provider" ], "metadata": {} - } }, { "cell_type": "code", @@ -200,7 +196,6 @@ "cred_provider = InitCredsSetCredentialProvider(username=\"init_user\", password=\"init_pass\")" ], "metadata": {} - } }, { "cell_type": "markdown", From e425674d84a63762f16d5b44b19aa70119fcd814 Mon Sep 17 00:00:00 2001 From: SyedTahaA <62918944+SyedTahaA@users.noreply.github.com> Date: Mon, 21 Nov 2022 05:57:30 -0500 Subject: [PATCH 3/5] Fix Sentinel.execute_command to execute across the entire sentinel cluster (#2459) * Change sentinel execute command to execute across the entire cluster * Add change to CHANGES file --- CHANGES | 1 + redis/sentinel.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 883c548f38..c0b183e8e3 100644 --- a/CHANGES +++ b/CHANGES @@ -28,6 +28,7 @@ * Fixed "cannot pickle '_thread.lock' object" bug (#2354, #2297) * Added CredentialsProvider class to support password rotation * Enable Lock for asyncio cluster mode + * Fix Sentinel.execute_command doesn't execute across the entire sentinel cluster bug (#2458) * 4.1.3 (Feb 8, 2022) * Fix flushdb and flushall (#1926) diff --git a/redis/sentinel.py b/redis/sentinel.py index d35abaf514..d70b7142b5 100644 --- a/redis/sentinel.py +++ b/redis/sentinel.py @@ -200,10 +200,10 @@ def execute_command(self, *args, **kwargs): kwargs.pop("once") if once: + random.choice(self.sentinels).execute_command(*args, **kwargs) + else: for sentinel in self.sentinels: sentinel.execute_command(*args, **kwargs) - else: - random.choice(self.sentinels).execute_command(*args, **kwargs) return True def __repr__(self): From e3e223bbd9ac49c8f17d0eec518caa55c55cc92e Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 21 Nov 2022 14:25:29 +0200 Subject: [PATCH 4/5] GitHub Workflows security hardening (#2444) * build: harden pypi-publish.yaml permissions Signed-off-by: Alex * build: harden stale-issues.yml permissions Signed-off-by: Alex * build: harden release-drafter.yml permissions Signed-off-by: Alex * build: harden integration.yaml permissions Signed-off-by: Alex Signed-off-by: Alex Co-authored-by: Chayim --- .github/workflows/integration.yaml | 3 +++ .github/workflows/pypi-publish.yaml | 3 +++ .github/workflows/release-drafter.yml | 5 +++++ .github/workflows/stale-issues.yml | 5 +++++ 4 files changed, 16 insertions(+) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index c2b84a6074..8d38cd45c7 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -16,6 +16,9 @@ on: schedule: - cron: '0 1 * * *' # nightly build +permissions: + contents: read # to fetch code (actions/checkout) + jobs: dependency-audit: diff --git a/.github/workflows/pypi-publish.yaml b/.github/workflows/pypi-publish.yaml index 3e7f80136c..50332c1995 100644 --- a/.github/workflows/pypi-publish.yaml +++ b/.github/workflows/pypi-publish.yaml @@ -4,6 +4,9 @@ on: release: types: [published] +permissions: + contents: read # to fetch code (actions/checkout) + jobs: build_and_package: diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index ec2d88bf6e..eebb3e678b 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -6,8 +6,13 @@ on: branches: - master +permissions: {} jobs: update_release_draft: + permissions: + pull-requests: write # to add label to PR (release-drafter/release-drafter) + contents: write # to create a github release (release-drafter/release-drafter) + runs-on: ubuntu-latest steps: # Drafts your next Release notes as Pull Requests are merged into "master" diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/stale-issues.yml index 562cd582b1..32fd9e8179 100644 --- a/.github/workflows/stale-issues.yml +++ b/.github/workflows/stale-issues.yml @@ -3,8 +3,13 @@ on: schedule: - cron: "0 0 * * *" +permissions: {} jobs: stale: + permissions: + issues: write # to close stale issues (actions/stale) + pull-requests: write # to close stale PRs (actions/stale) + runs-on: ubuntu-latest steps: - uses: actions/stale@v3 From f492f85af723b277fa1ec99f01e6612e5d567b3f Mon Sep 17 00:00:00 2001 From: Igor Malinovskiy Date: Mon, 21 Nov 2022 23:24:15 +0100 Subject: [PATCH 5/5] Install package deps in readthedocs build (#2465) Dependencies are required for "automodule" generation. Co-authored-by: Igor Malinovskiy Co-authored-by: Chayim --- .readthedocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 80b9738d82..800cb14816 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,6 +3,7 @@ version: 2 python: install: - requirements: ./docs/requirements.txt + - requirements: requirements.txt build: os: ubuntu-20.04