From 96612af0d9bff5c1ec67137c37ae89c2ef11a68c Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Tue, 23 Jun 2020 13:08:10 +1000 Subject: [PATCH 1/4] Fix broken curl in multiarch build --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 734043d..0ad69cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -132,6 +132,7 @@ jobs: name: Push multiarch Docker image command: | apk add -U make bash curl + apk upgrade make manifest-push DOCKER_IMAGE_TAG=${CIRCLE_TAG:-latest} workflows: From f6bd6859182360f345e02a04e0abbd6d10fd790b Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Wed, 24 Jun 2020 11:34:06 +1000 Subject: [PATCH 2/4] Fix counter on webhook retries to show correct total value of retries --- configmap-reload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configmap-reload.go b/configmap-reload.go index 9aa2ab5..8505a3f 100644 --- a/configmap-reload.go +++ b/configmap-reload.go @@ -116,7 +116,7 @@ func main() { } } for retries := *webhookRetries; retries != 0; retries-- { - log.Printf("performing webhook request (%d/%d)", retries, *webhookRetries+1) + log.Printf("performing webhook request (%d/%d)", retries, *webhookRetries) resp, err := http.DefaultClient.Do(req) if err != nil { setFailureMetrics(h.String(), "client_request_do") From 33231c9f8b91fed7e3f548c5e7ca7f591c76e50b Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Wed, 24 Jun 2020 16:37:14 +1000 Subject: [PATCH 3/4] Fix retry logic: Use break instead of return. --- configmap-reload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configmap-reload.go b/configmap-reload.go index 8505a3f..15167ba 100644 --- a/configmap-reload.go +++ b/configmap-reload.go @@ -134,7 +134,7 @@ func main() { } setSuccessMetrict(h.String(), begun) log.Println("successfully triggered reload") - return + break } } case err := <-watcher.Errors: From ca3148ff892afe853f9fb163809f00e23e1ab719 Mon Sep 17 00:00:00 2001 From: Miguel Elias dos Santos Date: Mon, 6 Jul 2020 10:16:12 +1000 Subject: [PATCH 4/4] Add proper metrics when retries are exhausted --- configmap-reload.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/configmap-reload.go b/configmap-reload.go index 15167ba..2f0431d 100644 --- a/configmap-reload.go +++ b/configmap-reload.go @@ -115,6 +115,9 @@ func main() { req.SetBasicAuth(userInfo.Username(), password) } } + + successfulReloadWebhook := false + for retries := *webhookRetries; retries != 0; retries-- { log.Printf("performing webhook request (%d/%d)", retries, *webhookRetries) resp, err := http.DefaultClient.Do(req) @@ -132,10 +135,17 @@ func main() { time.Sleep(time.Second * 10) continue } + setSuccessMetrict(h.String(), begun) log.Println("successfully triggered reload") + successfulReloadWebhook = true break } + + if !successfulReloadWebhook { + setFailureMetrics(h.String(), "retries_exhausted") + log.Println("error:", "Webhook reload retries exhausted") + } } case err := <-watcher.Errors: watcherErrors.Inc()