Skip to content

Commit

Permalink
Exponential backoff when encountering HTTP 429
Browse files Browse the repository at this point in the history
  • Loading branch information
@addreas
addreas committed Feb 19, 2022
1 parent 1b94fe6 commit 795616c
Show file tree
Hide file tree
Showing 6 changed files with 254 additions and 27 deletions.
33 changes: 24 additions & 9 deletions api/integreatly/v1alpha1/grafanadashboard_types.go
View file
Expand Up @@ -32,15 +32,17 @@ import (

// GrafanaDashboardSpec defines the desired state of GrafanaDashboard
type GrafanaDashboardSpec struct {
Json string `json:"json,omitempty"`
Jsonnet string `json:"jsonnet,omitempty"`
Plugins PluginList `json:"plugins,omitempty"`
Url string `json:"url,omitempty"`
ConfigMapRef *corev1.ConfigMapKeySelector `json:"configMapRef,omitempty"`
Datasources []GrafanaDashboardDatasource `json:"datasources,omitempty"`
CustomFolderName string `json:"customFolderName,omitempty"`
GrafanaCom *GrafanaDashboardGrafanaComSource `json:"grafanaCom,omitempty"`
Json string `json:"json,omitempty"`
Jsonnet string `json:"jsonnet,omitempty"`
Plugins PluginList `json:"plugins,omitempty"`
Url string `json:"url,omitempty"`
ConfigMapRef *corev1.ConfigMapKeySelector `json:"configMapRef,omitempty"`
Datasources []GrafanaDashboardDatasource `json:"datasources,omitempty"`
CustomFolderName string `json:"customFolderName,omitempty"`
GrafanaCom *GrafanaDashboardGrafanaComSource `json:"grafanaCom,omitempty"`
ContentCacheDuration *metav1.Duration `json:"contentCacheDuration,omitempty"`
}

type GrafanaDashboardDatasource struct {
InputName string `json:"inputName"`
DatasourceName string `json:"datasourceName"`
Expand All @@ -63,7 +65,20 @@ type GrafanaDashboardRef struct {
}

type GrafanaDashboardStatus struct {
// Empty
// +optional
Content string `json:"content"`
// +optional
ContentTimestamp *metav1.Time `json:"contentTimestamp"`
// +optional
ContentUrl string `json:"contentUrl"`
// +optional
Error *GrafanaDashboardError `json:"error"`
}

type GrafanaDashboardError struct {
Code int `json:"code"`
Message string `json:"error"`
Retries int `json:"retries,omitempty"`
}

// GrafanaDashboard is the Schema for the grafanadashboards API
Expand Down
31 changes: 30 additions & 1 deletion api/integreatly/v1alpha1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

22 changes: 22 additions & 0 deletions config/crd/bases/integreatly.org_grafanadashboards.yaml
View file
Expand Up @@ -53,6 +53,8 @@ spec:
required:
- key
type: object
contentCacheDuration:
type: string
customFolderName:
type: string
datasources:
Expand Down Expand Up @@ -97,6 +99,26 @@ spec:
type: string
type: object
status:
properties:
content:
type: string
contentTimestamp:
format: date-time
type: string
contentUrl:
type: string
error:
properties:
code:
type: integer
error:
type: string
retries:
type: integer
required:
- code
- error
type: object
type: object
type: object
served: true
Expand Down
85 changes: 69 additions & 16 deletions controllers/grafanadashboard/dashboard_pipeline.go
View file
Expand Up @@ -13,12 +13,14 @@ import (
"path"
"strconv"
"strings"
"time"

"github.com/go-logr/logr"
"github.com/google/go-jsonnet"
"github.com/grafana-operator/grafana-operator/v4/api/integreatly/v1alpha1"
"github.com/grafana-operator/grafana-operator/v4/controllers/config"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
)
Expand Down Expand Up @@ -51,6 +53,9 @@ type DashboardPipelineImpl struct {
}

func NewDashboardPipeline(client client.Client, dashboard *v1alpha1.GrafanaDashboard, ctx context.Context) DashboardPipeline {
if dashboard.Spec.ContentCacheDuration == nil {
dashboard.Spec.ContentCacheDuration = &metav1.Duration{Duration: 24 * time.Hour}
}
return &DashboardPipelineImpl{
Client: client,
Dashboard: dashboard,
Expand Down Expand Up @@ -124,6 +129,11 @@ func (r *DashboardPipelineImpl) obtainJson() error {
return errors.New("both dashboard url and grafana.com source specified")
}

if r.Dashboard.Status.Content != "" && r.Dashboard.Status.ContentTimestamp.Add(r.Dashboard.Spec.ContentCacheDuration.Duration).After(time.Now()) {
r.JSON = r.Dashboard.Status.Content
return nil
}

if r.Dashboard.Spec.GrafanaCom != nil {
if err := r.loadDashboardFromGrafanaCom(); err != nil {
r.Logger.Error(err, "failed to request dashboard from grafana.com, falling back to config map; if specified")
Expand Down Expand Up @@ -196,14 +206,30 @@ func (r *DashboardPipelineImpl) loadDashboardFromURL() error {
}
defer resp.Body.Close()

if resp.StatusCode != 200 {
return fmt.Errorf("request failed with status %v", resp.StatusCode)
}

body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode != 200 {
retries := 0
if r.Dashboard.Status.Error != nil {
retries = r.Dashboard.Status.Error.Retries
}
r.Dashboard.Status = v1alpha1.GrafanaDashboardStatus{
Error: &v1alpha1.GrafanaDashboardError{
Message: string(body),
Code: resp.StatusCode,
Retries: retries + 1,
},
ContentTimestamp: &metav1.Time{Time: time.Now()},
}

if err := r.Client.Status().Update(r.Context, r.Dashboard); err != nil {
return fmt.Errorf("failed to request dashboard and failed to update status %s: %w", string(body), err)
}

return fmt.Errorf("request failed with status %v", resp.StatusCode)
}
sourceType := r.getFileType(url.Path)

switch sourceType {
Expand All @@ -218,13 +244,14 @@ func (r *DashboardPipelineImpl) loadDashboardFromURL() error {
r.JSON = json
}

// Update dashboard spec so that URL would not be refetched
if r.JSON != r.Dashboard.Spec.Json {
r.Dashboard.Spec.Json = r.JSON
err := r.Client.Update(r.Context, r.Dashboard)
if err != nil {
return err
}
r.Dashboard.Status = v1alpha1.GrafanaDashboardStatus{
Content: r.JSON,
ContentTimestamp: &metav1.Time{Time: time.Now()},
ContentUrl: r.Dashboard.Spec.Url,
}

if err := r.Client.Status().Update(r.Context, r.Dashboard); err != nil {
return fmt.Errorf("failed to update status with content for dashboard %s/%s: %w", r.Dashboard.Namespace, r.Dashboard.Name, err)
}

return nil
Expand All @@ -246,14 +273,40 @@ func (r *DashboardPipelineImpl) loadDashboardFromGrafanaCom() error {
if err != nil {
return err
}

if resp.StatusCode != 200 {
retries := 0
if r.Dashboard.Status.Error != nil {
retries = r.Dashboard.Status.Error.Retries
}
r.Dashboard.Status = v1alpha1.GrafanaDashboardStatus{
Error: &v1alpha1.GrafanaDashboardError{
Message: string(body),
Code: resp.StatusCode,
Retries: retries + 1,
},
ContentTimestamp: &metav1.Time{Time: time.Now()},
}

if err := r.Client.Status().Update(r.Context, r.Dashboard); err != nil {
return fmt.Errorf("failed to request dashboard and failed to update status %s: %w", string(body), err)
}

return fmt.Errorf("failed to request dashboard: %s", string(body))
}

r.JSON = string(body)

// Update JSON so dashboard is not refetched
if r.JSON != r.Dashboard.Spec.Json {
r.Dashboard.Spec.Json = r.JSON
if err := r.Client.Update(r.Context, r.Dashboard); err != nil {
return err
}

r.Dashboard.Status = v1alpha1.GrafanaDashboardStatus{
Content: r.JSON,
ContentTimestamp: &metav1.Time{Time: time.Now()},
ContentUrl: url,
}

if err := r.Client.Status().Update(r.Context, r.Dashboard); err != nil {
return fmt.Errorf("failed to update status with content for dashboard %s/%s: %w", r.Dashboard.Namespace, r.Dashboard.Name, err)
}

return nil
Expand Down
10 changes: 10 additions & 0 deletions controllers/grafanadashboard/grafanadashboard_controller.go
View file
Expand Up @@ -21,6 +21,7 @@ import (
"crypto/tls"
"errors"
"fmt"
"math"
"net/http"
"os"

Expand Down Expand Up @@ -276,6 +277,15 @@ func (r *GrafanaDashboardReconciler) reconcileDashboards(request reconcile.Reque
folderName = dashboard.Spec.CustomFolderName
}

if dashboard.Status.Error != nil && dashboard.Status.Error.Code == 429 {
backoffDuration := 30 * time.Second * time.Duration(math.Pow(2, float64(dashboard.Status.Error.Retries)))

if dashboard.Status.ContentTimestamp.Add(backoffDuration).After(time.Now()) {
log.Log.Info("still awaiting rate limit for dashboard", "folder", folderName, "dashboard", request.Name)
continue
}
}

folder, err := grafanaClient.CreateOrUpdateFolder(folderName)

if err != nil {
Expand Down

0 comments on commit 795616c

Please sign in to comment.