Exponential backoff when encountering HTTP 429 and add status field to grafana

api/integreatly/v1alpha1/grafana_types.go

@@ -38,6 +38,10 @@ type GrafanaSpec struct {
 	InitImage                  string                   `json:"initImage,omitempty"`
 	LivenessProbeSpec          *LivenessProbeSpec       `json:"livenessProbeSpec,omitempty"`
 	ReadinessProbeSpec         *ReadinessProbeSpec      `json:"readinessProbeSpec,omitempty"`
+
+	// DashboardContentCacheDuration sets a default for when a `GrafanaDashboard` resource doesn't specify a `contentCacheDuration`.
+	// If left unset or 0 the default behavior is to cache indefinitely.
+	DashboardContentCacheDuration *metav1.Duration `json:"dashboardContentCacheDuration,omitempty"`
 }
 
 type ReadinessProbeSpec struct {

api/integreatly/v1alpha1/grafanadashboard_types.go

@@ -40,7 +40,14 @@ type GrafanaDashboardSpec struct {
 	Datasources      []GrafanaDashboardDatasource      `json:"datasources,omitempty"`
 	CustomFolderName string                            `json:"customFolderName,omitempty"`
 	GrafanaCom       *GrafanaDashboardGrafanaComSource `json:"grafanaCom,omitempty"`
+
+	// ContentCacheDuration sets how often the operator should resync with the external source when using
+	// the `grafanaCom.id` or `url` field to specify the source of the dashboard. The default value is
+	// decided by the `dashboardContentCacheDuration` field in the `Grafana` resource. The default is 0 which
+	// is interpreted as never refetching.
+	ContentCacheDuration *metav1.Duration `json:"contentCacheDuration,omitempty"`
 }
+
 type GrafanaDashboardDatasource struct {
 	InputName      string `json:"inputName"`
 	DatasourceName string `json:"datasourceName"`
@@ -63,7 +70,16 @@ type GrafanaDashboardRef struct {
 }
 
 type GrafanaDashboardStatus struct {
-	// Empty
+	Content          string                 `json:"content,omitempty"`
+	ContentTimestamp *metav1.Time           `json:"contentTimestamp,omitempty"`
+	ContentUrl       string                 `json:"contentUrl,omitempty"`
+	Error            *GrafanaDashboardError `json:"error,omitempty"`
+}
+
+type GrafanaDashboardError struct {
+	Code    int    `json:"code"`
+	Message string `json:"error"`
+	Retries int    `json:"retries,omitempty"`
 }
 
 // GrafanaDashboard is the Schema for the grafanadashboards API
@@ -117,6 +133,10 @@ func (d *GrafanaDashboard) Hash() string {
 		}
 	}
 
+	if d.Status.Content != "" {
+		io.WriteString(hash, d.Status.Content) //nolint
+	}
+
 	return fmt.Sprintf("%x", hash.Sum(nil))
 }
 

config/crd/bases/integreatly.org_grafanadashboards.yaml

@@ -53,6 +53,14 @@ spec:
                 required:
                 - key
                 type: object
+              contentCacheDuration:
+                description: ContentCacheDuration sets how often the operator should
+                  resync with the external source when using the `grafanaCom.id` or
+                  `url` field to specify the source of the dashboard. The default
+                  value is decided by the `dashboardContentCacheDuration` field in
+                  the `Grafana` resource. The default is 0 which is interpreted as
+                  never refetching.
+                type: string
               customFolderName:
                 type: string
               datasources:
@@ -97,6 +105,26 @@ spec:
                 type: string
             type: object
           status:
+            properties:
+              content:
+                type: string
+              contentTimestamp:
+                format: date-time
+                type: string
+              contentUrl:
+                type: string
+              error:
+                properties:
+                  code:
+                    type: integer
+                  error:
+                    type: string
+                  retries:
+                    type: integer
+                required:
+                - code
+                - error
+                type: object
             type: object
         type: object
     served: true

config/crd/bases/integreatly.org_grafanas.yaml

@@ -1839,6 +1839,11 @@ spec:
                   - name
                   type: object
                 type: array
+              dashboardContentCacheDuration:
+                description: DashboardContentCacheDuration sets a default for when
+                  a `GrafanaDashboard` resource doesn't specify a `contentCacheDuration`.
+                  If left unset or 0 the default behavior is to cache indefinitely.
+                type: string
               dashboardLabelSelector:
                 items:
                   description: A label selector is a label query over a set of resources.

controllers/common/controllerState.go

@@ -5,9 +5,10 @@ import v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 var ControllerEvents = make(chan ControllerState, 1)
 
 type ControllerState struct {
-	DashboardSelectors         []*v1.LabelSelector
-	DashboardNamespaceSelector *v1.LabelSelector
-	AdminUrl                   string
-	GrafanaReady               bool
-	ClientTimeout              int
+	DashboardSelectors            []*v1.LabelSelector
+	DashboardNamespaceSelector    *v1.LabelSelector
+	DashboardContentCacheDuration *v1.Duration
+	AdminUrl                      string
+	GrafanaReady                  bool
+	ClientTimeout                 int
 }

controllers/grafana/grafana_controller.go

@@ -16,6 +16,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	netv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/tools/record"
@@ -299,6 +300,10 @@ func (r *ReconcileGrafana) manageSuccess(cr *grafanav1alpha1.Grafana, state *com
 		cr.Status.InstalledDashboards = r.Config.GetDashboards("")
 	}
 
+	if cr.Spec.DashboardContentCacheDuration == nil {
+		cr.Spec.DashboardContentCacheDuration = &metav1.Duration{Duration: 0}
+	}
+
 	instance := &grafanav1alpha1.Grafana{}
 	err := r.Client.Get(r.Context, request.NamespacedName, instance)
 	if err != nil {
@@ -319,11 +324,12 @@ func (r *ReconcileGrafana) manageSuccess(cr *grafanav1alpha1.Grafana, state *com
 
 	// Publish controller state
 	controllerState := common.ControllerState{
-		DashboardSelectors:         cr.Spec.DashboardLabelSelector,
-		DashboardNamespaceSelector: cr.Spec.DashboardNamespaceSelector,
-		AdminUrl:                   url,
-		GrafanaReady:               true,
-		ClientTimeout:              DefaultClientTimeoutSeconds,
+		DashboardSelectors:            cr.Spec.DashboardLabelSelector,
+		DashboardNamespaceSelector:    cr.Spec.DashboardNamespaceSelector,
+		DashboardContentCacheDuration: cr.Spec.DashboardContentCacheDuration,
+		AdminUrl:                      url,
+		GrafanaReady:                  true,
+		ClientTimeout:                 DefaultClientTimeoutSeconds,
 	}
 
 	if cr.Spec.Client != nil && cr.Spec.Client.TimeoutSeconds != nil {