Add documenting comments to new fields and make the default cache dur…

…ation configurable.

api/integreatly/v1alpha1/grafana_types.go

@@ -38,6 +38,10 @@ type GrafanaSpec struct {
 	InitImage                  string                   `json:"initImage,omitempty"`
 	LivenessProbeSpec          *LivenessProbeSpec       `json:"livenessProbeSpec,omitempty"`
 	ReadinessProbeSpec         *ReadinessProbeSpec      `json:"readinessProbeSpec,omitempty"`
+
+	// DashboardContentCacheDuration sets a default for when a `GrafanaDashboard` resource doesn't specify a `contentCacheDuration`.
+	// If left unset or 0 the default behaviour is to cache indefinitely.
+	DashboardContentCacheDuration *metav1.Duration `json:"dashboardContentCacheDuration,omitempty"`
 }
 
 type ReadinessProbeSpec struct {

api/integreatly/v1alpha1/grafanadashboard_types.go

@@ -32,15 +32,20 @@ import (
 
 // GrafanaDashboardSpec defines the desired state of GrafanaDashboard
 type GrafanaDashboardSpec struct {
-	Json                 string                            `json:"json,omitempty"`
-	Jsonnet              string                            `json:"jsonnet,omitempty"`
-	Plugins              PluginList                        `json:"plugins,omitempty"`
-	Url                  string                            `json:"url,omitempty"`
-	ConfigMapRef         *corev1.ConfigMapKeySelector      `json:"configMapRef,omitempty"`
-	Datasources          []GrafanaDashboardDatasource      `json:"datasources,omitempty"`
-	CustomFolderName     string                            `json:"customFolderName,omitempty"`
-	GrafanaCom           *GrafanaDashboardGrafanaComSource `json:"grafanaCom,omitempty"`
-	ContentCacheDuration *metav1.Duration                  `json:"contentCacheDuration,omitempty"`
+	Json             string                            `json:"json,omitempty"`
+	Jsonnet          string                            `json:"jsonnet,omitempty"`
+	Plugins          PluginList                        `json:"plugins,omitempty"`
+	Url              string                            `json:"url,omitempty"`
+	ConfigMapRef     *corev1.ConfigMapKeySelector      `json:"configMapRef,omitempty"`
+	Datasources      []GrafanaDashboardDatasource      `json:"datasources,omitempty"`
+	CustomFolderName string                            `json:"customFolderName,omitempty"`
+	GrafanaCom       *GrafanaDashboardGrafanaComSource `json:"grafanaCom,omitempty"`
+
+	// ContentCacheDuration sets how often the operator should resync with the external source when using
+	// the `grafanaCom.id` or `url` field to specify the source of the dashboard. The default value is
+	// decided by the `dashboardContentCacheDuration` field in the `Grafana` resource. The default is 0 which
+	// is interpreted as never refetching.
+	ContentCacheDuration *metav1.Duration `json:"contentCacheDuration,omitempty"`
 }
 
 type GrafanaDashboardDatasource struct {
@@ -65,14 +70,10 @@ type GrafanaDashboardRef struct {
 }
 
 type GrafanaDashboardStatus struct {
-	// +optional
-	Content string `json:"content"`
-	// +optional
-	ContentTimestamp *metav1.Time `json:"contentTimestamp"`
-	// +optional
-	ContentUrl string `json:"contentUrl"`
-	// +optional
-	Error *GrafanaDashboardError `json:"error"`
+	Content          string                 `json:"content,omitempty"`
+	ContentTimestamp *metav1.Time           `json:"contentTimestamp,omitempty"`
+	ContentUrl       string                 `json:"contentUrl,omitempty"`
+	Error            *GrafanaDashboardError `json:"error,omitempty"`
 }
 
 type GrafanaDashboardError struct {
@@ -132,6 +133,10 @@ func (d *GrafanaDashboard) Hash() string {
 		}
 	}
 
+	if d.Status.Content != "" {
+		io.WriteString(hash, d.Status.Content)
+	}
+
 	return fmt.Sprintf("%x", hash.Sum(nil))
 }
 

config/crd/bases/integreatly.org_grafanadashboards.yaml

@@ -54,6 +54,12 @@ spec:
                 - key
                 type: object
               contentCacheDuration:
+                description: ContentCacheDuration sets how often the operator should
+                  resync with the external source when using the `grafanaCom.id` or
+                  `url` field to specify the source of the dashboard. The default
+                  value is decided by the `dashboardContentCacheDuration` field in
+                  the `Grafana` resource. The default is 0 which is interpreted as
+                  never refetching.
                 type: string
               customFolderName:
                 type: string

config/crd/bases/integreatly.org_grafanas.yaml

@@ -1833,6 +1833,11 @@ spec:
                   - name
                   type: object
                 type: array
+              dashboardContentCacheDuration:
+                description: DashboardContentCacheDuration sets a default for when
+                  a `GrafanaDashboard` resource doesn't specify a `contentCacheDuration`.
+                  If left unset or 0 the default behaviour is to cache indefinitely.
+                type: string
               dashboardLabelSelector:
                 items:
                   description: A label selector is a label query over a set of resources.

controllers/common/controllerState.go

@@ -5,9 +5,10 @@ import v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 var ControllerEvents = make(chan ControllerState, 1)
 
 type ControllerState struct {
-	DashboardSelectors         []*v1.LabelSelector
-	DashboardNamespaceSelector *v1.LabelSelector
-	AdminUrl                   string
-	GrafanaReady               bool
-	ClientTimeout              int
+	DashboardSelectors            []*v1.LabelSelector
+	DashboardNamespaceSelector    *v1.LabelSelector
+	DashboardContentCacheDuration *v1.Duration
+	AdminUrl                      string
+	GrafanaReady                  bool
+	ClientTimeout                 int
 }

controllers/grafana/grafana_controller.go

@@ -16,6 +16,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	netv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/tools/record"
@@ -286,6 +287,10 @@ func (r *ReconcileGrafana) manageSuccess(cr *grafanav1alpha1.Grafana, state *com
 		cr.Status.InstalledDashboards = r.Config.GetDashboards("")
 	}
 
+	if cr.Spec.DashboardContentCacheDuration == nil {
+		cr.Spec.DashboardContentCacheDuration = &metav1.Duration{Duration: 0}
+	}
+
 	instance := &grafanav1alpha1.Grafana{}
 	err := r.Client.Get(r.Context, request.NamespacedName, instance)
 	if err != nil {
@@ -306,11 +311,12 @@ func (r *ReconcileGrafana) manageSuccess(cr *grafanav1alpha1.Grafana, state *com
 
 	// Publish controller state
 	controllerState := common.ControllerState{
-		DashboardSelectors:         cr.Spec.DashboardLabelSelector,
-		DashboardNamespaceSelector: cr.Spec.DashboardNamespaceSelector,
-		AdminUrl:                   url,
-		GrafanaReady:               true,
-		ClientTimeout:              DefaultClientTimeoutSeconds,
+		DashboardSelectors:            cr.Spec.DashboardLabelSelector,
+		DashboardNamespaceSelector:    cr.Spec.DashboardNamespaceSelector,
+		DashboardContentCacheDuration: cr.Spec.DashboardContentCacheDuration,
+		AdminUrl:                      url,
+		GrafanaReady:                  true,
+		ClientTimeout:                 DefaultClientTimeoutSeconds,
 	}
 
 	if cr.Spec.Client != nil && cr.Spec.Client.TimeoutSeconds != nil {

controllers/grafanadashboard/dashboard_pipeline.go

@@ -21,6 +21,7 @@ import (
 	"github.com/grafana-operator/grafana-operator/v4/controllers/config"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
@@ -53,9 +54,6 @@ type DashboardPipelineImpl struct {
 }
 
 func NewDashboardPipeline(client client.Client, dashboard *v1alpha1.GrafanaDashboard, ctx context.Context) DashboardPipeline {
-	if dashboard.Spec.ContentCacheDuration == nil {
-		dashboard.Spec.ContentCacheDuration = &metav1.Duration{Duration: 24 * time.Hour}
-	}
 	return &DashboardPipelineImpl{
 		Client:    client,
 		Dashboard: dashboard,
@@ -117,6 +115,13 @@ func (r *DashboardPipelineImpl) validateJson() error {
 	return err
 }
 
+func (r *DashboardPipelineImpl) shouldUseContentCache() bool {
+	cacheDuration := r.Dashboard.Spec.ContentCacheDuration.Duration
+	contentTimeStamp := r.Dashboard.Status.ContentTimestamp
+
+	return cacheDuration > 0 && contentTimeStamp.Add(cacheDuration).After(time.Now())
+}
+
 // Try to get the dashboard json definition either from a provided URL or from the
 // raw json in the dashboard resource. The priority is as follows:
 // 1) try to fetch from url or grafanaCom if provided
@@ -129,7 +134,7 @@ func (r *DashboardPipelineImpl) obtainJson() error {
 		return errors.New("both dashboard url and grafana.com source specified")
 	}
 
-	if r.Dashboard.Status.Content != "" && r.Dashboard.Status.ContentTimestamp.Add(r.Dashboard.Spec.ContentCacheDuration.Duration).After(time.Now()) {
+	if r.Dashboard.Status.Content != "" && r.shouldUseContentCache() {
 		r.JSON = r.Dashboard.Status.Content
 		return nil
 	}
@@ -212,6 +217,7 @@ func (r *DashboardPipelineImpl) loadDashboardFromURL() error {
 	}
 	if resp.StatusCode != 200 {
 		retries := 0
+		r.refreshDashboard()
 		if r.Dashboard.Status.Error != nil {
 			retries = r.Dashboard.Status.Error.Retries
 		}
@@ -225,7 +231,7 @@ func (r *DashboardPipelineImpl) loadDashboardFromURL() error {
 		}
 
 		if err := r.Client.Status().Update(r.Context, r.Dashboard); err != nil {
-			return fmt.Errorf("failed to request dashboard and failed to update status %s: %w", string(body), err)
+			return fmt.Errorf("failed to request dashboard: %s\nfailed to update status : %w", string(body), err)
 		}
 
 		return fmt.Errorf("request failed with status %v", resp.StatusCode)
@@ -244,6 +250,7 @@ func (r *DashboardPipelineImpl) loadDashboardFromURL() error {
 		r.JSON = json
 	}
 
+	r.refreshDashboard()
 	r.Dashboard.Status = v1alpha1.GrafanaDashboardStatus{
 		Content:          r.JSON,
 		ContentTimestamp: &metav1.Time{Time: time.Now()},
@@ -257,6 +264,10 @@ func (r *DashboardPipelineImpl) loadDashboardFromURL() error {
 	return nil
 }
 
+func (r *DashboardPipelineImpl) refreshDashboard() error {
+	return r.Client.Get(r.Context, types.NamespacedName{Name: r.Dashboard.Name, Namespace: r.Dashboard.Namespace}, r.Dashboard)
+}
+
 func (r *DashboardPipelineImpl) loadDashboardFromGrafanaCom() error {
 	url, err := r.getGrafanaComDashboardUrl()
 	if err != nil {
@@ -275,6 +286,7 @@ func (r *DashboardPipelineImpl) loadDashboardFromGrafanaCom() error {
 	}
 
 	if resp.StatusCode != 200 {
+		r.refreshDashboard()
 		retries := 0
 		if r.Dashboard.Status.Error != nil {
 			retries = r.Dashboard.Status.Error.Retries
@@ -297,8 +309,7 @@ func (r *DashboardPipelineImpl) loadDashboardFromGrafanaCom() error {
 
 	r.JSON = string(body)
 
-	// Update JSON so dashboard is not refetched
-
+	r.refreshDashboard()
 	r.Dashboard.Status = v1alpha1.GrafanaDashboardStatus{
 		Content:          r.JSON,
 		ContentTimestamp: &metav1.Time{Time: time.Now()},

controllers/grafanadashboard/grafanadashboard_controller.go

@@ -264,9 +264,9 @@ func (r *GrafanaDashboardReconciler) reconcileDashboards(request reconcile.Reque
 	}
 
 	// Process new/updated dashboards
-	for i, dashboard := range namespaceDashboards.Items {
+	for _, dashboard := range namespaceDashboards.Items {
 		// Is this a dashboard we care about (matches the label selectors)?
-		if !r.isMatch(&namespaceDashboards.Items[i]) {
+		if !r.isMatch(&dashboard) {
 			log.Log.Info("dashboard found but selectors do not match",
 				"namespace", dashboard.Namespace, "name", dashboard.Name)
 			continue
@@ -277,11 +277,12 @@ func (r *GrafanaDashboardReconciler) reconcileDashboards(request reconcile.Reque
 			folderName = dashboard.Spec.CustomFolderName
 		}
 
-		if dashboard.Status.Error != nil && dashboard.Status.Error.Code == 429 {
+		if dashboard.Status.Error != nil {
 			backoffDuration := 30 * time.Second * time.Duration(math.Pow(2, float64(dashboard.Status.Error.Retries)))
+			retryTime := dashboard.Status.ContentTimestamp.Add(backoffDuration)
 
-			if dashboard.Status.ContentTimestamp.Add(backoffDuration).After(time.Now()) {
-				log.Log.Info("still awaiting rate limit for dashboard", "folder", folderName, "dashboard", request.Name)
+			if retryTime.After(time.Now()) {
+				log.Log.Info("delaying retry of failing dashboard", "folder", folderName, "dashboard", request.Name, "namespace", request.Namespace, "retryTime", retryTime)
 				continue
 			}
 		}
@@ -290,7 +291,7 @@ func (r *GrafanaDashboardReconciler) reconcileDashboards(request reconcile.Reque
 
 		if err != nil {
 			log.Log.Error(err, "failed to get or create namespace folder for dashboard", "folder", folderName, "dashboard", request.Name)
-			r.manageError(&namespaceDashboards.Items[i], err)
+			r.manageError(&dashboard, err)
 			continue
 		}
 
@@ -301,11 +302,15 @@ func (r *GrafanaDashboardReconciler) reconcileDashboards(request reconcile.Reque
 			folderId = *folder.ID
 		}
 
+		if dashboard.Spec.ContentCacheDuration == nil {
+			dashboard.Spec.ContentCacheDuration = r.state.DashboardContentCacheDuration
+		}
+
 		// Process the dashboard. Use the known hash of an existing dashboard
 		// to determine if an update is required
-		knownHash := findHash(knownDashboards, &namespaceDashboards.Items[i])
-		knownUid := findUid(knownDashboards, &namespaceDashboards.Items[i])
-		pipeline := NewDashboardPipeline(r.Client, &namespaceDashboards.Items[i], r.context)
+		knownHash := findHash(knownDashboards, &dashboard)
+		knownUid := findUid(knownDashboards, &dashboard)
+		pipeline := NewDashboardPipeline(r.Client, &dashboard, r.context)
 		processed, err := pipeline.ProcessDashboard(knownHash, &folderId, folderName, false)
 
 		// Check known dashboards exist on grafana instance and recreate if not
@@ -316,32 +321,32 @@ func (r *GrafanaDashboardReconciler) reconcileDashboards(request reconcile.Reque
 			}
 
 			if *response.Dashboard.ID == uint(0) {
-				log.Log.Info(fmt.Sprintf("Dashboard %v has been deleted via grafana console. Recreating.", namespaceDashboards.Items[i].ObjectMeta.Name))
+				log.Log.Info(fmt.Sprintf("Dashboard %v has been deleted via grafana console. Recreating.", dashboard.ObjectMeta.Name))
 				processed, err = pipeline.ProcessDashboard(knownHash, &folderId, folderName, true)
 
 				if err != nil {
 					log.Log.Error(err, "cannot process dashboard", "namespace", dashboard.Namespace, "name", dashboard.Name)
-					r.manageError(&namespaceDashboards.Items[i], err)
+					r.manageError(&dashboard, err)
 					continue
 				}
 			}
 		}
 
 		if err != nil {
-			log.Log.Error(err, "cannot process dashboard", "namespace", dashboard.Namespace, "name", dashboard.Name)
-			r.manageError(&namespaceDashboards.Items[i], err)
+			// log.Log.Error(err, "cannot process dashboard", "namespace", dashboard.Namespace, "name", dashboard.Name)
+			r.manageError(&dashboard, err)
 			continue
 		}
 
 		if processed == nil {
-			r.config.SetPluginsFor(&namespaceDashboards.Items[i])
+			r.config.SetPluginsFor(&dashboard)
 			continue
 		}
 		// Check labels only when DashboardNamespaceSelector isnt empty
 		if r.state.DashboardNamespaceSelector != nil {
-			matchesNamespaceLabels, err := r.checkNamespaceLabels(&namespaceDashboards.Items[i])
+			matchesNamespaceLabels, err := r.checkNamespaceLabels(&dashboard)
 			if err != nil {
-				r.manageError(&namespaceDashboards.Items[i], err)
+				r.manageError(&dashboard, err)
 				continue
 			}
 
@@ -354,12 +359,12 @@ func (r *GrafanaDashboardReconciler) reconcileDashboards(request reconcile.Reque
 		_, err = grafanaClient.CreateOrUpdateDashboard(processed, folderId, folderName)
 		if err != nil {
 			//log.Log.Error(err, "cannot submit dashboard %v/%v", "namespace", dashboard.Namespace, "name", dashboard.Name)
-			r.manageError(&namespaceDashboards.Items[i], err)
+			r.manageError(&dashboard, err)
 
 			continue
 		}
 
-		r.manageSuccess(&namespaceDashboards.Items[i], &folderId, folderName)
+		r.manageSuccess(&dashboard, &folderId, folderName)
 	}
 
 	for _, dashboard := range dashboardsToDelete {
@@ -474,7 +479,7 @@ func (r *GrafanaDashboardReconciler) manageError(dashboard *grafanav1alpha1.Graf
 	if k8serrors.IsConflict(issue) || k8serrors.IsServiceUnavailable(issue) {
 		return
 	}
-	log.Log.Error(issue, "error updating dashboard")
+	log.Log.Error(issue, "error updating dashboard", "name", dashboard.Name, "namespace", dashboard.Namespace)
 }
 
 func (r *GrafanaDashboardReconciler) SetupWithManager(mgr manager.Manager) error {

deploy/examples/dashboards/DashboardFromURLWithCacheDuration.yaml

@@ -0,0 +1,10 @@
+apiVersion: integreatly.org/v1alpha1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-from-url
+  labels:
+    app: grafana
+spec:
+  url: https://raw.githubusercontent.com/grafana-operator/grafana-operator/master/deploy/examples/remote/grafana-dashboard.json
+  # Only cache this for 15 minutes before checking if the upstream source has changed.
+  contentCacheDuration: 15m