From a694bdf59c6476036765ea42006944c9780bcd56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20M=C3=A5rtensson?= Date: Fri, 22 Jul 2022 15:17:47 +0200 Subject: [PATCH] Gzip content cache and bugfix cache time calculation (#790) * Gzip content cache and bugfix cache time ecalculation * Deduplicate remote dashboard fetching logic --- .../v1alpha1/grafanadashboard_types.go | 17 ++- .../v1alpha1/zz_generated.deepcopy.go | 5 + .../integreatly.org_grafanadashboards.yaml | 3 +- .../grafanadashboard/dashboard_pipeline.go | 137 ++++++++---------- .../grafanadashboard_controller.go | 4 +- deploy/manifests/latest/crds.yaml | 3 +- documentation/api.md | 4 +- documentation/dashboards.md | 2 +- 8 files changed, 87 insertions(+), 88 deletions(-) diff --git a/api/integreatly/v1alpha1/grafanadashboard_types.go b/api/integreatly/v1alpha1/grafanadashboard_types.go index 2874ec571..cb990887c 100644 --- a/api/integreatly/v1alpha1/grafanadashboard_types.go +++ b/api/integreatly/v1alpha1/grafanadashboard_types.go @@ -81,7 +81,7 @@ type GrafanaDashboardRef struct { } type GrafanaDashboardStatus struct { - Content string `json:"content,omitempty"` + ContentCache []byte `json:"contentCache,omitempty"` ContentTimestamp *metav1.Time `json:"contentTimestamp,omitempty"` ContentUrl string `json:"contentUrl,omitempty"` Error *GrafanaDashboardError `json:"error,omitempty"` @@ -149,8 +149,8 @@ func (d *GrafanaDashboard) Hash() string { } } - if d.Status.Content != "" { - io.WriteString(hash, d.Status.Content) //nolint + if d.Status.ContentCache != nil { + hash.Write(d.Status.ContentCache) } return fmt.Sprintf("%x", hash.Sum(nil)) @@ -198,3 +198,14 @@ func Gunzip(compressed []byte) ([]byte, error) { } return ioutil.ReadAll(decoder) } + +func Gzip(content string) ([]byte, error) { + buf := bytes.NewBuffer([]byte{}) + writer := gzip.NewWriter(buf) + _, err := writer.Write([]byte(content)) + if err != nil { + return nil, err + } + writer.Close() + return ioutil.ReadAll(buf) +} diff --git a/api/integreatly/v1alpha1/zz_generated.deepcopy.go b/api/integreatly/v1alpha1/zz_generated.deepcopy.go index 752d14644..c82c442c0 100644 --- a/api/integreatly/v1alpha1/zz_generated.deepcopy.go +++ b/api/integreatly/v1alpha1/zz_generated.deepcopy.go @@ -1492,6 +1492,11 @@ func (in *GrafanaDashboardSpec) DeepCopy() *GrafanaDashboardSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GrafanaDashboardStatus) DeepCopyInto(out *GrafanaDashboardStatus) { *out = *in + if in.ContentCache != nil { + in, out := &in.ContentCache, &out.ContentCache + *out = make([]byte, len(*in)) + copy(*out, *in) + } if in.ContentTimestamp != nil { in, out := &in.ContentTimestamp, &out.ContentTimestamp *out = (*in).DeepCopy() diff --git a/config/crd/bases/integreatly.org_grafanadashboards.yaml b/config/crd/bases/integreatly.org_grafanadashboards.yaml index 24ae15c17..11c4c6450 100644 --- a/config/crd/bases/integreatly.org_grafanadashboards.yaml +++ b/config/crd/bases/integreatly.org_grafanadashboards.yaml @@ -131,7 +131,8 @@ spec: type: object status: properties: - content: + contentCache: + format: byte type: string contentTimestamp: format: date-time diff --git a/controllers/grafanadashboard/dashboard_pipeline.go b/controllers/grafanadashboard/dashboard_pipeline.go index 86b219d3e..6ce2a2dc6 100644 --- a/controllers/grafanadashboard/dashboard_pipeline.go +++ b/controllers/grafanadashboard/dashboard_pipeline.go @@ -20,6 +20,7 @@ import ( "github.com/grafana-operator/grafana-operator/v4/api/integreatly/v1alpha1" "github.com/grafana-operator/grafana-operator/v4/controllers/config" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -119,15 +120,20 @@ func (r *DashboardPipelineImpl) validateJson() error { } func (r *DashboardPipelineImpl) shouldUseContentCache() bool { - cacheDuration := r.Dashboard.Spec.ContentCacheDuration.Duration - contentTimeStamp := r.Dashboard.Status.ContentTimestamp + if r.Dashboard.Status.ContentCache != nil && r.Dashboard.Spec.ContentCacheDuration != nil && r.Dashboard.Status.ContentTimestamp != nil { + cacheDuration := r.Dashboard.Spec.ContentCacheDuration.Duration + contentTimeStamp := r.Dashboard.Status.ContentTimestamp - return cacheDuration > 0 && contentTimeStamp.Add(cacheDuration).After(time.Now()) + return cacheDuration <= 0 || contentTimeStamp.Add(cacheDuration).After(time.Now()) + } + return false } // Try to get the dashboard json definition either from a provided URL or from the // raw json in the dashboard resource. The priority is as follows: +// 0) try to use previously fetched content from url or grafanaCom if it is valid // 1) try to fetch from url or grafanaCom if provided +// 1.1) if downloaded content is identical to spec.json, clear spec.json to clean up from fetch behavior pre 4.5.0 // 2) url or grafanaCom fails or not provided: try to fetch from configmap ref // 3) no configmap specified: try to use embedded json // 4) no json specified: try to use embedded jsonnet @@ -137,23 +143,34 @@ func (r *DashboardPipelineImpl) obtainJson() error { return errors.New("both dashboard url and grafana.com source specified") } - if r.Dashboard.Status.Content != "" && r.shouldUseContentCache() { - r.JSON = r.Dashboard.Status.Content - return nil + var returnErr error + + if r.shouldUseContentCache() { + jsonBytes, err := v1alpha1.Gunzip(r.Dashboard.Status.ContentCache) + if err != nil { + returnErr = fmt.Errorf("failed to decode/decompress gzipped json: %w", err) + } else { + r.JSON = string(jsonBytes) + return nil + } } if r.Dashboard.Spec.GrafanaCom != nil { - if err := r.loadDashboardFromGrafanaCom(); err != nil { - r.Logger.Error(err, "failed to request dashboard from grafana.com, falling back to config map; if specified") + url, err := r.getGrafanaComDashboardUrl() + if err != nil { + return fmt.Errorf("failed to get grafana.com dashboard url: %w", err) + } + if err := r.loadDashboardFromURL(url); err != nil { + returnErr = fmt.Errorf("failed to request dashboard from grafana.com, falling back to url; if specified: %w", err) } else { return nil } } if r.Dashboard.Spec.Url != "" { - err := r.loadDashboardFromURL() + err := r.loadDashboardFromURL(r.Dashboard.Spec.Url) if err != nil { - r.Logger.Error(err, "failed to request dashboard url, falling back to config map; if specified") + returnErr = fmt.Errorf("failed to request dashboard url, falling back to config map; if specified: %w", err) } else { return nil } @@ -162,7 +179,7 @@ func (r *DashboardPipelineImpl) obtainJson() error { if r.Dashboard.Spec.ConfigMapRef != nil { err := r.loadDashboardFromConfigMap(r.Dashboard.Spec.ConfigMapRef, false) if err != nil { - r.Logger.Error(err, "failed to get config map, falling back to raw json") + returnErr = fmt.Errorf("failed to get config map, falling back to raw json: %w", err) } else { return nil } @@ -171,7 +188,7 @@ func (r *DashboardPipelineImpl) obtainJson() error { if r.Dashboard.Spec.GzipConfigMapRef != nil { err := r.loadDashboardFromConfigMap(r.Dashboard.Spec.GzipConfigMapRef, true) if err != nil { - r.Logger.Error(err, "failed to get config map, falling back to raw json") + returnErr = fmt.Errorf("failed to get config map, falling back to raw json: %w", err) } else { return nil } @@ -180,7 +197,7 @@ func (r *DashboardPipelineImpl) obtainJson() error { if r.Dashboard.Spec.GzipJson != nil { jsonBytes, err := v1alpha1.Gunzip(r.Dashboard.Spec.GzipJson) if err != nil { - r.Logger.Error(err, "failed to decode/decompress gzipped json") + returnErr = fmt.Errorf("failed to decode/decompress gzipped json: %w", err) } else { r.JSON = string(jsonBytes) return nil @@ -195,14 +212,14 @@ func (r *DashboardPipelineImpl) obtainJson() error { if r.Dashboard.Spec.Jsonnet != "" { json, err := r.loadJsonnet(r.Dashboard.Spec.Jsonnet) if err != nil { - r.Logger.Error(err, "failed to parse jsonnet") + returnErr = fmt.Errorf("failed to parse jsonnet: %w", err) } else { r.JSON = json return nil } } - return errors.New("unable to obtain dashboard contents") + return returnErr } // Compiles jsonnet to json and makes the grafonnet library available to @@ -221,15 +238,15 @@ func (r *DashboardPipelineImpl) loadJsonnet(source string) (string, error) { } // Try to obtain the dashboard json from a provided url -func (r *DashboardPipelineImpl) loadDashboardFromURL() error { - url, err := url.ParseRequestURI(r.Dashboard.Spec.Url) +func (r *DashboardPipelineImpl) loadDashboardFromURL(source string) error { + url, err := url.ParseRequestURI(source) if err != nil { - return fmt.Errorf("invalid url %v", r.Dashboard.Spec.Url) + return fmt.Errorf("invalid url %v", source) } - resp, err := http.Get(r.Dashboard.Spec.Url) + resp, err := http.Get(url.String()) if err != nil { - return fmt.Errorf("cannot request %v", r.Dashboard.Spec.Url) + return fmt.Errorf("cannot request %v", source) } defer resp.Body.Close() @@ -272,15 +289,32 @@ func (r *DashboardPipelineImpl) loadDashboardFromURL() error { r.JSON = json } - r.refreshDashboard() + if r.Dashboard.Spec.Json == r.JSON { + // Content downloaded to `json` field pre 4.5.0 can be removed since it is identical to the downloaded content. + r.refreshDashboard() + r.Dashboard.Spec.Json = "" + err = r.Client.Update(r.Context, r.Dashboard) + if err != nil { + return err + } + } + + content, err := v1alpha1.Gzip(r.JSON) + if err != nil { + return err + } + r.Dashboard.Status = v1alpha1.GrafanaDashboardStatus{ - Content: r.JSON, + ContentCache: content, ContentTimestamp: &metav1.Time{Time: time.Now()}, - ContentUrl: r.Dashboard.Spec.Url, + ContentUrl: source, } + r.refreshDashboard() if err := r.Client.Status().Update(r.Context, r.Dashboard); err != nil { - return fmt.Errorf("failed to update status with content for dashboard %s/%s: %w", r.Dashboard.Namespace, r.Dashboard.Name, err) + if !k8serrors.IsConflict(err) { + return fmt.Errorf("failed to update status with content for dashboard %s/%s: %w", r.Dashboard.Namespace, r.Dashboard.Name, err) + } } return nil @@ -293,61 +327,6 @@ func (r *DashboardPipelineImpl) refreshDashboard() { } } -func (r *DashboardPipelineImpl) loadDashboardFromGrafanaCom() error { - url, err := r.getGrafanaComDashboardUrl() - if err != nil { - return fmt.Errorf("failed to get grafana.com dashboard url: %w", err) - } - - resp, err := http.Get(url) // nolint:gosec - if err != nil { - return fmt.Errorf("failed to request dashboard url '%s': %w", url, err) - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - if resp.StatusCode != 200 { - r.refreshDashboard() - retries := 0 - if r.Dashboard.Status.Error != nil { - retries = r.Dashboard.Status.Error.Retries - } - r.Dashboard.Status = v1alpha1.GrafanaDashboardStatus{ - Error: &v1alpha1.GrafanaDashboardError{ - Message: string(body), - Code: resp.StatusCode, - Retries: retries + 1, - }, - ContentTimestamp: &metav1.Time{Time: time.Now()}, - } - - if err := r.Client.Status().Update(r.Context, r.Dashboard); err != nil { - return fmt.Errorf("failed to request dashboard and failed to update status %s: %w", string(body), err) - } - - return fmt.Errorf("failed to request dashboard: %s", string(body)) - } - - r.JSON = string(body) - - r.refreshDashboard() - r.Dashboard.Status = v1alpha1.GrafanaDashboardStatus{ - Content: r.JSON, - ContentTimestamp: &metav1.Time{Time: time.Now()}, - ContentUrl: url, - } - - if err := r.Client.Status().Update(r.Context, r.Dashboard); err != nil { - return fmt.Errorf("failed to update status with content for dashboard %s/%s: %w", r.Dashboard.Namespace, r.Dashboard.Name, err) - } - - return nil -} - func (r *DashboardPipelineImpl) getGrafanaComDashboardUrl() (string, error) { grafanaComSource := r.Dashboard.Spec.GrafanaCom var revision int diff --git a/controllers/grafanadashboard/grafanadashboard_controller.go b/controllers/grafanadashboard/grafanadashboard_controller.go index b45bcf76f..6e5a384b9 100644 --- a/controllers/grafanadashboard/grafanadashboard_controller.go +++ b/controllers/grafanadashboard/grafanadashboard_controller.go @@ -283,7 +283,7 @@ func (r *GrafanaDashboardReconciler) reconcileDashboards(request reconcile.Reque retryTime := dashboard.Status.ContentTimestamp.Add(backoffDuration) if retryTime.After(time.Now()) { - log.Log.Info("delaying retry of failing dashboard", "folder", folderName, "dashboard", request.Name, "namespace", request.Namespace, "retryTime", retryTime) + log.Log.V(1).Info("delaying retry of failing dashboard", "folder", folderName, "dashboard", dashboard.Name, "namespace", dashboard.Namespace, "retryTime", retryTime, "backoffDuration", backoffDuration) continue } } @@ -322,7 +322,7 @@ func (r *GrafanaDashboardReconciler) reconcileDashboards(request reconcile.Reque } if *response.Dashboard.ID == uint(0) { - log.Log.Info(fmt.Sprintf("Dashboard %v has been deleted via grafana console. Recreating.", dashboard.ObjectMeta.Name)) + log.Log.Info(fmt.Sprintf("Dashboard %v (%s) has been deleted via grafana console. Recreating.", dashboard.ObjectMeta.Name, knownUid)) processed, err = pipeline.ProcessDashboard(knownHash, &folderId, folderName, true) if err != nil { diff --git a/deploy/manifests/latest/crds.yaml b/deploy/manifests/latest/crds.yaml index 2f4e638c3..081fd89e7 100644 --- a/deploy/manifests/latest/crds.yaml +++ b/deploy/manifests/latest/crds.yaml @@ -129,7 +129,8 @@ spec: type: object status: properties: - content: + contentCache: + format: byte type: string contentTimestamp: format: date-time diff --git a/documentation/api.md b/documentation/api.md index 5862400cc..bb571ff97 100644 --- a/documentation/api.md +++ b/documentation/api.md @@ -373,10 +373,12 @@ GrafanaPlugin contains information about a single plugin - content + contentCache string
+
+ Format: byte
false diff --git a/documentation/dashboards.md b/documentation/dashboards.md index e2318aad1..70635380c 100644 --- a/documentation/dashboards.md +++ b/documentation/dashboards.md @@ -16,7 +16,7 @@ The following properties are accepted in the `spec`: * *jsonnet*: Jsonnet source. The [Grafonnet](https://grafana.github.io/grafonnet-lib/) library is made available automatically and can be imported. * *url*: Url address to download a json or jsonnet string with the dashboard contents. - * ***Warning***: If both url and json are specified then the json field will be updated with fetched. + * ***Warning***: If both url and json are specified then the json field will be cleared if the downloaded content is identical. * *The dashboard fetch priority by parameter is: url > configmap > json > jsonnet.* * *plugins*: A list of plugins required by the dashboard. They will be installed by the operator if not already present. * *datasources*: A list of datasources to be used as inputs. See [datasource inputs](#datasource-inputs).