diff --git a/api/integreatly/v1alpha1/grafanadashboard_types.go b/api/integreatly/v1alpha1/grafanadashboard_types.go
index 03c190a35..1db66d582 100644
--- a/api/integreatly/v1alpha1/grafanadashboard_types.go
+++ b/api/integreatly/v1alpha1/grafanadashboard_types.go
@@ -32,15 +32,17 @@ import (
// GrafanaDashboardSpec defines the desired state of GrafanaDashboard
type GrafanaDashboardSpec struct {
- Json string `json:"json,omitempty"`
- Jsonnet string `json:"jsonnet,omitempty"`
- Plugins PluginList `json:"plugins,omitempty"`
- Url string `json:"url,omitempty"`
- ConfigMapRef *corev1.ConfigMapKeySelector `json:"configMapRef,omitempty"`
- Datasources []GrafanaDashboardDatasource `json:"datasources,omitempty"`
- CustomFolderName string `json:"customFolderName,omitempty"`
- GrafanaCom *GrafanaDashboardGrafanaComSource `json:"grafanaCom,omitempty"`
+ Json string `json:"json,omitempty"`
+ Jsonnet string `json:"jsonnet,omitempty"`
+ Plugins PluginList `json:"plugins,omitempty"`
+ Url string `json:"url,omitempty"`
+ ConfigMapRef *corev1.ConfigMapKeySelector `json:"configMapRef,omitempty"`
+ Datasources []GrafanaDashboardDatasource `json:"datasources,omitempty"`
+ CustomFolderName string `json:"customFolderName,omitempty"`
+ GrafanaCom *GrafanaDashboardGrafanaComSource `json:"grafanaCom,omitempty"`
+ ContentCacheDuration *metav1.Duration `json:"contentCacheDuration,omitempty"`
}
+
type GrafanaDashboardDatasource struct {
InputName string `json:"inputName"`
DatasourceName string `json:"datasourceName"`
@@ -63,7 +65,20 @@ type GrafanaDashboardRef struct {
}
type GrafanaDashboardStatus struct {
- // Empty
+ // +optional
+ Content string `json:"content"`
+ // +optional
+ ContentTimestamp *metav1.Time `json:"contentTimestamp"`
+ // +optional
+ ContentUrl string `json:"contentUrl"`
+ // +optional
+ Error *GrafanaDashboardError `json:"error"`
+}
+
+type GrafanaDashboardError struct {
+ Code int `json:"code"`
+ Message string `json:"error"`
+ Retries int `json:"retries,omitempty"`
}
// GrafanaDashboard is the Schema for the grafanadashboards API
diff --git a/api/integreatly/v1alpha1/zz_generated.deepcopy.go b/api/integreatly/v1alpha1/zz_generated.deepcopy.go
index 10b78df33..d9f84d859 100644
--- a/api/integreatly/v1alpha1/zz_generated.deepcopy.go
+++ b/api/integreatly/v1alpha1/zz_generated.deepcopy.go
@@ -1315,7 +1315,7 @@ func (in *GrafanaDashboard) DeepCopyInto(out *GrafanaDashboard) {
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Spec.DeepCopyInto(&out.Spec)
- out.Status = in.Status
+ in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GrafanaDashboard.
@@ -1351,6 +1351,21 @@ func (in *GrafanaDashboardDatasource) DeepCopy() *GrafanaDashboardDatasource {
return out
}
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *GrafanaDashboardError) DeepCopyInto(out *GrafanaDashboardError) {
+ *out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GrafanaDashboardError.
+func (in *GrafanaDashboardError) DeepCopy() *GrafanaDashboardError {
+ if in == nil {
+ return nil
+ }
+ out := new(GrafanaDashboardError)
+ in.DeepCopyInto(out)
+ return out
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GrafanaDashboardGrafanaComSource) DeepCopyInto(out *GrafanaDashboardGrafanaComSource) {
*out = *in
@@ -1446,6 +1461,11 @@ func (in *GrafanaDashboardSpec) DeepCopyInto(out *GrafanaDashboardSpec) {
*out = new(GrafanaDashboardGrafanaComSource)
(*in).DeepCopyInto(*out)
}
+ if in.ContentCacheDuration != nil {
+ in, out := &in.ContentCacheDuration, &out.ContentCacheDuration
+ *out = new(metav1.Duration)
+ **out = **in
+ }
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GrafanaDashboardSpec.
@@ -1461,6 +1481,15 @@ func (in *GrafanaDashboardSpec) DeepCopy() *GrafanaDashboardSpec {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GrafanaDashboardStatus) DeepCopyInto(out *GrafanaDashboardStatus) {
*out = *in
+ if in.ContentTimestamp != nil {
+ in, out := &in.ContentTimestamp, &out.ContentTimestamp
+ *out = (*in).DeepCopy()
+ }
+ if in.Error != nil {
+ in, out := &in.Error, &out.Error
+ *out = new(GrafanaDashboardError)
+ **out = **in
+ }
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GrafanaDashboardStatus.
diff --git a/config/crd/bases/integreatly.org_grafanadashboards.yaml b/config/crd/bases/integreatly.org_grafanadashboards.yaml
index a40764f87..b20a3586d 100644
--- a/config/crd/bases/integreatly.org_grafanadashboards.yaml
+++ b/config/crd/bases/integreatly.org_grafanadashboards.yaml
@@ -53,6 +53,8 @@ spec:
required:
- key
type: object
+ contentCacheDuration:
+ type: string
customFolderName:
type: string
datasources:
@@ -97,6 +99,26 @@ spec:
type: string
type: object
status:
+ properties:
+ content:
+ type: string
+ contentTimestamp:
+ format: date-time
+ type: string
+ contentUrl:
+ type: string
+ error:
+ properties:
+ code:
+ type: integer
+ error:
+ type: string
+ retries:
+ type: integer
+ required:
+ - code
+ - error
+ type: object
type: object
type: object
served: true
diff --git a/controllers/grafanadashboard/dashboard_pipeline.go b/controllers/grafanadashboard/dashboard_pipeline.go
index b484feb5e..bbdf4ba13 100644
--- a/controllers/grafanadashboard/dashboard_pipeline.go
+++ b/controllers/grafanadashboard/dashboard_pipeline.go
@@ -13,12 +13,14 @@ import (
"path"
"strconv"
"strings"
+ "time"
"github.com/go-logr/logr"
"github.com/google/go-jsonnet"
"github.com/grafana-operator/grafana-operator/v4/api/integreatly/v1alpha1"
"github.com/grafana-operator/grafana-operator/v4/controllers/config"
corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
)
@@ -51,6 +53,9 @@ type DashboardPipelineImpl struct {
}
func NewDashboardPipeline(client client.Client, dashboard *v1alpha1.GrafanaDashboard, ctx context.Context) DashboardPipeline {
+ if dashboard.Spec.ContentCacheDuration == nil {
+ dashboard.Spec.ContentCacheDuration = &metav1.Duration{Duration: 24 * time.Hour}
+ }
return &DashboardPipelineImpl{
Client: client,
Dashboard: dashboard,
@@ -124,6 +129,11 @@ func (r *DashboardPipelineImpl) obtainJson() error {
return errors.New("both dashboard url and grafana.com source specified")
}
+ if r.Dashboard.Status.Content != "" && r.Dashboard.Status.ContentTimestamp.Add(r.Dashboard.Spec.ContentCacheDuration.Duration).After(time.Now()) {
+ r.JSON = r.Dashboard.Status.Content
+ return nil
+ }
+
if r.Dashboard.Spec.GrafanaCom != nil {
if err := r.loadDashboardFromGrafanaCom(); err != nil {
r.Logger.Error(err, "failed to request dashboard from grafana.com, falling back to config map; if specified")
@@ -196,14 +206,30 @@ func (r *DashboardPipelineImpl) loadDashboardFromURL() error {
}
defer resp.Body.Close()
- if resp.StatusCode != 200 {
- return fmt.Errorf("request failed with status %v", resp.StatusCode)
- }
-
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
+ if resp.StatusCode != 200 {
+ retries := 0
+ if r.Dashboard.Status.Error != nil {
+ retries = r.Dashboard.Status.Error.Retries
+ }
+ r.Dashboard.Status = v1alpha1.GrafanaDashboardStatus{
+ Error: &v1alpha1.GrafanaDashboardError{
+ Message: string(body),
+ Code: resp.StatusCode,
+ Retries: retries + 1,
+ },
+ ContentTimestamp: &metav1.Time{Time: time.Now()},
+ }
+
+ if err := r.Client.Status().Update(r.Context, r.Dashboard); err != nil {
+ return fmt.Errorf("failed to request dashboard and failed to update status %s: %w", string(body), err)
+ }
+
+ return fmt.Errorf("request failed with status %v", resp.StatusCode)
+ }
sourceType := r.getFileType(url.Path)
switch sourceType {
@@ -218,13 +244,14 @@ func (r *DashboardPipelineImpl) loadDashboardFromURL() error {
r.JSON = json
}
- // Update dashboard spec so that URL would not be refetched
- if r.JSON != r.Dashboard.Spec.Json {
- r.Dashboard.Spec.Json = r.JSON
- err := r.Client.Update(r.Context, r.Dashboard)
- if err != nil {
- return err
- }
+ r.Dashboard.Status = v1alpha1.GrafanaDashboardStatus{
+ Content: r.JSON,
+ ContentTimestamp: &metav1.Time{Time: time.Now()},
+ ContentUrl: r.Dashboard.Spec.Url,
+ }
+
+ if err := r.Client.Status().Update(r.Context, r.Dashboard); err != nil {
+ return fmt.Errorf("failed to update status with content for dashboard %s/%s: %w", r.Dashboard.Namespace, r.Dashboard.Name, err)
}
return nil
@@ -246,14 +273,40 @@ func (r *DashboardPipelineImpl) loadDashboardFromGrafanaCom() error {
if err != nil {
return err
}
+
+ if resp.StatusCode != 200 {
+ retries := 0
+ if r.Dashboard.Status.Error != nil {
+ retries = r.Dashboard.Status.Error.Retries
+ }
+ r.Dashboard.Status = v1alpha1.GrafanaDashboardStatus{
+ Error: &v1alpha1.GrafanaDashboardError{
+ Message: string(body),
+ Code: resp.StatusCode,
+ Retries: retries + 1,
+ },
+ ContentTimestamp: &metav1.Time{Time: time.Now()},
+ }
+
+ if err := r.Client.Status().Update(r.Context, r.Dashboard); err != nil {
+ return fmt.Errorf("failed to request dashboard and failed to update status %s: %w", string(body), err)
+ }
+
+ return fmt.Errorf("failed to request dashboard: %s", string(body))
+ }
+
r.JSON = string(body)
// Update JSON so dashboard is not refetched
- if r.JSON != r.Dashboard.Spec.Json {
- r.Dashboard.Spec.Json = r.JSON
- if err := r.Client.Update(r.Context, r.Dashboard); err != nil {
- return err
- }
+
+ r.Dashboard.Status = v1alpha1.GrafanaDashboardStatus{
+ Content: r.JSON,
+ ContentTimestamp: &metav1.Time{Time: time.Now()},
+ ContentUrl: url,
+ }
+
+ if err := r.Client.Status().Update(r.Context, r.Dashboard); err != nil {
+ return fmt.Errorf("failed to update status with content for dashboard %s/%s: %w", r.Dashboard.Namespace, r.Dashboard.Name, err)
}
return nil
diff --git a/controllers/grafanadashboard/grafanadashboard_controller.go b/controllers/grafanadashboard/grafanadashboard_controller.go
index 9c01f1228..cf7d1764c 100644
--- a/controllers/grafanadashboard/grafanadashboard_controller.go
+++ b/controllers/grafanadashboard/grafanadashboard_controller.go
@@ -21,6 +21,7 @@ import (
"crypto/tls"
"errors"
"fmt"
+ "math"
"net/http"
"os"
@@ -276,6 +277,15 @@ func (r *GrafanaDashboardReconciler) reconcileDashboards(request reconcile.Reque
folderName = dashboard.Spec.CustomFolderName
}
+ if dashboard.Status.Error != nil && dashboard.Status.Error.Code == 429 {
+ backoffDuration := 30 * time.Second * time.Duration(math.Pow(2, float64(dashboard.Status.Error.Retries)))
+
+ if dashboard.Status.ContentTimestamp.Add(backoffDuration).After(time.Now()) {
+ log.Log.Info("still awaiting rate limit for dashboard", "folder", folderName, "dashboard", request.Name)
+ continue
+ }
+ }
+
folder, err := grafanaClient.CreateOrUpdateFolder(folderName)
if err != nil {
diff --git a/documentation/api.md b/documentation/api.md
index 6a5636ed5..a7546b49d 100644
--- a/documentation/api.md
+++ b/documentation/api.md
@@ -63,7 +63,7 @@ GrafanaDashboard is the Schema for the grafanadashboards API
| false |
- | status |
+ status |
object |
@@ -96,6 +96,13 @@ GrafanaDashboardSpec defines the desired state of GrafanaDashboard
Selects a key from a ConfigMap.
|
false |
+
+ | contentCacheDuration |
+ string |
+
+
+ |
+ false |
| customFolderName |
string |
@@ -291,6 +298,97 @@ GrafanaPlugin contains information about a single plugin
+
+### GrafanaDashboard.status
+[↩ Parent](#grafanadashboard)
+
+
+
+
+
+
+
+
+ | Name |
+ Type |
+ Description |
+ Required |
+
+
+
+ | content |
+ string |
+
+
+ |
+ false |
+
+ | contentTimestamp |
+ string |
+
+
+
+ Format: date-time
+ |
+ false |
+
+ | contentUrl |
+ string |
+
+
+ |
+ false |
+
+ | error |
+ object |
+
+
+ |
+ false |
+
+
+
+
+### GrafanaDashboard.status.error
+[↩ Parent](#grafanadashboardstatus)
+
+
+
+
+
+
+
+
+ | Name |
+ Type |
+ Description |
+ Required |
+
+
+
+ | code |
+ integer |
+
+
+ |
+ true |
+
+ | error |
+ string |
+
+
+ |
+ true |
+
+ | retries |
+ integer |
+
+
+ |
+ false |
+
+
+
## GrafanaDataSource
[↩ Parent](#integreatlyorgv1alpha1 )