From d92fb6aa22c02a609970f3e4496177599f911390 Mon Sep 17 00:00:00 2001 From: Andrew Melnick Date: Fri, 27 May 2022 15:43:09 -0600 Subject: [PATCH 1/8] Add ability to provide compressed JSON for large dashboards * Added fields gzipJson and gzipConfigMapRef to GrafanaDashboardSpec * Added unit tests for decoding/decompression * Added docs explaining how to compress dashboard data * Added example compressed dashboards to verify functionality --- .../v1alpha1/grafanadashboard_types.go | 40 ++++++- .../v1alpha1/grafanadashboard_types_test.go | 104 ++++++++++++++++++ .../v1alpha1/zz_generated.deepcopy.go | 6 + .../integreatly.org_grafanadashboards.yaml | 19 ++++ config/manager/kustomization.yaml | 2 +- .../grafanadashboard/dashboard_pipeline.go | 37 ++++++- .../dashboards/CompressedDashboard.yaml | 13 +++ .../DashboardFromCompressedConfigMap.yaml | 23 ++++ deploy/manifests/latest/crds.yaml | 19 ++++ deploy/manifests/latest/deployment.yaml | 2 +- documentation/api.md | 55 +++++++++ documentation/dashboards.md | 17 +++ 12 files changed, 330 insertions(+), 7 deletions(-) create mode 100644 api/integreatly/v1alpha1/grafanadashboard_types_test.go create mode 100644 deploy/examples/dashboards/CompressedDashboard.yaml create mode 100644 deploy/examples/dashboards/DashboardFromCompressedConfigMap.yaml diff --git a/api/integreatly/v1alpha1/grafanadashboard_types.go b/api/integreatly/v1alpha1/grafanadashboard_types.go index 03c190a35..2a5624f8c 100644 --- a/api/integreatly/v1alpha1/grafanadashboard_types.go +++ b/api/integreatly/v1alpha1/grafanadashboard_types.go @@ -17,11 +17,17 @@ limitations under the License. package v1alpha1 import ( + "bytes" "crypto/sha1" // nolint "crypto/sha256" + "encoding/base64" "encoding/json" "fmt" "io" + "io/ioutil" + "strings" + + "compress/gzip" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,10 +39,12 @@ import ( // GrafanaDashboardSpec defines the desired state of GrafanaDashboard type GrafanaDashboardSpec struct { Json string `json:"json,omitempty"` + GzipJson string `json:"gzipJson,omitempty"` Jsonnet string `json:"jsonnet,omitempty"` Plugins PluginList `json:"plugins,omitempty"` Url string `json:"url,omitempty"` ConfigMapRef *corev1.ConfigMapKeySelector `json:"configMapRef,omitempty"` + GzipConfigMapRef *corev1.ConfigMapKeySelector `json:"gzipConfigMapRef,omitempty"` Datasources []GrafanaDashboardDatasource `json:"datasources,omitempty"` CustomFolderName string `json:"customFolderName,omitempty"` GrafanaCom *GrafanaDashboardGrafanaComSource `json:"grafanaCom,omitempty"` @@ -100,6 +108,7 @@ func (d *GrafanaDashboard) Hash() string { } io.WriteString(hash, d.Spec.Json) // nolint + io.WriteString(hash, d.Spec.GzipJson) // nolint io.WriteString(hash, d.Spec.Url) // nolint io.WriteString(hash, d.Spec.Jsonnet) // nolint io.WriteString(hash, d.Namespace) // nolint @@ -109,6 +118,10 @@ func (d *GrafanaDashboard) Hash() string { io.WriteString(hash, d.Spec.ConfigMapRef.Name) // nolint io.WriteString(hash, d.Spec.ConfigMapRef.Key) // nolint } + if d.Spec.GzipConfigMapRef != nil { + io.WriteString(hash, d.Spec.GzipConfigMapRef.Name) // nolint + io.WriteString(hash, d.Spec.GzipConfigMapRef.Key) // nolint + } if d.Spec.GrafanaCom != nil { io.WriteString(hash, fmt.Sprint((d.Spec.GrafanaCom.Id))) // nolint @@ -121,7 +134,16 @@ func (d *GrafanaDashboard) Hash() string { } func (d *GrafanaDashboard) Parse(optional string) (map[string]interface{}, error) { - var dashboardBytes = []byte(d.Spec.Json) + var dashboardBytes []byte + if d.Spec.GzipJson != "" { + var err error + dashboardBytes, err = DecodeBase64Gzip(d.Spec.GzipJson) + if err != nil { + return nil, err + } + } else { + dashboardBytes = []byte(d.Spec.Json) + } if optional != "" { dashboardBytes = []byte(optional) } @@ -145,3 +167,19 @@ func (d *GrafanaDashboard) UID() string { // Grafana allows for UIDs return fmt.Sprintf("%x", sha1.Sum([]byte(d.Namespace+d.Name))) // nolint } + +func DecodeBase64Gzip(encoded string) ([]byte, error) { + decoder, err := gzip.NewReader(base64.NewDecoder(base64.StdEncoding, strings.NewReader(encoded))) + if err != nil { + return nil, err + } + return ioutil.ReadAll(decoder) +} + +func DecodeGzip(compressed []byte) ([]byte, error) { + decoder, err := gzip.NewReader(bytes.NewReader(compressed)) + if err != nil { + return nil, err + } + return ioutil.ReadAll(decoder) +} diff --git a/api/integreatly/v1alpha1/grafanadashboard_types_test.go b/api/integreatly/v1alpha1/grafanadashboard_types_test.go new file mode 100644 index 000000000..d36ae0ee6 --- /dev/null +++ b/api/integreatly/v1alpha1/grafanadashboard_types_test.go @@ -0,0 +1,104 @@ +package v1alpha1 + +import ( + "encoding/base64" + "encoding/json" + "io/ioutil" + "reflect" + "strings" + "testing" +) + +// Encoded via cat | gzip | base64 +const encodedCompressedDashboard = ` +H4sIAAAAAAAAA3WQMU/DQAyF9/6KU2aQYAAkVliZqFgQqpzGSaxczief2wqq/nd8l5DAwOb3+dnP +8nnjXEVN9ejCwfurrJTUo4Hqlcbo0T1D6msGaVwrPLonDi117gViNdmhS+Z+/ygq6ec03IAMs4FG +/OJQaC18SihTAxtSqItd5YCF9dSgJaiwz1tb8GlqdAKx3zJ7pWiN2wIjBPS/0nOUqbPVpvK5OTTw +6fq+L5nZwzOrTF+WsUj7wQ5bhjPbcVTisAYYF2wFU7+joChHmNPXVWg/A6XQras8Jf3rghBY4Wf3 +v7Y5K997l6afpX2PI7yhJBvOf3go+LiAm6I9hWG+7LL5BgwYIaHkAQAA +` + +const decodedDashboard = ` +{ + "id": null, + "title": "Simple Dashboard from Config Map", + "tags": [], + "style": "dark", + "timezone": "browser", + "editable": true, + "hideControls": false, + "graphTooltip": 1, + "panels": [], + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "time_options": [], + "refresh_intervals": [] + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "refresh": "5s", + "schemaVersion": 17, + "version": 0, + "links": [] +} +` + +func TestDecodeDecompress(t *testing.T) { + var expected map[string]interface{} + var actual map[string]interface{} + decoded, err := DecodeBase64Gzip(encodedCompressedDashboard) + if err != nil { + t.Log("Failed to decompress/decode", err) + t.Fail() + } + err = json.Unmarshal([]byte(decoded), &actual) + if err != nil { + t.Log("Failed to parse JSON from decoded", err) + t.Fail() + } + err = json.Unmarshal([]byte(decodedDashboard), &expected) + if err != nil { + t.Log("Failed to parse JSON from ground truth", err) + t.Fail() + } + if !reflect.DeepEqual(expected, actual) { + t.Log("Decoded JSONs were not the same") + t.Fail() + } +} + +func TestDecompress(t *testing.T) { + var expected map[string]interface{} + var actual map[string]interface{} + decoded, err := ioutil.ReadAll(base64.NewDecoder(base64.StdEncoding, strings.NewReader(encodedCompressedDashboard))) + if err != nil { + t.Log("Failed to decode", err) + t.Fail() + } + decompressed, err := DecodeGzip(decoded) + if err != nil { + t.Log("Failed to decompress", err) + t.Fail() + } + err = json.Unmarshal([]byte(decompressed), &actual) + if err != nil { + t.Log("Failed to parse JSON from decoded", err) + t.Fail() + } + err = json.Unmarshal([]byte(decodedDashboard), &expected) + if err != nil { + t.Log("Failed to parse JSON from ground truth", err) + t.Fail() + } + if !reflect.DeepEqual(expected, actual) { + t.Log("Decoded JSONs were not the same") + t.Fail() + } +} diff --git a/api/integreatly/v1alpha1/zz_generated.deepcopy.go b/api/integreatly/v1alpha1/zz_generated.deepcopy.go index 10b78df33..92e0af375 100644 --- a/api/integreatly/v1alpha1/zz_generated.deepcopy.go +++ b/api/integreatly/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* @@ -1436,6 +1437,11 @@ func (in *GrafanaDashboardSpec) DeepCopyInto(out *GrafanaDashboardSpec) { *out = new(v1.ConfigMapKeySelector) (*in).DeepCopyInto(*out) } + if in.GzipConfigMapRef != nil { + in, out := &in.GzipConfigMapRef, &out.GzipConfigMapRef + *out = new(v1.ConfigMapKeySelector) + (*in).DeepCopyInto(*out) + } if in.Datasources != nil { in, out := &in.Datasources, &out.Datasources *out = make([]GrafanaDashboardDatasource, len(*in)) diff --git a/config/crd/bases/integreatly.org_grafanadashboards.yaml b/config/crd/bases/integreatly.org_grafanadashboards.yaml index a40764f87..44232a90f 100644 --- a/config/crd/bases/integreatly.org_grafanadashboards.yaml +++ b/config/crd/bases/integreatly.org_grafanadashboards.yaml @@ -76,6 +76,25 @@ spec: required: - id type: object + gzipConfigMapRef: + description: Selects a key from a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its key must be + defined + type: boolean + required: + - key + type: object + gzipJson: + type: string json: type: string jsonnet: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 87c48a1ac..27c68ef81 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -13,7 +13,7 @@ kind: Kustomization images: - name: controller newName: quay.io/grafana-operator/grafana-operator - newTag: v4.4.1 + newTag: vtesting # Protect the /metrics endpoint by putting it behind auth. # If you want your controller-manager to expose the /metrics diff --git a/controllers/grafanadashboard/dashboard_pipeline.go b/controllers/grafanadashboard/dashboard_pipeline.go index b484feb5e..d9fd57fcf 100644 --- a/controllers/grafanadashboard/dashboard_pipeline.go +++ b/controllers/grafanadashboard/dashboard_pipeline.go @@ -142,7 +142,7 @@ func (r *DashboardPipelineImpl) obtainJson() error { } if r.Dashboard.Spec.ConfigMapRef != nil { - err := r.loadDashboardFromConfigMap() + err := r.loadDashboardFromConfigMap(r.Dashboard.Spec.ConfigMapRef, false) if err != nil { r.Logger.Error(err, "failed to get config map, falling back to raw json") } else { @@ -150,6 +150,27 @@ func (r *DashboardPipelineImpl) obtainJson() error { } } + if r.Dashboard.Spec.GzipConfigMapRef != nil { + r.Logger.Info("TODO: REMOVE --- gzipConfigMapRef was provided") + err := r.loadDashboardFromConfigMap(r.Dashboard.Spec.GzipConfigMapRef, true) + if err != nil { + r.Logger.Error(err, "failed to get config map, falling back to raw json") + } else { + return nil + } + } + + if r.Dashboard.Spec.GzipJson != "" { + r.Logger.Info("TODO: REMOVE --- gzipJson was provided") + jsonBytes, err := v1alpha1.DecodeBase64Gzip(r.Dashboard.Spec.GzipJson) + if err != nil { + r.Logger.Error(err, "failed to decode/decompress gzipped json") + } else { + r.JSON = string(jsonBytes) + return nil + } + } + if r.Dashboard.Spec.Json != "" { r.JSON = r.Dashboard.Spec.Json return nil @@ -385,9 +406,9 @@ func (r *DashboardPipelineImpl) getFileType(path string) SourceType { } // Try to obtain the dashboard json from a config map -func (r *DashboardPipelineImpl) loadDashboardFromConfigMap() error { +func (r *DashboardPipelineImpl) loadDashboardFromConfigMap(ref *corev1.ConfigMapKeySelector, binaryCompressed bool) error { ctx := context.Background() - objectKey := client.ObjectKey{Name: r.Dashboard.Spec.ConfigMapRef.Name, Namespace: r.Dashboard.Namespace} + objectKey := client.ObjectKey{Name: ref.Name, Namespace: r.Dashboard.Namespace} var cm corev1.ConfigMap err := r.Client.Get(ctx, objectKey, &cm) @@ -395,7 +416,15 @@ func (r *DashboardPipelineImpl) loadDashboardFromConfigMap() error { return err } - r.JSON = cm.Data[r.Dashboard.Spec.ConfigMapRef.Key] + if binaryCompressed { + jsonBytes, err := v1alpha1.DecodeGzip(cm.BinaryData[ref.Key]) + if err != nil { + return err + } + r.JSON = string(jsonBytes) + } else { + r.JSON = cm.Data[ref.Key] + } return nil } diff --git a/deploy/examples/dashboards/CompressedDashboard.yaml b/deploy/examples/dashboards/CompressedDashboard.yaml new file mode 100644 index 000000000..d25b2b25e --- /dev/null +++ b/deploy/examples/dashboards/CompressedDashboard.yaml @@ -0,0 +1,13 @@ +apiVersion: integreatly.org/v1alpha1 +kind: GrafanaDashboard +metadata: + name: compressed-dashboard + labels: + app: grafana +spec: + gzipJson: |- + H4sIAAAAAAAAA3WQwU7DMAyG73uKqmeQ4ABIXMcjIC4ITe7iNtbSOLK9TTDt3UnS0sKBm//Pv/1b + vmyapiXXPjfxGMJNUUYWMIN2y2MSVEXXvID6jkFcO1lg0Ox4/6hK7XMacCCH2UAjfnGstBM+K8rU + QEcGXbWbHLEyTw63HE04lK09BJ0ag0Dyr8zBKOXGfYUJIoZf6SUqq0uus+qFxxIa+Xz76Gtm8fDM + 2qyvy1ii/SEftgwXtuNkxHENyFywz3/wO4qGcoI5fV2FYwpgFId1VSC1vy6IkQ1+dv9rm7PKvQ86 + /Uz3Hkd4Q9E8XP7wVPFpAXdVB4qH+bLr5huze+uv2AEAAA== diff --git a/deploy/examples/dashboards/DashboardFromCompressedConfigMap.yaml b/deploy/examples/dashboards/DashboardFromCompressedConfigMap.yaml new file mode 100644 index 000000000..7de089bef --- /dev/null +++ b/deploy/examples/dashboards/DashboardFromCompressedConfigMap.yaml @@ -0,0 +1,23 @@ +apiVersion: integreatly.org/v1alpha1 +kind: GrafanaDashboard +metadata: + name: grafana-dashboard-from-compressed-config-map + labels: + app: grafana +spec: + json: "" + gzipConfigMapRef: + name: simple-dashboard-from-compressed-cm + key: foo +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: simple-dashboard-from-compressed-cm +binaryData: + foo: |- + H4sIAAAAAAAAA3VQPU/EMAzd71dEnUGCAZBYYWXixILQyb24jdU0jhLfneB0/x0nLS0MbH7P78Py + eWNMQ7Z5NOHg/VVBQuJRieaVxujRPEN2LUOypks8miceY8Kc0eoYOurNC8RmckKf1fj+UVGWzynH + QhpmAY34xaGybeJTxjQt0JJAW+WSDlg5Rxa1QRL7ktqBz9OiTxDdltkLRV3cVjJCQP+rvVQpOuus + qFxeSgOfru9d7SwanrlG8WWxRdoPethiLtyOoxCHtUD5hJ0+wu0oCKYjzO1rFOr7QCj0a5SnLH9V + EAIL/GT/K5u7yr13efpZ3jsc4Q1TVnP5w0OljwtxU7GnMMyXXTbf/Wift+8BAAA= diff --git a/deploy/manifests/latest/crds.yaml b/deploy/manifests/latest/crds.yaml index bac6a931b..e13128008 100644 --- a/deploy/manifests/latest/crds.yaml +++ b/deploy/manifests/latest/crds.yaml @@ -74,6 +74,25 @@ spec: required: - id type: object + gzipConfigMapRef: + description: Selects a key from a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its key must be + defined + type: boolean + required: + - key + type: object + gzipJson: + type: string json: type: string jsonnet: diff --git a/deploy/manifests/latest/deployment.yaml b/deploy/manifests/latest/deployment.yaml index 6867129cd..216ba8379 100644 --- a/deploy/manifests/latest/deployment.yaml +++ b/deploy/manifests/latest/deployment.yaml @@ -62,7 +62,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: quay.io/grafana-operator/grafana-operator:v4.4.1 + image: quay.io/grafana-operator/grafana-operator:vtesting imagePullPolicy: Always livenessProbe: httpGet: diff --git a/documentation/api.md b/documentation/api.md index 2186a7106..95fa49aad 100644 --- a/documentation/api.md +++ b/documentation/api.md @@ -117,6 +117,20 @@ GrafanaDashboardSpec defines the desired state of GrafanaDashboard
false + + gzipConfigMapRef + object + + Selects a key from a ConfigMap.
+ + false + + gzipJson + string + +
+ + false json string @@ -258,6 +272,47 @@ Selects a key from a ConfigMap. +### GrafanaDashboard.spec.gzipConfigMapRef +[↩ Parent](#grafanadashboardspec) + + + +Selects a key from a ConfigMap. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
keystring + The key to select.
+ 		true
namestring + Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?
+ 		false
optionalboolean + Specify whether the ConfigMap or its key must be defined
+ 		false
+ + ### GrafanaDashboard.spec.plugins[index] [↩ Parent](#grafanadashboardspec) diff --git a/documentation/dashboards.md b/documentation/dashboards.md index 06efdf8c6..2f985e26f 100644 --- a/documentation/dashboards.md +++ b/documentation/dashboards.md @@ -12,6 +12,7 @@ The following properties are accepted in the `spec`: * *json*: Raw json string with the dashboard contents. Check the [official documentation](https://grafana.com/docs/reference/dashboard/#dashboard-json). +* *gzipJson*: Raw json that has been gzipped, and then base64-encoded, similar to a Secret. * *jsonnet*: Jsonnet source. The [Grafonnet](https://grafana.github.io/grafonnet-lib/) library is made available automatically and can be imported. * *url*: Url address to download a json or jsonnet string with the dashboard contents. @@ -20,6 +21,7 @@ The following properties are accepted in the `spec`: * *plugins*: A list of plugins required by the dashboard. They will be installed by the operator if not already present. * *datasources*: A list of datasources to be used as inputs. See [datasource inputs](#datasource-inputs). * *configMapRef*: Import dashboards from config maps. See [config map references](#config-map-references). +* *gzipConfigMapRef*: Same as `configMapRef`, but the referenced field is decoded like `gzipJson` is. * *customFolderName*: Assign this dashboard to a custom folder, if no folder with this name exists on the instance, then a new one will be created. * _Note_: Folders with custom names are not managed by the operator, by purposeful design they won't be deleted when @@ -200,3 +202,18 @@ _Note_ : Deletion of unmanaged folders requires manual intervention. To move a dashboard between managed and unmanaged folders, simply remove or add the `CustomFolderName` field value from the dashboard spec, this will update the hash of the dashboard on the next reconcile loop, and re-add the dashboard to the desired folder. + +### Compressed Dashboard JSON + +Grafana dashboards can get quite large, and Kubernetes has a rather small maximum size of resources, which is made +all the smaller when including the "kubectl.kubernetes.io/last-applied-configuration" annotation. Because JSON is +mostly ASCII text, it compresses quite well. To support these situations, the GrafanaDashboard has two fields, +`GzipJson` and `GzipConfigMapRef` which will first decode the data in question as Base 64, much in the same way +that a standard Kubernetes Secret does, and then decompress it with Gzip. + +You can compress a dashboard from the command line like so: + +```bash +# yq can be obtained from https://github.com/mikefarah/yq +COMPRESSED_DASHBOARD="$(cat ${dashboard_json_file} | gzip | base64)" yq -i '.spec.gzipJson = strenv(COMPRESSED_DASHBOARD)' ${grafanadashboard_yaml_file} +``` From 345f8dc4cd1bd8959bd14830588a616d1e8d626b Mon Sep 17 00:00:00 2001 From: Andrew Melnick Date: Fri, 27 May 2022 19:09:17 -0600 Subject: [PATCH 2/8] Make GrafanaDashboard.spec.gzipJson bytes, not string --- .../v1alpha1/grafanadashboard_types.go | 20 ++++---------- .../v1alpha1/grafanadashboard_types_test.go | 26 +------------------ .../v1alpha1/zz_generated.deepcopy.go | 5 ++++ .../integreatly.org_grafanadashboards.yaml | 1 + .../grafanadashboard/dashboard_pipeline.go | 6 ++--- .../dashboards/CompressedDashboard.yaml | 6 +---- deploy/manifests/latest/crds.yaml | 1 + documentation/api.md | 2 ++ documentation/dashboards.md | 8 +++++- 9 files changed, 26 insertions(+), 49 deletions(-) diff --git a/api/integreatly/v1alpha1/grafanadashboard_types.go b/api/integreatly/v1alpha1/grafanadashboard_types.go index 2a5624f8c..135673d1a 100644 --- a/api/integreatly/v1alpha1/grafanadashboard_types.go +++ b/api/integreatly/v1alpha1/grafanadashboard_types.go @@ -20,12 +20,10 @@ import ( "bytes" "crypto/sha1" // nolint "crypto/sha256" - "encoding/base64" "encoding/json" "fmt" "io" "io/ioutil" - "strings" "compress/gzip" @@ -39,7 +37,7 @@ import ( // GrafanaDashboardSpec defines the desired state of GrafanaDashboard type GrafanaDashboardSpec struct { Json string `json:"json,omitempty"` - GzipJson string `json:"gzipJson,omitempty"` + GzipJson []byte `json:"gzipJson,omitempty"` Jsonnet string `json:"jsonnet,omitempty"` Plugins PluginList `json:"plugins,omitempty"` Url string `json:"url,omitempty"` @@ -108,7 +106,7 @@ func (d *GrafanaDashboard) Hash() string { } io.WriteString(hash, d.Spec.Json) // nolint - io.WriteString(hash, d.Spec.GzipJson) // nolint + hash.Write(d.Spec.GzipJson) // nolint io.WriteString(hash, d.Spec.Url) // nolint io.WriteString(hash, d.Spec.Jsonnet) // nolint io.WriteString(hash, d.Namespace) // nolint @@ -135,9 +133,9 @@ func (d *GrafanaDashboard) Hash() string { func (d *GrafanaDashboard) Parse(optional string) (map[string]interface{}, error) { var dashboardBytes []byte - if d.Spec.GzipJson != "" { + if d.Spec.GzipJson != nil { var err error - dashboardBytes, err = DecodeBase64Gzip(d.Spec.GzipJson) + dashboardBytes, err = Gunzip(d.Spec.GzipJson) if err != nil { return nil, err } @@ -168,15 +166,7 @@ func (d *GrafanaDashboard) UID() string { return fmt.Sprintf("%x", sha1.Sum([]byte(d.Namespace+d.Name))) // nolint } -func DecodeBase64Gzip(encoded string) ([]byte, error) { - decoder, err := gzip.NewReader(base64.NewDecoder(base64.StdEncoding, strings.NewReader(encoded))) - if err != nil { - return nil, err - } - return ioutil.ReadAll(decoder) -} - -func DecodeGzip(compressed []byte) ([]byte, error) { +func Gunzip(compressed []byte) ([]byte, error) { decoder, err := gzip.NewReader(bytes.NewReader(compressed)) if err != nil { return nil, err diff --git a/api/integreatly/v1alpha1/grafanadashboard_types_test.go b/api/integreatly/v1alpha1/grafanadashboard_types_test.go index d36ae0ee6..161f479a4 100644 --- a/api/integreatly/v1alpha1/grafanadashboard_types_test.go +++ b/api/integreatly/v1alpha1/grafanadashboard_types_test.go @@ -50,30 +50,6 @@ const decodedDashboard = ` } ` -func TestDecodeDecompress(t *testing.T) { - var expected map[string]interface{} - var actual map[string]interface{} - decoded, err := DecodeBase64Gzip(encodedCompressedDashboard) - if err != nil { - t.Log("Failed to decompress/decode", err) - t.Fail() - } - err = json.Unmarshal([]byte(decoded), &actual) - if err != nil { - t.Log("Failed to parse JSON from decoded", err) - t.Fail() - } - err = json.Unmarshal([]byte(decodedDashboard), &expected) - if err != nil { - t.Log("Failed to parse JSON from ground truth", err) - t.Fail() - } - if !reflect.DeepEqual(expected, actual) { - t.Log("Decoded JSONs were not the same") - t.Fail() - } -} - func TestDecompress(t *testing.T) { var expected map[string]interface{} var actual map[string]interface{} @@ -82,7 +58,7 @@ func TestDecompress(t *testing.T) { t.Log("Failed to decode", err) t.Fail() } - decompressed, err := DecodeGzip(decoded) + decompressed, err := Gunzip(decoded) if err != nil { t.Log("Failed to decompress", err) t.Fail() diff --git a/api/integreatly/v1alpha1/zz_generated.deepcopy.go b/api/integreatly/v1alpha1/zz_generated.deepcopy.go index 92e0af375..a0e213b2d 100644 --- a/api/integreatly/v1alpha1/zz_generated.deepcopy.go +++ b/api/integreatly/v1alpha1/zz_generated.deepcopy.go @@ -1427,6 +1427,11 @@ func (in *GrafanaDashboardRef) DeepCopy() *GrafanaDashboardRef { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GrafanaDashboardSpec) DeepCopyInto(out *GrafanaDashboardSpec) { *out = *in + if in.GzipJson != nil { + in, out := &in.GzipJson, &out.GzipJson + *out = make([]byte, len(*in)) + copy(*out, *in) + } if in.Plugins != nil { in, out := &in.Plugins, &out.Plugins *out = make(PluginList, len(*in)) diff --git a/config/crd/bases/integreatly.org_grafanadashboards.yaml b/config/crd/bases/integreatly.org_grafanadashboards.yaml index 44232a90f..9c56feca6 100644 --- a/config/crd/bases/integreatly.org_grafanadashboards.yaml +++ b/config/crd/bases/integreatly.org_grafanadashboards.yaml @@ -94,6 +94,7 @@ spec: - key type: object gzipJson: + format: byte type: string json: type: string diff --git a/controllers/grafanadashboard/dashboard_pipeline.go b/controllers/grafanadashboard/dashboard_pipeline.go index d9fd57fcf..7b185ba20 100644 --- a/controllers/grafanadashboard/dashboard_pipeline.go +++ b/controllers/grafanadashboard/dashboard_pipeline.go @@ -160,9 +160,9 @@ func (r *DashboardPipelineImpl) obtainJson() error { } } - if r.Dashboard.Spec.GzipJson != "" { + if r.Dashboard.Spec.GzipJson != nil { r.Logger.Info("TODO: REMOVE --- gzipJson was provided") - jsonBytes, err := v1alpha1.DecodeBase64Gzip(r.Dashboard.Spec.GzipJson) + jsonBytes, err := v1alpha1.Gunzip(r.Dashboard.Spec.GzipJson) if err != nil { r.Logger.Error(err, "failed to decode/decompress gzipped json") } else { @@ -417,7 +417,7 @@ func (r *DashboardPipelineImpl) loadDashboardFromConfigMap(ref *corev1.ConfigMap } if binaryCompressed { - jsonBytes, err := v1alpha1.DecodeGzip(cm.BinaryData[ref.Key]) + jsonBytes, err := v1alpha1.Gunzip(cm.BinaryData[ref.Key]) if err != nil { return err } diff --git a/deploy/examples/dashboards/CompressedDashboard.yaml b/deploy/examples/dashboards/CompressedDashboard.yaml index d25b2b25e..76afaa5f1 100644 --- a/deploy/examples/dashboards/CompressedDashboard.yaml +++ b/deploy/examples/dashboards/CompressedDashboard.yaml @@ -6,8 +6,4 @@ metadata: app: grafana spec: gzipJson: |- - H4sIAAAAAAAAA3WQwU7DMAyG73uKqmeQ4ABIXMcjIC4ITe7iNtbSOLK9TTDt3UnS0sKBm//Pv/1b - vmyapiXXPjfxGMJNUUYWMIN2y2MSVEXXvID6jkFcO1lg0Ox4/6hK7XMacCCH2UAjfnGstBM+K8rU - QEcGXbWbHLEyTw63HE04lK09BJ0ag0Dyr8zBKOXGfYUJIoZf6SUqq0uus+qFxxIa+Xz76Gtm8fDM - 2qyvy1ii/SEftgwXtuNkxHENyFywz3/wO4qGcoI5fV2FYwpgFId1VSC1vy6IkQ1+dv9rm7PKvQ86 - /Uz3Hkd4Q9E8XP7wVPFpAXdVB4qH+bLr5huze+uv2AEAAA== + H4sIAAAAAAAAA3WQwU7DMAyG73uKqmeQ4ABIXMcjIC4ITe7iNtbSOLK9TTDt3UnS0sKBm//Pv/1bvmyapiXXPjfxGMJNUUYWMIN2y2MSVEXXvID6jkFcO1lg0Ox4/6hK7XMacCCH2UAjfnGstBM+K8rUQEcGXbWbHLEyTw63HE04lK09BJ0ag0Dyr8zBKOXGfYUJIoZf6SUqq0uus+qFxxIa+Xz76Gtm8fDM2qyvy1ii/SEftgwXtuNkxHENyFywz3/wO4qGcoI5fV2FYwpgFId1VSC1vy6IkQ1+dv9rm7PKvQ86/Uz3Hkd4Q9E8XP7wVPFpAXdVB4qH+bLr5huze+uv2AEAAA== diff --git a/deploy/manifests/latest/crds.yaml b/deploy/manifests/latest/crds.yaml index e13128008..ec6678682 100644 --- a/deploy/manifests/latest/crds.yaml +++ b/deploy/manifests/latest/crds.yaml @@ -92,6 +92,7 @@ spec: - key type: object gzipJson: + format: byte type: string json: type: string diff --git a/documentation/api.md b/documentation/api.md index 95fa49aad..c43e4c55c 100644 --- a/documentation/api.md +++ b/documentation/api.md @@ -129,6 +129,8 @@ GrafanaDashboardSpec defines the desired state of GrafanaDashboard string
+
+ Format: byte
false diff --git a/documentation/dashboards.md b/documentation/dashboards.md index 2f985e26f..04d3760b5 100644 --- a/documentation/dashboards.md +++ b/documentation/dashboards.md @@ -215,5 +215,11 @@ You can compress a dashboard from the command line like so: ```bash # yq can be obtained from https://github.com/mikefarah/yq -COMPRESSED_DASHBOARD="$(cat ${dashboard_json_file} | gzip | base64)" yq -i '.spec.gzipJson = strenv(COMPRESSED_DASHBOARD)' ${grafanadashboard_yaml_file} +COMPRESSED_DASHBOARD="$(cat ${dashboard_json_file} | gzip | base64 -w0)" yq -i '.spec.gzipJson = strenv(COMPRESSED_DASHBOARD)' ${grafanadashboard_yaml_file} +``` + +You can similarly compress a dashboard into a ConfigMap like so: + +```bash +COMPRESSED_DASHBOARD="$(cat ${dashboard_json_file} | gzip | base64 -w0)" yq -i ".binaryData.${dashboard_key} = strenv(COMPRESSED_DASHBOARD)" ${configmap_yaml_file} ``` From 04bdf2bfa15e8fdd41aee3d27509cbc84da16afb Mon Sep 17 00:00:00 2001 From: Andrew Melnick Date: Fri, 27 May 2022 19:31:10 -0600 Subject: [PATCH 3/8] Remove testing leftovers --- config/manager/kustomization.yaml | 2 +- controllers/grafanadashboard/dashboard_pipeline.go | 2 -- deploy/manifests/latest/deployment.yaml | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 27c68ef81..87c48a1ac 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -13,7 +13,7 @@ kind: Kustomization images: - name: controller newName: quay.io/grafana-operator/grafana-operator - newTag: vtesting + newTag: v4.4.1 # Protect the /metrics endpoint by putting it behind auth. # If you want your controller-manager to expose the /metrics diff --git a/controllers/grafanadashboard/dashboard_pipeline.go b/controllers/grafanadashboard/dashboard_pipeline.go index 7b185ba20..08b0f9e53 100644 --- a/controllers/grafanadashboard/dashboard_pipeline.go +++ b/controllers/grafanadashboard/dashboard_pipeline.go @@ -151,7 +151,6 @@ func (r *DashboardPipelineImpl) obtainJson() error { } if r.Dashboard.Spec.GzipConfigMapRef != nil { - r.Logger.Info("TODO: REMOVE --- gzipConfigMapRef was provided") err := r.loadDashboardFromConfigMap(r.Dashboard.Spec.GzipConfigMapRef, true) if err != nil { r.Logger.Error(err, "failed to get config map, falling back to raw json") @@ -161,7 +160,6 @@ func (r *DashboardPipelineImpl) obtainJson() error { } if r.Dashboard.Spec.GzipJson != nil { - r.Logger.Info("TODO: REMOVE --- gzipJson was provided") jsonBytes, err := v1alpha1.Gunzip(r.Dashboard.Spec.GzipJson) if err != nil { r.Logger.Error(err, "failed to decode/decompress gzipped json") diff --git a/deploy/manifests/latest/deployment.yaml b/deploy/manifests/latest/deployment.yaml index 216ba8379..6867129cd 100644 --- a/deploy/manifests/latest/deployment.yaml +++ b/deploy/manifests/latest/deployment.yaml @@ -62,7 +62,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: quay.io/grafana-operator/grafana-operator:vtesting + image: quay.io/grafana-operator/grafana-operator:v4.4.1 imagePullPolicy: Always livenessProbe: httpGet: From d53dd0763abdf9223023919f402cdc3877c07f48 Mon Sep 17 00:00:00 2001 From: Andrew Melnick Date: Sun, 29 May 2022 14:46:59 -0600 Subject: [PATCH 4/8] Remove line auto-added by vim --- api/integreatly/v1alpha1/zz_generated.deepcopy.go | 1 - 1 file changed, 1 deletion(-) diff --git a/api/integreatly/v1alpha1/zz_generated.deepcopy.go b/api/integreatly/v1alpha1/zz_generated.deepcopy.go index a0e213b2d..aecdfd259 100644 --- a/api/integreatly/v1alpha1/zz_generated.deepcopy.go +++ b/api/integreatly/v1alpha1/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* From c088fb13a6e7681e32bf33c2fcda37afb9f2647e Mon Sep 17 00:00:00 2001 From: edvin norling Date: Mon, 30 May 2022 07:50:57 +0200 Subject: [PATCH 5/8] Fix markdown liniting Even thgouh this file wast updated in this PR orignially I fix it here to pass the CI. --- documentation/dashboards.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/dashboards.md b/documentation/dashboards.md index 04d3760b5..e2318aad1 100644 --- a/documentation/dashboards.md +++ b/documentation/dashboards.md @@ -24,7 +24,7 @@ The following properties are accepted in the `spec`: * *gzipConfigMapRef*: Same as `configMapRef`, but the referenced field is decoded like `gzipJson` is. * *customFolderName*: Assign this dashboard to a custom folder, if no folder with this name exists on the instance, then a new one will be created. - * _Note_: Folders with custom names are not managed by the operator, by purposeful design they won't be deleted when + * *Note*: Folders with custom names are not managed by the operator, by purposeful design they won't be deleted when empty, deletion for these requires manual intervention. ## Creating a new dashboard @@ -184,16 +184,16 @@ field is an empty string `""`) then the dashboard will be assigned to the namesp into which the dashboard was deployed, i.e if deployed to `test-ns` then a new folder (if one with that name doesn't exist already) will be created and named `test-ns` and the dashboard assigned to it. -Default assignment of the dashboards to namespace-named folders is consider as a _managed folder_, this means that when +Default assignment of the dashboards to namespace-named folders is consider as a *managed folder*, this means that when a managed folder has no dashboards assigned to it, it will be deleted to clean up the UI. ### Unmanaged folders When defining `customFolderName` in a dashboard, the resulting folder will be named as the string in this field -specifies, this is considered as an _unmanaged folder_ and won't be deleted even if empty and will remain on the UI. +specifies, this is considered as an *unmanaged folder* and won't be deleted even if empty and will remain on the UI. Custom folders can have multiple dashboards assigned to them. -_Note_ : Deletion of unmanaged folders requires manual intervention. +*Note* : Deletion of unmanaged folders requires manual intervention. ![dashboard-folder-assignment.svg](./resources/dashboard-folder-assignment.svg) From cc93aca37d03ab1c0b26087c4f98f457b1f7a4ce Mon Sep 17 00:00:00 2001 From: Andrew Melnick Date: Wed, 8 Jun 2022 20:00:09 -0600 Subject: [PATCH 6/8] Add comments for GrafanaDashboard fields --- .../v1alpha1/grafanadashboard_types.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/api/integreatly/v1alpha1/grafanadashboard_types.go b/api/integreatly/v1alpha1/grafanadashboard_types.go index 135673d1a..3732c2c9b 100644 --- a/api/integreatly/v1alpha1/grafanadashboard_types.go +++ b/api/integreatly/v1alpha1/grafanadashboard_types.go @@ -36,12 +36,17 @@ import ( // GrafanaDashboardSpec defines the desired state of GrafanaDashboard type GrafanaDashboardSpec struct { - Json string `json:"json,omitempty"` - GzipJson []byte `json:"gzipJson,omitempty"` - Jsonnet string `json:"jsonnet,omitempty"` - Plugins PluginList `json:"plugins,omitempty"` - Url string `json:"url,omitempty"` - ConfigMapRef *corev1.ConfigMapKeySelector `json:"configMapRef,omitempty"` + // Json is the dashboard's JSON + Json string `json:"json,omitempty"` + // GzipJson the dashboard's JSON compressed with Gzip. Base64-encoded when in YAML. + GzipJson []byte `json:"gzipJson,omitempty"` + Jsonnet string `json:"jsonnet,omitempty"` + Plugins PluginList `json:"plugins,omitempty"` + Url string `json:"url,omitempty"` + // ConfigMapRef is a reference to a ConfigMap data field containing the dashboard's JSON + ConfigMapRef *corev1.ConfigMapKeySelector `json:"configMapRef,omitempty"` + // GzipConfigMapRef is a reference to a ConfigMap binaryData field containing + // the dashboard's JSON, compressed with Gzip. GzipConfigMapRef *corev1.ConfigMapKeySelector `json:"gzipConfigMapRef,omitempty"` Datasources []GrafanaDashboardDatasource `json:"datasources,omitempty"` CustomFolderName string `json:"customFolderName,omitempty"` From 3b6c75a65a5e997b70d5b0b640697a11e430e03b Mon Sep 17 00:00:00 2001 From: Andrew Melnick Date: Sun, 12 Jun 2022 10:17:36 -0600 Subject: [PATCH 7/8] Regen manifests w/ comments --- .../crd/bases/integreatly.org_grafanadashboards.yaml | 9 +++++++-- deploy/manifests/latest/crds.yaml | 9 +++++++-- documentation/api.md | 12 ++++++------ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/config/crd/bases/integreatly.org_grafanadashboards.yaml b/config/crd/bases/integreatly.org_grafanadashboards.yaml index 9c56feca6..fd99a97cf 100644 --- a/config/crd/bases/integreatly.org_grafanadashboards.yaml +++ b/config/crd/bases/integreatly.org_grafanadashboards.yaml @@ -37,7 +37,8 @@ spec: description: GrafanaDashboardSpec defines the desired state of GrafanaDashboard properties: configMapRef: - description: Selects a key from a ConfigMap. + description: ConfigMapRef is a reference to a ConfigMap data field + containing the dashboard's JSON properties: key: description: The key to select. @@ -77,7 +78,8 @@ spec: - id type: object gzipConfigMapRef: - description: Selects a key from a ConfigMap. + description: GzipConfigMapRef is a reference to a ConfigMap binaryData + field containing the dashboard's JSON, compressed with Gzip. properties: key: description: The key to select. @@ -94,9 +96,12 @@ spec: - key type: object gzipJson: + description: GzipJson the dashboard's JSON compressed with Gzip. Base64-encoded + when in YAML. format: byte type: string json: + description: Json is the dashboard's JSON type: string jsonnet: type: string diff --git a/deploy/manifests/latest/crds.yaml b/deploy/manifests/latest/crds.yaml index ec6678682..56567638a 100644 --- a/deploy/manifests/latest/crds.yaml +++ b/deploy/manifests/latest/crds.yaml @@ -35,7 +35,8 @@ spec: description: GrafanaDashboardSpec defines the desired state of GrafanaDashboard properties: configMapRef: - description: Selects a key from a ConfigMap. + description: ConfigMapRef is a reference to a ConfigMap data field + containing the dashboard's JSON properties: key: description: The key to select. @@ -75,7 +76,8 @@ spec: - id type: object gzipConfigMapRef: - description: Selects a key from a ConfigMap. + description: GzipConfigMapRef is a reference to a ConfigMap binaryData + field containing the dashboard's JSON, compressed with Gzip. properties: key: description: The key to select. @@ -92,9 +94,12 @@ spec: - key type: object gzipJson: + description: GzipJson the dashboard's JSON compressed with Gzip. Base64-encoded + when in YAML. format: byte type: string json: + description: Json is the dashboard's JSON type: string jsonnet: type: string diff --git a/documentation/api.md b/documentation/api.md index c43e4c55c..296e617d3 100644 --- a/documentation/api.md +++ b/documentation/api.md @@ -93,7 +93,7 @@ GrafanaDashboardSpec defines the desired state of GrafanaDashboard configMapRef object - Selects a key from a ConfigMap.
+ ConfigMapRef is a reference to a ConfigMap data field containing the dashboard's JSON
false @@ -121,14 +121,14 @@ GrafanaDashboardSpec defines the desired state of GrafanaDashboard gzipConfigMapRef object - Selects a key from a ConfigMap.
+ GzipConfigMapRef is a reference to a ConfigMap binaryData field containing the dashboard's JSON, compressed with Gzip.
false gzipJson string -
+ GzipJson the dashboard's JSON compressed with Gzip. Base64-encoded when in YAML.

Format: byte
@@ -137,7 +137,7 @@ GrafanaDashboardSpec defines the desired state of GrafanaDashboard json string -
+ Json is the dashboard's JSON
false @@ -170,7 +170,7 @@ GrafanaDashboardSpec defines the desired state of GrafanaDashboard -Selects a key from a ConfigMap. +ConfigMapRef is a reference to a ConfigMap data field containing the dashboard's JSON @@ -279,7 +279,7 @@ Selects a key from a ConfigMap. -Selects a key from a ConfigMap. +GzipConfigMapRef is a reference to a ConfigMap binaryData field containing the dashboard's JSON, compressed with Gzip.
From 57c93b18fbba0711e3d18fb4f5cff83043cb0116 Mon Sep 17 00:00:00 2001 From: Edvin Norling Date: Tue, 12 Jul 2022 13:14:09 +0200 Subject: [PATCH 8/8] Fix lint error --- api/integreatly/v1alpha1/grafanadashboard_types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/integreatly/v1alpha1/grafanadashboard_types.go b/api/integreatly/v1alpha1/grafanadashboard_types.go index 054af4140..2874ec571 100644 --- a/api/integreatly/v1alpha1/grafanadashboard_types.go +++ b/api/integreatly/v1alpha1/grafanadashboard_types.go @@ -126,8 +126,8 @@ func (d *GrafanaDashboard) Hash() string { io.WriteString(hash, input.InputName) // nolint } - io.WriteString(hash, d.Spec.Json) // nolint - hash.Write(d.Spec.GzipJson) // nolint + io.WriteString(hash, d.Spec.Json) // nolint + hash.Write(d.Spec.GzipJson) io.WriteString(hash, d.Spec.Url) // nolint io.WriteString(hash, d.Spec.Jsonnet) // nolint io.WriteString(hash, d.Namespace) // nolint