Merge pull request #767 from meln5674/feature/grafana-operator-726

Add ability to provide compressed JSON for large dashboards

api/integreatly/v1alpha1/grafanadashboard_types.go

@@ -17,11 +17,15 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"bytes"
 	"crypto/sha1" // nolint
 	"crypto/sha256"
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
+
+	"compress/gzip"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -32,11 +36,18 @@ import (
 
 // GrafanaDashboardSpec defines the desired state of GrafanaDashboard
 type GrafanaDashboardSpec struct {
-	Json             string                            `json:"json,omitempty"`
-	Jsonnet          string                            `json:"jsonnet,omitempty"`
-	Plugins          PluginList                        `json:"plugins,omitempty"`
-	Url              string                            `json:"url,omitempty"`
-	ConfigMapRef     *corev1.ConfigMapKeySelector      `json:"configMapRef,omitempty"`
+	// Json is the dashboard's JSON
+	Json string `json:"json,omitempty"`
+	// GzipJson the dashboard's JSON compressed with Gzip. Base64-encoded when in YAML.
+	GzipJson []byte     `json:"gzipJson,omitempty"`
+	Jsonnet  string     `json:"jsonnet,omitempty"`
+	Plugins  PluginList `json:"plugins,omitempty"`
+	Url      string     `json:"url,omitempty"`
+	// ConfigMapRef is a reference to a ConfigMap data field containing the dashboard's JSON
+	ConfigMapRef *corev1.ConfigMapKeySelector `json:"configMapRef,omitempty"`
+	// GzipConfigMapRef is a reference to a ConfigMap binaryData field containing
+	// the dashboard's JSON, compressed with Gzip.
+	GzipConfigMapRef *corev1.ConfigMapKeySelector      `json:"gzipConfigMapRef,omitempty"`
 	Datasources      []GrafanaDashboardDatasource      `json:"datasources,omitempty"`
 	CustomFolderName string                            `json:"customFolderName,omitempty"`
 	GrafanaCom       *GrafanaDashboardGrafanaComSource `json:"grafanaCom,omitempty"`
@@ -115,7 +126,8 @@ func (d *GrafanaDashboard) Hash() string {
 		io.WriteString(hash, input.InputName)      // nolint
 	}
 
-	io.WriteString(hash, d.Spec.Json)             // nolint
+	io.WriteString(hash, d.Spec.Json) // nolint
+	hash.Write(d.Spec.GzipJson)
 	io.WriteString(hash, d.Spec.Url)              // nolint
 	io.WriteString(hash, d.Spec.Jsonnet)          // nolint
 	io.WriteString(hash, d.Namespace)             // nolint
@@ -125,6 +137,10 @@ func (d *GrafanaDashboard) Hash() string {
 		io.WriteString(hash, d.Spec.ConfigMapRef.Name) // nolint
 		io.WriteString(hash, d.Spec.ConfigMapRef.Key)  // nolint
 	}
+	if d.Spec.GzipConfigMapRef != nil {
+		io.WriteString(hash, d.Spec.GzipConfigMapRef.Name) // nolint
+		io.WriteString(hash, d.Spec.GzipConfigMapRef.Key)  // nolint
+	}
 
 	if d.Spec.GrafanaCom != nil {
 		io.WriteString(hash, fmt.Sprint((d.Spec.GrafanaCom.Id))) // nolint
@@ -141,7 +157,16 @@ func (d *GrafanaDashboard) Hash() string {
 }
 
 func (d *GrafanaDashboard) Parse(optional string) (map[string]interface{}, error) {
-	var dashboardBytes = []byte(d.Spec.Json)
+	var dashboardBytes []byte
+	if d.Spec.GzipJson != nil {
+		var err error
+		dashboardBytes, err = Gunzip(d.Spec.GzipJson)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		dashboardBytes = []byte(d.Spec.Json)
+	}
 	if optional != "" {
 		dashboardBytes = []byte(optional)
 	}
@@ -165,3 +190,11 @@ func (d *GrafanaDashboard) UID() string {
 	// Grafana allows for UIDs
 	return fmt.Sprintf("%x", sha1.Sum([]byte(d.Namespace+d.Name))) // nolint
 }
+
+func Gunzip(compressed []byte) ([]byte, error) {
+	decoder, err := gzip.NewReader(bytes.NewReader(compressed))
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.ReadAll(decoder)
+}

api/integreatly/v1alpha1/grafanadashboard_types_test.go

@@ -0,0 +1,80 @@
+package v1alpha1
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"io/ioutil"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// Encoded via cat | gzip | base64
+const encodedCompressedDashboard = `
+H4sIAAAAAAAAA3WQMU/DQAyF9/6KU2aQYAAkVliZqFgQqpzGSaxczief2wqq/nd8l5DAwOb3+dnP
+8nnjXEVN9ejCwfurrJTUo4Hqlcbo0T1D6msGaVwrPLonDi117gViNdmhS+Z+/ygq6ec03IAMs4FG
+/OJQaC18SihTAxtSqItd5YCF9dSgJaiwz1tb8GlqdAKx3zJ7pWiN2wIjBPS/0nOUqbPVpvK5OTTw
+6fq+L5nZwzOrTF+WsUj7wQ5bhjPbcVTisAYYF2wFU7+joChHmNPXVWg/A6XQras8Jf3rghBY4Wf3
+v7Y5K997l6afpX2PI7yhJBvOf3go+LiAm6I9hWG+7LL5BgwYIaHkAQAA
+`
+
+const decodedDashboard = `
+{
+  "id": null,
+  "title": "Simple Dashboard from Config Map",
+  "tags": [],
+  "style": "dark",
+  "timezone": "browser",
+  "editable": true,
+  "hideControls": false,
+  "graphTooltip": 1,
+  "panels": [],
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "time_options": [],
+    "refresh_intervals": []
+  },
+  "templating": {
+    "list": []
+  },
+  "annotations": {
+    "list": []
+  },
+  "refresh": "5s",
+  "schemaVersion": 17,
+  "version": 0,
+  "links": []
+}
+`
+
+func TestDecompress(t *testing.T) {
+	var expected map[string]interface{}
+	var actual map[string]interface{}
+	decoded, err := ioutil.ReadAll(base64.NewDecoder(base64.StdEncoding, strings.NewReader(encodedCompressedDashboard)))
+	if err != nil {
+		t.Log("Failed to decode", err)
+		t.Fail()
+	}
+	decompressed, err := Gunzip(decoded)
+	if err != nil {
+		t.Log("Failed to decompress", err)
+		t.Fail()
+	}
+	err = json.Unmarshal([]byte(decompressed), &actual)
+	if err != nil {
+		t.Log("Failed to parse JSON from decoded", err)
+		t.Fail()
+	}
+	err = json.Unmarshal([]byte(decodedDashboard), &expected)
+	if err != nil {
+		t.Log("Failed to parse JSON from ground truth", err)
+		t.Fail()
+	}
+	if !reflect.DeepEqual(expected, actual) {
+		t.Log("Decoded JSONs were not the same")
+		t.Fail()
+	}
+}

config/crd/bases/integreatly.org_grafanadashboards.yaml

@@ -37,7 +37,8 @@ spec:
             description: GrafanaDashboardSpec defines the desired state of GrafanaDashboard
             properties:
               configMapRef:
-                description: Selects a key from a ConfigMap.
+                description: ConfigMapRef is a reference to a ConfigMap data field
+                  containing the dashboard's JSON
                 properties:
                   key:
                     description: The key to select.
@@ -84,7 +85,31 @@ spec:
                 required:
                 - id
                 type: object
+              gzipConfigMapRef:
+                description: GzipConfigMapRef is a reference to a ConfigMap binaryData
+                  field containing the dashboard's JSON, compressed with Gzip.
+                properties:
+                  key:
+                    description: The key to select.
+                    type: string
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                      TODO: Add other useful fields. apiVersion, kind, uid?'
+                    type: string
+                  optional:
+                    description: Specify whether the ConfigMap or its key must be
+                      defined
+                    type: boolean
+                required:
+                - key
+                type: object
+              gzipJson:
+                description: GzipJson the dashboard's JSON compressed with Gzip. Base64-encoded
+                  when in YAML.
+                format: byte
+                type: string
               json:
+                description: Json is the dashboard's JSON
                 type: string
               jsonnet:
                 type: string

controllers/grafanadashboard/dashboard_pipeline.go

@@ -160,14 +160,33 @@ func (r *DashboardPipelineImpl) obtainJson() error {
 	}
 
 	if r.Dashboard.Spec.ConfigMapRef != nil {
-		err := r.loadDashboardFromConfigMap()
+		err := r.loadDashboardFromConfigMap(r.Dashboard.Spec.ConfigMapRef, false)
 		if err != nil {
 			r.Logger.Error(err, "failed to get config map, falling back to raw json")
 		} else {
 			return nil
 		}
 	}
 
+	if r.Dashboard.Spec.GzipConfigMapRef != nil {
+		err := r.loadDashboardFromConfigMap(r.Dashboard.Spec.GzipConfigMapRef, true)
+		if err != nil {
+			r.Logger.Error(err, "failed to get config map, falling back to raw json")
+		} else {
+			return nil
+		}
+	}
+
+	if r.Dashboard.Spec.GzipJson != nil {
+		jsonBytes, err := v1alpha1.Gunzip(r.Dashboard.Spec.GzipJson)
+		if err != nil {
+			r.Logger.Error(err, "failed to decode/decompress gzipped json")
+		} else {
+			r.JSON = string(jsonBytes)
+			return nil
+		}
+	}
+
 	if r.Dashboard.Spec.Json != "" {
 		r.JSON = r.Dashboard.Spec.Json
 		return nil
@@ -456,17 +475,25 @@ func (r *DashboardPipelineImpl) getFileType(path string) SourceType {
 }
 
 // Try to obtain the dashboard json from a config map
-func (r *DashboardPipelineImpl) loadDashboardFromConfigMap() error {
+func (r *DashboardPipelineImpl) loadDashboardFromConfigMap(ref *corev1.ConfigMapKeySelector, binaryCompressed bool) error {
 	ctx := context.Background()
-	objectKey := client.ObjectKey{Name: r.Dashboard.Spec.ConfigMapRef.Name, Namespace: r.Dashboard.Namespace}
+	objectKey := client.ObjectKey{Name: ref.Name, Namespace: r.Dashboard.Namespace}
 
 	var cm corev1.ConfigMap
 	err := r.Client.Get(ctx, objectKey, &cm)
 	if err != nil {
 		return err
 	}
 
-	r.JSON = cm.Data[r.Dashboard.Spec.ConfigMapRef.Key]
+	if binaryCompressed {
+		jsonBytes, err := v1alpha1.Gunzip(cm.BinaryData[ref.Key])
+		if err != nil {
+			return err
+		}
+		r.JSON = string(jsonBytes)
+	} else {
+		r.JSON = cm.Data[ref.Key]
+	}
 
 	return nil
 }

deploy/examples/dashboards/CompressedDashboard.yaml

@@ -0,0 +1,9 @@
+apiVersion: integreatly.org/v1alpha1
+kind: GrafanaDashboard
+metadata:
+  name: compressed-dashboard
+  labels:
+    app: grafana
+spec:
+  gzipJson: |-
+    H4sIAAAAAAAAA3WQwU7DMAyG73uKqmeQ4ABIXMcjIC4ITe7iNtbSOLK9TTDt3UnS0sKBm//Pv/1bvmyapiXXPjfxGMJNUUYWMIN2y2MSVEXXvID6jkFcO1lg0Ox4/6hK7XMacCCH2UAjfnGstBM+K8rUQEcGXbWbHLEyTw63HE04lK09BJ0ag0Dyr8zBKOXGfYUJIoZf6SUqq0uus+qFxxIa+Xz76Gtm8fDM2qyvy1ii/SEftgwXtuNkxHENyFywz3/wO4qGcoI5fV2FYwpgFId1VSC1vy6IkQ1+dv9rm7PKvQ86/Uz3Hkd4Q9E8XP7wVPFpAXdVB4qH+bLr5huze+uv2AEAAA==

deploy/examples/dashboards/DashboardFromCompressedConfigMap.yaml

@@ -0,0 +1,23 @@
+apiVersion: integreatly.org/v1alpha1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-dashboard-from-compressed-config-map
+  labels:
+    app: grafana
+spec:
+  json: ""
+  gzipConfigMapRef:
+    name: simple-dashboard-from-compressed-cm
+    key: foo
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: simple-dashboard-from-compressed-cm
+binaryData:
+  foo: |-
+    H4sIAAAAAAAAA3VQPU/EMAzd71dEnUGCAZBYYWXixILQyb24jdU0jhLfneB0/x0nLS0MbH7P78Py
+    eWNMQ7Z5NOHg/VVBQuJRieaVxujRPEN2LUOypks8miceY8Kc0eoYOurNC8RmckKf1fj+UVGWzynH
+    QhpmAY34xaGybeJTxjQt0JJAW+WSDlg5Rxa1QRL7ktqBz9OiTxDdltkLRV3cVjJCQP+rvVQpOuus
+    qFxeSgOfru9d7SwanrlG8WWxRdoPethiLtyOoxCHtUD5hJ0+wu0oCKYjzO1rFOr7QCj0a5SnLH9V
+    EAIL/GT/K5u7yr13efpZ3jsc4Q1TVnP5w0OljwtxU7GnMMyXXTbf/Wift+8BAAA=

deploy/manifests/latest/crds.yaml

@@ -35,7 +35,8 @@ spec:
             description: GrafanaDashboardSpec defines the desired state of GrafanaDashboard
             properties:
               configMapRef:
-                description: Selects a key from a ConfigMap.
+                description: ConfigMapRef is a reference to a ConfigMap data field
+                  containing the dashboard's JSON
                 properties:
                   key:
                     description: The key to select.
@@ -82,7 +83,31 @@ spec:
                 required:
                 - id
                 type: object
+              gzipConfigMapRef:
+                description: GzipConfigMapRef is a reference to a ConfigMap binaryData
+                  field containing the dashboard's JSON, compressed with Gzip.
+                properties:
+                  key:
+                    description: The key to select.
+                    type: string
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                      TODO: Add other useful fields. apiVersion, kind, uid?'
+                    type: string
+                  optional:
+                    description: Specify whether the ConfigMap or its key must be
+                      defined
+                    type: boolean
+                required:
+                - key
+                type: object
+              gzipJson:
+                description: GzipJson the dashboard's JSON compressed with Gzip. Base64-encoded
+                  when in YAML.
+                format: byte
+                type: string
               json:
+                description: Json is the dashboard's JSON
                 type: string
               jsonnet:
                 type: string