.ops: add ecamp3-logging

This adds the EFFK (Elasticsearch, fluentbit, fluentd, Kibana) stack
to the ops folder.
Kibana and Elasticsearch taken from
https://www.digitalocean.com/community/tutorials/how-to-set-up-an-elasticsearch-fluentd-and-kibana-efk-logging-stack-on-kubernetes

.ops/ecamp3-logging/.gitignore

@@ -0,0 +1 @@
+/charts

.ops/ecamp3-logging/.helmignore

@@ -0,0 +1 @@
+/deploy.sh

.ops/ecamp3-logging/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: fluent-operator
+  repository: https://fluent.github.io/helm-charts
+  version: 2.7.0
+digest: sha256:5b2fd2ab5e6e313924a47c8825f2c15a6d288b7125d58c2202a897ee2981213b
+generated: "2024-04-13T23:13:02.185555289+02:00"

.ops/ecamp3-logging/Chart.yaml

@@ -0,0 +1,29 @@
+apiVersion: v2
+name: ecamp3-logging
+description: Helm chart for deploying ecamp3-logging on Kubernetes
+home: https://github.com/ecamp/ecamp3
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+appVersion: 0.1.0
+
+dependencies:
+  - name: fluent-operator
+    version: 2.7.0
+    repository:  https://fluent.github.io/helm-charts

.ops/ecamp3-logging/README.md

@@ -0,0 +1,34 @@
+# ecamp3-logging
+
+This is a helm chart to deploy an EFFK Stack (Elasticsearch, fluentbit, fluentd, Kibana) to a cluster where
+ecamp3 is running.
+
+## Prerequisites
+
+You need to add the fluent helm repository:
+
+```shell
+helm repo add fluent https://fluent.github.io/helm-charts 
+helm repo update
+```
+
+## Provisioning of Kibana Configuration
+
+There are 2 scripts to help create a consistent configuration between
+different clusters. You need [curl](https://github.com/curl/curl) and [jq](https://github.com/jqlang/jq) for the scripts
+to work.
+
+To store the current dashboard, index-pattern and search in [kibana-objects.ndjson](files%2Fkibana%2Fkibana-objects.ndjson),
+you can do the following:
+
+```shell
+kubectl -n ecamp3-logging port-forward services/kibana 5601:5601
+sh files/kibana/dump-kibana-objects.sh
+```
+
+To restore [kibana-objects.ndjson](files%2Fkibana%2Fkibana-objects.ndjson) to a cluster, you can do the follwing:
+
+```shell
+kubectl -n ecamp3-logging port-forward services/kibana 5601:5601
+sh files/kibana/restore-kibana-objects.sh
+```

.ops/ecamp3-logging/deploy.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT_DIR=$(realpath "$(dirname "$0")")
+cd $SCRIPT_DIR
+# to debug: --dry-run --debug
+helm dep build && helm upgrade --install ecamp3-logging --namespace=ecamp3-logging --create-namespace $SCRIPT_DIR

.ops/ecamp3-logging/files/kibana/dump-kibana-objects.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT_DIR=$(realpath "$(dirname "$0")")
+
+KIBANA_HOST=${KIBANA_HOST:-localhost:5601}
+
+curl -X POST $KIBANA_HOST/api/saved_objects/_export \
+      -H 'kbn-xsrf: true' \
+      -H 'Content-Type: application/json' \
+      -d '
+      {
+        "type": [
+          "dashboard", 
+          "index-pattern",
+          "search"
+        ],
+        "excludeExportDetails": true
+      }' \
+      --silent \
+      | jq -S \
+      > $SCRIPT_DIR/kibana-objects.ndjson

.ops/ecamp3-logging/files/kibana/restore-kibana-objects.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+SCRIPT_DIR=$(realpath "$(dirname "$0")")
+
+KIBANA_HOST=${KIBANA_HOST:-localhost:5601}
+
+tmp_file=/tmp/$(uuidgen).ndjson
+
+cat $SCRIPT_DIR/kibana-objects.ndjson | jq -c > $tmp_file 
+
+curl -X POST "$KIBANA_HOST/api/saved_objects/_import?createNewCopies=false&overwrite=true" \
+     -H "kbn-xsrf: true" \
+     --form file=@$tmp_file

.ops/ecamp3-logging/templates/_helpers.tpl

@@ -0,0 +1,20 @@
+{{/*
+Common labels
+*/}}
+{{- define "app.commonLabels" -}}
+chart: {{ .Chart.Name }}
+helm.sh/chart: {{ .Chart.Name }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Common selector labels
+*/}}
+{{- define "app.commonSelectorLabels" -}}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/part-of: {{ .Chart.Name }}
+chart: {{ .Chart.Name }}
+{{- end }}

.ops/ecamp3-logging/templates/elasticsearch/elasticsearch_service.yaml

@@ -0,0 +1,19 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: elasticsearch
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: elasticsearch
+    {{- include "app.commonLabels" . | nindent 4 }}
+    {{- include "app.commonSelectorLabels" . | nindent 4 }}
+spec:
+  selector:
+    app: elasticsearch
+    {{- include "app.commonSelectorLabels" . | nindent 4 }}
+  clusterIP: None
+  ports:
+    - port: 9200
+      name: rest
+    - port: 9300
+      name: inter-node

.ops/ecamp3-logging/templates/elasticsearch/elasticsearch_stateful_set.yaml

@@ -0,0 +1,88 @@
+{{- $resourceLimitInMb := .Values.elasticsearch.elasticNode.resources.limits.memory | replace "Mi" "" | int -}}
+{{- $javaResources := div $resourceLimitInMb 2 -}}
+{{- $minHeapSpace :=  $javaResources -}}
+{{- $maxHeapSpace := $javaResources -}}
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: elasticsearch-cluster
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "app.commonLabels" . | nindent 4 }}
+    {{- include "app.commonSelectorLabels" . | nindent 4 }}
+spec:
+  serviceName: elasticsearch
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elasticsearch
+      {{- include "app.commonSelectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        app: elasticsearch
+        {{- include "app.commonLabels" . | nindent 8 }}
+        {{- include "app.commonSelectorLabels" . | nindent 8 }}
+    spec:
+      containers:
+        - name: elasticsearch
+          image: "docker.elastic.co/elasticsearch/elasticsearch:{{ .Values.elastic.version }}"
+          resources:
+            {{- toYaml .Values.elasticsearch.elasticNode.resources | nindent 12 }}
+          ports:
+            - containerPort: 9200
+              name: rest
+              protocol: TCP
+            - containerPort: 9300
+              name: inter-node
+              protocol: TCP
+          volumeMounts:
+            - name: data
+              mountPath: /usr/share/elasticsearch/data
+          env:
+            - name: cluster.name
+              value: ecamp3-logs
+            - name: xpack.security.enabled
+              value: "false"
+            - name: node.name
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: cluster.initial_master_nodes
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: ES_JAVA_OPTS
+              value: "-Xms{{ $minHeapSpace }}m -Xmx{{ $maxHeapSpace }}m"
+      initContainers:
+        - name: fix-permissions
+          image: busybox
+          command: [ "sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data" ]
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: data
+              mountPath: /usr/share/elasticsearch/data
+        - name: increase-vm-max-map
+          image: busybox
+          command: [ "sysctl", "-w", "vm.max_map_count=262144" ]
+          securityContext:
+            privileged: true
+        - name: increase-fd-ulimit
+          image: busybox
+          command: [ "sh", "-c", "ulimit -n 65536" ]
+          securityContext:
+            privileged: true
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+        labels:
+          app: elasticsearch
+          {{- include "app.commonLabels" . | nindent 10 }}
+          {{- include "app.commonSelectorLabels" . | nindent 10 }}
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        storageClassName: {{ .Values.elasticsearch.persistence.storageClassName }}
+        resources:
+          {{- toYaml .Values.elasticsearch.persistence.resources | nindent 10 }}

.ops/ecamp3-logging/templates/fluentd/fluentd_clusterfilter.yaml

@@ -0,0 +1,50 @@
+# from: https://stackoverflow.com/questions/41991128/parsing-inner-json-inside-fluentd
+apiVersion: fluentd.fluent.io/v1alpha1
+kind: ClusterFilter
+metadata:
+  name: cluster-filters
+  labels:
+    filter.fluentd.fluent.io/enabled: "true"
+    {{- include "app.commonLabels" . | nindent 4 }}
+    {{- include "app.commonSelectorLabels" . | nindent 4 }}
+spec:
+  filters:
+    - customPlugin:
+        config: |
+          <filter **>
+            @type parser
+            key_name log
+            reserve_data true
+            remove_key_name_field false
+            hash_value_field json
+            <parse>
+              @type multi_format
+              <pattern>
+                format json
+              </pattern>
+              <pattern>
+                format none
+              </pattern>
+            </parse>
+          </filter>
+          <filter **>
+            @type record_transformer
+            enable_ruby
+            <record>
+              requestUrl ${if record['json'] && record['json']['httpRequest'] && record['json']['httpRequest']['requestUrl']; then record['json']['httpRequest']['requestUrl']; else nil; end}
+            </record>
+          </filter>
+          <filter **>
+            @type record_transformer
+            enable_ruby
+            <record>
+              escapedUrl ${if record['requestUrl']; then record['requestUrl'].gsub(/\/[0-9a-f]{6,}\/?/, "/{id}/").gsub(/%2F[0-9a-f]{6,}(%2F)?/, "%2F{id}%2F"); else nil; end}
+            </record>
+          </filter>
+          <filter **>
+            @type record_transformer
+            enable_ruby
+            <record>
+              escapedUrlWithoutQuery ${if record['escapedUrl']; then record['escapedUrl'].gsub(/\?.*$/, ""); else nil; end}
+            </record>
+          </filter>

.ops/ecamp3-logging/templates/fluentd/fluentd_output_elasticsearch.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.fluentd.output.elasticsearch.enable -}}
+apiVersion: fluentd.fluent.io/v1alpha1
+kind: ClusterOutput
+metadata:
+  name: fluentd-output-elasticsearch
+  labels:
+    output.fluentd.fluent.io/enabled: "true"
+    {{- include "app.commonLabels" . | nindent 4 }}
+    {{- include "app.commonSelectorLabels" . | nindent 4 }}
+spec:
+  outputs:
+    - elasticsearch:
+        host: {{ .Values.fluentd.output.elasticsearch.host }}
+        port: {{ .Values.fluentd.output.elasticsearch.port }}
+        scheme: {{ .Values.fluentd.output.elasticsearch.scheme }}
+        logstashFormat: true
+        logstashPrefix: {{ .Values.fluentd.output.elasticsearch.logstashPrefix }}
+        sslVerify: {{ .Values.fluentd.output.elasticsearch.sslVerify }}
+      {{- if .Values.fluentd.output.elasticsearch.buffer }}
+      buffer:
+        type: {{ .Values.fluentd.output.elasticsearch.buffer.type }}
+        path: {{ .Values.fluentd.output.elasticsearch.buffer.path }}
+      {{- end }}
+{{- end }}

.ops/ecamp3-logging/templates/kibana/kibana_deployment.yaml

@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kibana
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: kibana
+    {{- include "app.commonLabels" . | nindent 4 }}
+    {{- include "app.commonSelectorLabels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kibana
+      {{- include "app.commonSelectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        app: kibana
+        {{- include "app.commonLabels" . | nindent 8 }}
+        {{- include "app.commonSelectorLabels" . | nindent 8 }}
+    spec:
+      containers:
+        - name: kibana
+          image: "docker.elastic.co/kibana/kibana:{{ .Values.elastic.version }}"
+          resources:
+            {{- toYaml .Values.elasticsearch.elasticNode.resources | nindent 12 }}
+          env:
+            - name: ELASTICSEARCH_URL
+              value: http://elasticsearch:9200
+          ports:
+            - containerPort: 5601

.ops/ecamp3-logging/templates/kibana/kibana_service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kibana
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: kibana
+    {{- include "app.commonLabels" . | nindent 4 }}
+    {{- include "app.commonSelectorLabels" . | nindent 4 }}
+spec:
+  ports:
+    - port: 5601
+  selector:
+    app: kibana
+    {{- include "app.commonSelectorLabels" . | nindent 4 }}