feat: add signoz_error_index_v2 (#1734)

prashant-shahi · Jul 13, 2022 · f8435c3 · f8435c3
1 parent 47e24e2
commit f8435c3
Show file tree

Hide file tree

Showing 7 changed files with 34 additions and 5 deletions.
diff --git a/examples/signoz-otel-collector.yaml b/examples/signoz-otel-collector.yaml
@@ -54,10 +54,10 @@ exporters:
     resource_to_telemetry_conversion:
       enabled: true
   clickhousetraces:
-    datasource: http://stagingapp.signoz.io:9000/?database=signoz_traces
+    datasource: http://localhost:9000/?database=signoz_traces
     migrations: ../../exporter/clickhousetracesexporter/migrations
   clickhousemetricswrite:
-    endpoint: http://stagingapp.signoz.io:9000/?database=signoz_metrics
+    endpoint: http://localhost:9000/?database=signoz_metrics
     resource_to_telemetry_conversion:
       enabled: true
 

diff --git a/exporter/clickhousetracesexporter/clickhouse_exporter.go b/exporter/clickhousetracesexporter/clickhouse_exporter.go
@@ -16,7 +16,9 @@ package clickhousetracesexporter
 
 import (
 	"context"
+	"crypto/md5"
 	"encoding/json"
+	"fmt"
 	"net/url"
 	"strconv"
 	"strings"
@@ -195,6 +197,8 @@ func populateEvents(events pdata.SpanEventSlice, span *Span) {
 			uuidWithHyphen := uuid.New()
 			uuid := strings.Replace(uuidWithHyphen.String(), "-", "", -1)
 			span.ErrorID = uuid
+			hmd5 := md5.Sum([]byte(span.ServiceName + span.ErrorEvent.AttributeMap["exception.type"] + span.ErrorEvent.AttributeMap["exception.message"]))
+			span.ErrorGroupID = fmt.Sprintf("%x", hmd5)
 		}
 		stringEvent, _ := json.Marshal(event)
 		span.Events = append(span.Events, string(stringEvent))

diff --git a/exporter/clickhousetracesexporter/migrations/000012_init_signoz_error_index_v2.down.sql b/exporter/clickhousetracesexporter/migrations/000012_init_signoz_error_index_v2.down.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS signoz_traces.signoz_error_index_v2
diff --git a/exporter/clickhousetracesexporter/migrations/000012_init_signoz_error_index_v2.up.sql b/exporter/clickhousetracesexporter/migrations/000012_init_signoz_error_index_v2.up.sql
@@ -0,0 +1,15 @@
+CREATE TABLE IF NOT EXISTS signoz_traces.signoz_error_index_v2 (
+  timestamp DateTime64(9) CODEC(DoubleDelta, LZ4),
+  errorID FixedString(32) CODEC(ZSTD(1)),
+  groupID FixedString(32) CODEC(ZSTD(1)),
+  traceID FixedString(32) CODEC(ZSTD(1)),
+  spanID String CODEC(ZSTD(1)),
+  serviceName LowCardinality(String) CODEC(ZSTD(1)),
+  exceptionType LowCardinality(String) CODEC(ZSTD(1)),
+  exceptionMessage String CODEC(ZSTD(1)),
+  exceptionStacktrace String CODEC(ZSTD(1)),
+  exceptionEscaped bool CODEC(T64, ZSTD(1)),
+  INDEX idx_error_id errorID TYPE bloom_filter GRANULARITY 4
+) ENGINE MergeTree()
+PARTITION BY toDate(timestamp)
+ORDER BY (timestamp, groupID)
diff --git a/exporter/clickhousetracesexporter/options.go b/exporter/clickhousetracesexporter/options.go
@@ -31,7 +31,7 @@ const (
 	defaultMigrations        string        = "/migrations"
 	defaultOperationsTable   string        = "signoz_operations"
 	defaultIndexTable        string        = "signoz_index_v2"
-	defaultErrorTable        string        = "signoz_error_index"
+	defaultErrorTable        string        = "signoz_error_index_v2"
 	defaultSpansTable        string        = "signoz_spans"
 	defaultArchiveSpansTable string        = "signoz_archive_spans"
 	defaultWriteBatchDelay   time.Duration = 5 * time.Second

diff --git a/exporter/clickhousetracesexporter/schema-signoz.go b/exporter/clickhousetracesexporter/schema-signoz.go
@@ -63,6 +63,7 @@ type Span struct {
 	Events             []string          `json:"event,omitempty"`
 	ErrorEvent         Event             `json:"errorEvent,omitempty"`
 	ErrorID            string            `json:"errorID,omitempty"`
+	ErrorGroupID       string            `json:"errorGroupID,omitempty"`
 	TagMap             map[string]string `json:"tagMap,omitempty"`
 	HasError           bool              `json:"hasError,omitempty"`
 	TraceModel         TraceModel        `json:"traceModel,omitempty"`

diff --git a/exporter/clickhousetracesexporter/writer.go b/exporter/clickhousetracesexporter/writer.go
@@ -18,6 +18,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -195,14 +196,14 @@ func (w *SpanWriter) writeErrorBatch(batchSpans []*Span) error {
 		err = statement.Append(
 			time.Unix(0, int64(span.ErrorEvent.TimeUnixNano)),
 			span.ErrorID,
+			span.ErrorGroupID,
 			span.TraceId,
 			span.SpanId,
-			span.ParentSpanId,
 			span.ServiceName,
 			span.ErrorEvent.AttributeMap["exception.type"],
 			span.ErrorEvent.AttributeMap["exception.message"],
 			span.ErrorEvent.AttributeMap["exception.stacktrace"],
-			span.ErrorEvent.AttributeMap["exception.escaped"],
+			stringToBool(span.ErrorEvent.AttributeMap["exception.escaped"]),
 		)
 		if err != nil {
 			return err
@@ -212,6 +213,13 @@ func (w *SpanWriter) writeErrorBatch(batchSpans []*Span) error {
 	return statement.Send()
 }
 
+func stringToBool(s string) bool {
+	if strings.ToLower(s) == "true" {
+		return true
+	}
+	return false
+}
+
 func (w *SpanWriter) writeModelBatch(batchSpans []*Span) error {
 	ctx := context.Background()
 	statement, err := w.db.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.%s", w.traceDatabase, w.spansTable))