internal: appsec: switch appsec event tag from json to messagepack using meta_struct

contrib/99designs/gqlgen/appsec_test.go

@@ -121,27 +121,18 @@ func TestAppSec(t *testing.T) {
 				require.Equal(t, 1, spans[len(spans)-1].Tag("_dd.appsec.enabled"))
 
 				events := make(map[string]string)
-				type ddAppsecJSON struct {
-					Triggers []struct {
-						Rule struct {
-							ID string `json:"id"`
-						} `json:"rule"`
-					} `json:"triggers"`
-				}
 
 				// Search for AppSec events in the set of spans
 				for _, span := range spans {
-					jsonText, ok := span.Tag("_dd.appsec.json").(string)
-					if !ok || jsonText == "" {
+					if span.Tag("_dd.appsec.json") == nil {
 						continue
 					}
-					var parsed ddAppsecJSON
-					err := json.Unmarshal([]byte(jsonText), &parsed)
-					require.NoError(t, err)
 
-					require.Len(t, parsed.Triggers, 1, "expected exactly 1 trigger on %s span", span.OperationName())
-					ruleID := parsed.Triggers[0].Rule.ID
-					_, duplicate := events[ruleID]
+					tag := span.Tag("_dd.appsec.json").(map[string][]any)
+
+					require.Len(t, tag["triggers"], 1, "expected exactly 1 trigger on %s span", span.OperationName())
+					ruleID := tag["triggers"][0].(map[string]any)["rule"].(map[string]any)["id"].(string)
+					_, duplicate := tag[ruleID]
 					require.False(t, duplicate, "found duplicated hit for rule %s", ruleID)
 					var origin string
 					switch name := span.OperationName(); name {

contrib/graph-gophers/graphql-go/appsec_test.go

@@ -105,24 +105,17 @@ func TestAppSec(t *testing.T) {
 				// The last finished span (which is GraphQL entry) should have the "_dd.appsec.enabled" tag.
 				require.Equal(t, 1, spans[len(spans)-1].Tag("_dd.appsec.enabled"))
 				events := make(map[string]string)
-				type ddAppsecJSON struct {
-					Triggers []struct {
-						Rule struct {
-							ID string `json:"id"`
-						} `json:"rule"`
-					} `json:"triggers"`
-				}
+
 				// Search for AppSec events in the set of spans
 				for _, span := range spans {
-					jsonText, ok := span.Tag("_dd.appsec.json").(string)
-					if !ok || jsonText == "" {
+					if span.Tag("_dd.appsec.json") == nil {
 						continue
 					}
-					var parsed ddAppsecJSON
-					err := json.Unmarshal([]byte(jsonText), &parsed)
-					require.NoError(t, err)
-					require.Len(t, parsed.Triggers, 1, "expected exactly 1 trigger on %s span", span.OperationName())
-					ruleID := parsed.Triggers[0].Rule.ID
+
+					tag := span.Tag("_dd.appsec.json").(map[string][]any)
+
+					require.Len(t, tag["triggers"], 1, "expected exactly 1 trigger on %s span", span.OperationName())
+					ruleID := tag["triggers"][0].(map[string]any)["rule"].(map[string]any)["id"].(string)
 					_, duplicate := events[ruleID]
 					require.False(t, duplicate, "found duplicated hit for rule %s", ruleID)
 					var origin string

ddtrace/ddtrace.go

@@ -57,6 +57,10 @@ type Tracer interface {
 // spans in a request, buffer and submit them to the server.
 type Span interface {
 	// SetTag sets a key/value pair as metadata on the span.
+	// The value can be of any trivial type that can be encoded as a string or support the fmt.Stringer interface.
+	// Structs implementing the msgp.Marshaler interface are also supported using msgp.AppendIntf.
+	// It can also be a map[string]any composed recursively of trivial types or msgp.Encodable types.
+	// Cf. https://pkg.go.dev/github.com/tinylib/msgp/msgp#AppendIntf
 	SetTag(key string, value interface{})
 
 	// SetOperationName sets the operation name for this span. An operation name should be

ddtrace/tracer/meta_struct.go

@@ -0,0 +1,101 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016 Datadog, Inc.
+
+package tracer
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+var (
+	_ msgp.Encodable = (*metaStructMap)(nil)
+	_ msgp.Decodable = (*metaStructMap)(nil)
+	_ msgp.Sizer     = (*metaStructMap)(nil)
+)
+
+// metaStructMap is a map of string to any of metadata embedded in each span
+// We export special messagepack methods to handle the encoding and decoding of the map
+// Because the agent expects the metadata to be a map of string to byte array, we have to create sub-messages of messagepack for each value
+type metaStructMap map[string]any
+
+// EncodeMsg transforms the map[string]any into a map[string][]byte agent-side (which is parsed back into a map[string]any in the backend)
+func (m *metaStructMap) EncodeMsg(en *msgp.Writer) error {
+	if m == nil {
+		return en.WriteNil()
+	}
+
+	err := en.WriteMapHeader(uint32(len(*m)))
+	if err != nil {
+		return msgp.WrapError(err, "MetaStruct")
+	}
+
+	for key, value := range *m {
+		err = en.WriteString(key)
+		if err != nil {
+			return msgp.WrapError(err, "MetaStruct")
+		}
+
+		// Wrap the encoded value in a byte array that will not be parsed by the agent
+		msg, err := msgp.AppendIntf(nil, value)
+		if err != nil {
+			return err
+		}
+		if err != nil {
+			return msgp.WrapError(err, "MetaStruct", key)
+		}
+
+		err = en.WriteBytes(msg)
+		if err != nil {
+			return msgp.WrapError(err, "MetaStruct", key)
+		}
+	}
+
+	return nil
+}
+
+// DecodeMsg transforms the map[string][]byte agent-side into a map[string]any where values are sub-messages in messagepack
+func (m *metaStructMap) DecodeMsg(de *msgp.Reader) error {
+	err := de.ReadNil()
+	if err == nil {
+		*m = nil
+		return nil
+	}
+
+	header, err := de.ReadMapHeader()
+	if err != nil {
+		return msgp.WrapError(err, "MetaStruct")
+	}
+
+	*m = make(metaStructMap, header)
+	for i := uint32(0); i < header; i++ {
+		var key string
+		key, err = de.ReadString()
+		if err != nil {
+			return msgp.WrapError(err, "MetaStruct")
+		}
+
+		subMsg, err := de.ReadBytes(nil)
+		if err != nil {
+			return msgp.WrapError(err, "MetaStruct", key)
+		}
+
+		(*m)[key], _, err = msgp.ReadIntfBytes(subMsg)
+		if err != nil {
+			return msgp.WrapError(err, "MetaStruct", key)
+		}
+	}
+
+	return nil
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (m *metaStructMap) Msgsize() int {
+	size := msgp.MapHeaderSize
+	for key, value := range *m {
+		size += msgp.StringPrefixSize + len(key)
+		size += msgp.BytesPrefixSize + msgp.GuessSize(value)
+	}
+	return size
+}

ddtrace/tracer/meta_struct_test.go

@@ -0,0 +1,138 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016 Datadog, Inc.
+
+package tracer
+
+import (
+	"bytes"
+	"errors"
+	"github.com/stretchr/testify/require"
+	"reflect"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMetaStructMap_EncodeDecode(t *testing.T) {
+	// Create a sample metaStructMap
+	meta := map[string]any{
+		"key1": "value1",
+		"key2": "value2",
+	}
+
+	for _, tc := range []struct {
+		name          string
+		input         metaStructMap
+		encodingError error
+		output        map[string]any
+		decodingError error
+	}{
+		{
+			name:   "empty",
+			input:  metaStructMap{},
+			output: map[string]any{},
+		},
+		{
+			name:   "non-empty",
+			input:  meta,
+			output: meta,
+		},
+		{
+			name:   "nil",
+			input:  nil,
+			output: nil,
+		},
+		{
+			name: "nested-map",
+			input: metaStructMap{
+				"key": map[string]any{
+					"nested-key": "nested-value",
+				},
+			},
+			output: map[string]any{
+				"key": map[string]any{
+					"nested-key": "nested-value",
+				},
+			},
+		},
+		{
+			name: "nested-slice",
+			input: metaStructMap{
+				"key": []any{
+					"nested-value",
+				},
+			},
+			output: map[string]any{
+				"key": []any{
+					"nested-value",
+				},
+			},
+		},
+		{
+			name: "encoding-error/nested-chan",
+			input: metaStructMap{
+				"key": map[string]any{
+					"nested-key": make(chan struct{}),
+				},
+			},
+			encodingError: errors.New("msgp: type \"chan struct {}\" not supported"),
+		},
+		{
+			name: "encoding-error/channel",
+			input: metaStructMap{
+				"key": make(chan struct{}),
+			},
+			encodingError: errors.New("msgp: type \"chan struct {}\" not supported"),
+		},
+		{
+			name: "encoding-error/func",
+			input: metaStructMap{
+				"key": func() {},
+			},
+			encodingError: errors.New("msgp: type \"func()\" not supported"),
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			// Encode the metaStructMap
+			var buf bytes.Buffer
+			enc := msgp.NewWriter(&buf)
+			err := tc.input.EncodeMsg(enc)
+			if tc.encodingError != nil {
+				require.EqualError(t, err, tc.encodingError.Error())
+				return
+			}
+
+			require.NoError(t, err)
+			require.NoError(t, enc.Flush())
+
+			// Decode the encoded metaStructMap
+			dec := msgp.NewReader(bytes.NewReader(buf.Bytes()))
+			var decodedMeta metaStructMap
+			err = decodedMeta.DecodeMsg(dec)
+			if tc.decodingError != nil {
+				require.EqualError(t, err, tc.decodingError.Error())
+				return
+			}
+
+			require.NoError(t, err)
+
+			// Compare the original and decoded metaStructMap
+			compareMetaStructMaps(t, tc.output, decodedMeta)
+		})
+	}
+}
+
+func compareMetaStructMaps(t *testing.T, m1, m2 metaStructMap) {
+	require.Equal(t, len(m1), len(m2), "mismatched map lengths: %d != %d", len(m1), len(m2))
+
+	for k, v := range m1 {
+		m2v, ok := m2[k]
+		require.Truef(t, ok, "missing key %s", k)
+
+		if !reflect.DeepEqual(v, m2v) {
+			require.Fail(t, "compareMetaStructMaps", "mismatched key %s: expected '%v' but got '%v'", k, v, m2v)
+		}
+	}
+}

ddtrace/tracer/span.go

@@ -64,19 +64,20 @@ type errorConfig struct {
 type span struct {
 	sync.RWMutex `msg:"-"` // all fields are protected by this RWMutex
 
-	Name      string             `msg:"name"`              // operation name
-	Service   string             `msg:"service"`           // service name (i.e. "grpc.server", "http.request")
-	Resource  string             `msg:"resource"`          // resource name (i.e. "/user?id=123", "SELECT * FROM users")
-	Type      string             `msg:"type"`              // protocol associated with the span (i.e. "web", "db", "cache")
-	Start     int64              `msg:"start"`             // span start time expressed in nanoseconds since epoch
-	Duration  int64              `msg:"duration"`          // duration of the span expressed in nanoseconds
-	Meta      map[string]string  `msg:"meta,omitempty"`    // arbitrary map of metadata
-	Metrics   map[string]float64 `msg:"metrics,omitempty"` // arbitrary map of numeric metrics
-	SpanID    uint64             `msg:"span_id"`           // identifier of this span
-	TraceID   uint64             `msg:"trace_id"`          // lower 64-bits of the root span identifier
-	ParentID  uint64             `msg:"parent_id"`         // identifier of the span's direct parent
-	Error     int32              `msg:"error"`             // error status of the span; 0 means no errors
-	SpanLinks []ddtrace.SpanLink `msg:"span_links"`        // links to other spans
+	Name       string             `msg:"name"`                  // operation name
+	Service    string             `msg:"service"`               // service name (i.e. "grpc.server", "http.request")
+	Resource   string             `msg:"resource"`              // resource name (i.e. "/user?id=123", "SELECT * FROM users")
+	Type       string             `msg:"type"`                  // protocol associated with the span (i.e. "web", "db", "cache")
+	Start      int64              `msg:"start"`                 // span start time expressed in nanoseconds since epoch
+	Duration   int64              `msg:"duration"`              // duration of the span expressed in nanoseconds
+	Meta       map[string]string  `msg:"meta,omitempty"`        // arbitrary map of metadata
+	MetaStruct metaStructMap      `msg:"meta_struct,omitempty"` // arbitrary map of metadata with structured values
+	Metrics    map[string]float64 `msg:"metrics,omitempty"`     // arbitrary map of numeric metrics
+	SpanID     uint64             `msg:"span_id"`               // identifier of this span
+	TraceID    uint64             `msg:"trace_id"`              // lower 64-bits of the root span identifier
+	ParentID   uint64             `msg:"parent_id"`             // identifier of the span's direct parent
+	Error      int32              `msg:"error"`                 // error status of the span; 0 means no errors
+	SpanLinks  []ddtrace.SpanLink `msg:"span_links"`            // links to other spans
 
 	goExecTraced bool         `msg:"-"`
 	noDebugStack bool         `msg:"-"` // disables debug stack traces
@@ -162,6 +163,13 @@ func (s *span) SetTag(key string, value interface{}) {
 		s.setMeta(key, v.String())
 		return
 	}
+
+	// Can be sent as messagepack in `meta_struct` instead of `meta`
+	if _, ok := value.(msgp.Marshaler); ok {
+		s.setMetaStruct(key, value)
+		return
+	}
+
 	if value != nil {
 		// Arrays will be translated to dot notation. e.g.
 		// {"myarr.0": "foo", "myarr.1": "bar"}
@@ -179,6 +187,10 @@ func (s *span) SetTag(key string, value interface{}) {
 				}
 			}
 			return
+		case reflect.Map:
+			// `meta_struct` does not support slice as a top-level value.
+			s.setMetaStruct(key, value)
+			return
 		}
 	}
 	// not numeric, not a string, not a fmt.Stringer, not a bool, and not an error
@@ -390,6 +402,13 @@ func (s *span) setMeta(key, v string) {
 	}
 }
 
+func (s *span) setMetaStruct(key string, v any) {
+	if s.MetaStruct == nil {
+		s.MetaStruct = make(metaStructMap, 1)
+	}
+	s.MetaStruct[key] = v
+}
+
 // setTagBool sets a boolean tag on the span.
 func (s *span) setTagBool(key string, v bool) {
 	switch key {