Skip to content

Commit

Permalink
Merge branch 'main' into patch-1
Browse files Browse the repository at this point in the history
  • Loading branch information
@JDeuce
JDeuce committed Apr 18, 2023
2 parents 1d3c9bf + 58ba73d commit a973f04
Show file tree
Hide file tree
Showing 12 changed files with 296 additions and 109 deletions.
2 changes: 1 addition & 1 deletion contrib/google.golang.org/grpc/appsec_test.go
View file
Expand Up @@ -76,7 +76,7 @@ func TestAppSec(t *testing.T) {
require.NoError(t, err)

// Send a SQLi attack
err = stream.Send(&FixtureRequest{Name: "something UNION SELECT * from users"})
err = stream.Send(&FixtureRequest{Name: "-1' and 1=1 union/* foo */select load_file('/etc/passwd')--"})
require.NoError(t, err)

// Check that the handler was properly called
Expand Down
8 changes: 8 additions & 0 deletions ddtrace/tracer/telemetry_test.go
View file
Expand Up @@ -34,6 +34,14 @@ func TestTelemetryEnabled(t *testing.T) {
telemetry.Check(t, telemetryClient.Configuration, "service", "test-serv")
telemetry.Check(t, telemetryClient.Configuration, "env", "test-env")
telemetry.Check(t, telemetryClient.Configuration, "runtime_metrics_enabled", true)
if metrics, ok := telemetryClient.Metrics[telemetry.NamespaceTracers]; ok {
if initTime, ok := metrics["tracer_init_time"]; ok {
assert.True(t, initTime > 0)
return
}
t.Fatalf("could not find tracer init time in telemetry client metrics")
}
t.Fatalf("could not find tracer namespace in telemetry client metrics")
})
t.Run("profiler start, tracer start", func(t *testing.T) {
telemetryClient := new(telemetrytest.MockClient)
Expand Down
2 changes: 2 additions & 0 deletions ddtrace/tracer/tracer.go
View file
Expand Up @@ -22,6 +22,7 @@ import (
"gopkg.in/DataDog/dd-trace-go.v1/internal/hostname"
"gopkg.in/DataDog/dd-trace-go.v1/internal/log"
"gopkg.in/DataDog/dd-trace-go.v1/internal/remoteconfig"
"gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry"
"gopkg.in/DataDog/dd-trace-go.v1/internal/traceprof"

"github.com/DataDog/datadog-agent/pkg/obfuscate"
Expand Down Expand Up @@ -123,6 +124,7 @@ func Start(opts ...StartOption) {
if internal.Testing {
return // mock tracer active
}
defer telemetry.Time(telemetry.NamespaceTracers, "tracer_init_time", nil, true)()
t := newTracer(opts...)
if !t.config.enabled {
// TODO: instrumentation telemetry client won't get started
Expand Down
12 changes: 6 additions & 6 deletions internal/appsec/ruleset_builder.go
View file
Expand Up @@ -37,12 +37,12 @@ type (
}

ruleEntry struct {
ID string `json:"id"`
Name string `json:"name"`
Tags map[string]interface{} `json:"tags"`
Conditions interface{} `json:"conditions"`
Transformers interface{} `json:"transformers"`
OnMatch []interface{} `json:"on_match,omitempty"`
ID string `json:"id"`
Name interface{} `json:"name,omitempty"`
Tags interface{} `json:"tags"`
Conditions interface{} `json:"conditions"`
Transformers interface{} `json:"transformers"`
OnMatch []interface{} `json:"on_match,omitempty"`
}

rulesOverrideEntry struct {
Expand Down
79 changes: 49 additions & 30 deletions internal/appsec/testdata/blocking.json
View file
Expand Up @@ -55,46 +55,65 @@
]
},
{
"id": "crs-933-200-block",
"name": "PHP Injection Attack: Wrapper scheme detected",
"id": "crs-933-130-block",
"name": "PHP Injection Attack: Global Variables Found",
"tags": {
"type": "php_code_injection",
"crs_id": "933200",
"category": "attack_attempt"
"type": "php_code_injection",
"crs_id": "933130",
"category": "attack_attempt",
"confidence": "1"
},
"conditions": [
{
"parameters": {
"inputs": [
{
"address": "server.request.query"
},
{
"address": "server.request.body"
},
{
"address": "server.request.path_params"
{
"parameters": {
"inputs": [
{
"address": "server.request.query"
},
{
"address": "server.request.body"
},
{
"address": "server.request.path_params"
},
{
"address": "grpc.server.request.message"
}
],
"list": [
"$globals",
"$_cookie",
"$_env",
"$_files",
"$_get",
"$_post",
"$_request",
"$_server",
"$_session",
"$argc",
"$argv",
"$http_\\u200bresponse_\\u200bheader",
"$php_\\u200berrormsg",
"$http_cookie_vars",
"$http_env_vars",
"$http_get_vars",
"$http_post_files",
"$http_post_vars",
"$http_raw_post_data",
"$http_request_vars",
"$http_server_vars"
]
},
{
"address": "grpc.server.request.message"
}
],
"regex": "(?:(?:bzip|ssh)2|z(?:lib|ip)|(?:ph|r)ar|expect|glob|ogg)://",
"options": {
"case_sensitive": true,
"min_length": 6
}
},
"operator": "match_regex"
}
"operator": "phrase_match"
}
],
"transformers": [
"removeNulls"
"lowercase"
],
"on_match": [
"block"
]
},
},
{
"id": "crs-941-110",
"name": "XSS Filter - Category 1: Script Tag Vector",
Expand Down
4 changes: 2 additions & 2 deletions internal/appsec/waf.go
View file
Expand Up @@ -142,9 +142,9 @@ func newHTTPWAFEventListener(handle *waf.Handle, addresses map[string]struct{},
operation.Error = sharedsec.NewUserMonitoringError("Request blocked")
}
}
op.AddSecurityEvents(matches)
log.Debug("appsec: WAF detected a suspicious user: %s", args.UserID)
}
op.AddSecurityEvents(matches)
log.Debug("appsec: WAF detected a suspicious user: %s", args.UserID)
}))
}

Expand Down
2 changes: 1 addition & 1 deletion internal/appsec/waf_test.go
View file
Expand Up @@ -241,7 +241,7 @@ func TestBlocking(t *testing.T) {
const (
ipBlockingRule = "blk-001-001"
userBlockingRule = "blk-001-002"
bodyBlockingRule = "crs-933-200-block"
bodyBlockingRule = "crs-933-130-block"
)

// Start and trace an HTTP server
Expand Down
99 changes: 65 additions & 34 deletions internal/telemetry/client.go
View file
Expand Up @@ -32,7 +32,7 @@ import (
// agent).
type Client interface {
ProductStart(namespace Namespace, configuration []Configuration)
Gauge(namespace Namespace, name string, value float64, tags []string, common bool)
Record(namespace Namespace, metric MetricKind, name string, value float64, tags []string, common bool)
Count(namespace Namespace, name string, value float64, tags []string, common bool)
ApplyOps(opts ...Option)
Stop()
Expand Down Expand Up @@ -259,16 +259,24 @@ func collectDependencies() bool {
return internal.BoolEnv("DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED", true)
}

type metricKind string
// MetricKind specifies the type of metric being reported.
// Metric types mirror Datadog metric types - for a more detailed
// description of metric types, see:
// https://docs.datadoghq.com/metrics/types/?tab=count#metric-types
type MetricKind string

var (
metricKindGauge metricKind = "gauge"
metricKindCount metricKind = "count"
// MetricKindGauge represents a gauge type metric
MetricKindGauge MetricKind = "gauge"
// MetricKindCount represents a count type metric
MetricKindCount MetricKind = "count"
// MetricKindDist represents a distribution type metric
MetricKindDist MetricKind = "distribution"
)

type metric struct {
name string
kind metricKind
kind MetricKind
value float64
// Unix timestamp
ts float64
Expand All @@ -279,7 +287,7 @@ type metric struct {
// TODO: Can there be identically named/tagged metrics with a "common" and "not
// common" variant?

func newmetric(name string, kind metricKind, tags []string, common bool) *metric {
func newMetric(name string, kind MetricKind, tags []string, common bool) *metric {
return &metric{
name: name,
kind: kind,
Expand All @@ -288,13 +296,13 @@ func newmetric(name string, kind metricKind, tags []string, common bool) *metric
}
}

func metricKey(name string, tags []string) string {
return name + strings.Join(tags, "-")
func metricKey(name string, tags []string, kind MetricKind) string {
return name + string(kind) + strings.Join(tags, "-")
}

// Gauge sets the value for a gauge with the given name and tags. If the metric
// is not language-specific, common should be set to true
func (c *client) Gauge(namespace Namespace, name string, value float64, tags []string, common bool) {
// Record sets the value for a gauge or distribution metric type
// with the given name and tags. If the metric is not language-specific, common should be set to true
func (c *client) Record(namespace Namespace, kind MetricKind, name string, value float64, tags []string, common bool) {
c.mu.Lock()
defer c.mu.Unlock()
if !c.started {
Expand All @@ -303,10 +311,10 @@ func (c *client) Gauge(namespace Namespace, name string, value float64, tags []s
if _, ok := c.metrics[namespace]; !ok {
c.metrics[namespace] = map[string]*metric{}
}
key := metricKey(name, tags)
key := metricKey(name, tags, kind)
m, ok := c.metrics[namespace][key]
if !ok {
m = newmetric(name, metricKindGauge, tags, common)
m = newMetric(name, kind, tags, common)
c.metrics[namespace][key] = m
}
m.value = value
Expand All @@ -325,10 +333,10 @@ func (c *client) Count(namespace Namespace, name string, value float64, tags []s
if _, ok := c.metrics[namespace]; !ok {
c.metrics[namespace] = map[string]*metric{}
}
key := metricKey(name, tags)
key := metricKey(name, tags, MetricKindCount)
m, ok := c.metrics[namespace][key]
if !ok {
m = newmetric(name, metricKindCount, tags, common)
m = newMetric(name, MetricKindCount, tags, common)
c.metrics[namespace][key] = m
}
m.value += value
Expand All @@ -340,35 +348,58 @@ func (c *client) Count(namespace Namespace, name string, value float64, tags []s
// sent to the backend. Requests are sent in the background. Must be called
// with c.mu locked
func (c *client) flush() {
submissions := make([]*Request, 0, len(c.requests)+1)
// initialize submissions slice of capacity len(c.requests) + 2
// to hold all the new events, plus two potential metric events
submissions := make([]*Request, 0, len(c.requests)+2)

// copy over requests so we can do the actual submission without holding
// the lock. Zero out the old stuff so we don't leak references
for i, r := range c.requests {
submissions = append(submissions, r)
c.requests[i] = nil
}
c.requests = c.requests[:0]

if c.newMetrics {
c.newMetrics = false
r := c.newRequest(RequestTypeGenerateMetrics)
for namespace := range c.metrics {
payload := &Metrics{
// metrics can either be request type generate-metrics or distributions
dPayload := &DistributionMetrics{
Namespace: namespace,
}
gPayload := &Metrics{
Namespace: namespace,
}
for _, m := range c.metrics[namespace] {
s := Series{
Metric: m.name,
Type: string(m.kind),
Tags: m.tags,
Common: m.common,
if m.kind == MetricKindDist {
dPayload.Series = append(dPayload.Series, DistributionSeries{
Metric: m.name,
Tags: m.tags,
Common: m.common,
Points: []float64{m.value},
})
} else {
gPayload.Series = append(gPayload.Series, Series{
Metric: m.name,
Type: string(m.kind),
Tags: m.tags,
Common: m.common,
Points: [][2]float64{{m.ts, m.value}},
})
}
s.Points = [][2]float64{{m.ts, m.value}}
payload.Series = append(payload.Series, s)
}
r.Body.Payload = payload
submissions = append(submissions, r)
if len(dPayload.Series) > 0 {
distributions := c.newRequest(RequestTypeDistributions)
distributions.Body.Payload = dPayload
submissions = append(submissions, distributions)
}
if len(gPayload.Series) > 0 {
generateMetrics := c.newRequest(RequestTypeGenerateMetrics)
generateMetrics.Body.Payload = gPayload
submissions = append(submissions, generateMetrics)
}
}
}
// copy over requests so we can do the actual submission without holding
// the lock. Zero out the old stuff so we don't leak references
for i, r := range c.requests {
submissions = append(submissions, r)
c.requests[i] = nil
}
c.requests = c.requests[:0]

go func() {
for _, r := range submissions {
Expand Down

0 comments on commit a973f04

Please sign in to comment.