Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/v1' into knusbaum/bump-sketches-go
Browse files Browse the repository at this point in the history
  • Loading branch information
Julio-Guerra committed Mar 9, 2022
2 parents 6b9c64f + 0c1b381 commit cbf7f04
Show file tree
Hide file tree
Showing 157 changed files with 9,276 additions and 2,631 deletions.
101 changes: 73 additions & 28 deletions .circleci/config.yml
Expand Up @@ -15,11 +15,20 @@ jobs:
description: "go build tags used to compile"
default: ""
type: string
goflags:
description: "extra goflags to pass to go test"
default: ""
type: string
docker:
- image: circleci/golang:1.12
environment:
GOPATH: "/home/circleci/go"
working_directory: /home/circleci/dd-trace-go.v1
environment:
# Go 1.12 doesn't have the proxy turned on by default but we need it to
# fetch some dependencies that aren't handled by fetching directly
# (such as github.com/go-chi/chi/v4@v4.0.0-rc1)
GOPROXY: "https://proxy.golang.org"
steps:
- checkout

Expand Down Expand Up @@ -106,9 +115,14 @@ jobs:
description: "go build tags to use to compile the tests"
default: ""
type: string
goflags:
description: "extra goflags to pass to go test"
default: ""
type: string
resource_class: xlarge
environment: # environment variables for the build itself
TEST_RESULTS: /tmp/test-results # path to where test results will be saved
DD_APPSEC_WAF_TIMEOUT: 1s
<<: *plain-go114

steps:
Expand All @@ -118,16 +132,11 @@ jobs:
- restore_cache: # restores saved cache if no changes are detected since last run
keys:
- go-mod-v5-core-{{ checksum "go.sum.orig" }}
- run:
name: Enforce some dependencies
command: |
# last version compatible with go1.14, needed for testtraceprof
echo 'replace golang.org/x/net => golang.org/x/net d418f374d30933c6c7db22cf349625c295a5afaa' >> go.mod
- run:
name: Testing
command: |
PACKAGE_NAMES=$(go list ./... | grep -v /contrib/ | circleci tests split --split-by=timings --timings-type=classname)
env DD_APPSEC_ENABLED=$(test "<< parameters.build_tags >>" = appsec && echo -n true) gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v -race -coverprofile=coverage.txt -covermode=atomic -tags "<< parameters.build_tags >>"
env DD_APPSEC_ENABLED=$(test "<< parameters.build_tags >>" = appsec && echo -n true) gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v << parameters.goflags >> -coverprofile=coverage.txt -covermode=atomic -tags "<< parameters.build_tags >>"
- save_cache:
key: go-mod-v5-core-{{ checksum "go.sum.orig" }}
Expand All @@ -152,9 +161,14 @@ jobs:
description: "go build tags to use to compile the tests"
default: ""
type: string
goflags:
description: "extra goflags to pass to go test"
default: ""
type: string
resource_class: xlarge
environment: # environment variables for the build itself
TEST_RESULTS: /tmp/test-results # path to where test results will be saved
DD_APPSEC_WAF_TIMEOUT: 1s
working_directory: /home/circleci/dd-trace-go.v1
docker:
- image: circleci/golang:1.14
Expand All @@ -174,6 +188,10 @@ jobs:
POSTGRES_PASSWORD: postgres
POSTGRES_USER: postgres
POSTGRES_DB: postgres
- image: mcr.microsoft.com/mssql/server:2019-latest
environment:
SA_PASSWORD: myPassw0rd
ACCEPT_EULA: Y
- image: consul:1.6.0
- image: redis:3.2
- image: elasticsearch:2
Expand All @@ -199,16 +217,19 @@ jobs:
DD_API_KEY: invalid_key_but_this_is_fine
- image: circleci/mongo:latest-ram
- image: memcached:1.5.9
- image: confluentinc/cp-zookeeper:5.0.0
- image: bitnami/zookeeper:latest
environment:
ALLOW_ANONYMOUS_LOGIN: yes
- image: bitnami/kafka:2
environment:
ZOOKEEPER_CLIENT_PORT: "2181"
- image: confluentinc/cp-kafka:5.0.0
KAFKA_CFG_ZOOKEEPER_CONNECT: localhost:2181
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
KAFKA_CFG_LISTENERS: PLAINTEXT://0.0.0.0:9092
ALLOW_PLAINTEXT_LISTENER: yes
- image: bitnami/kafka:2
environment:
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
KAFKA_CREATE_TOPICS: gotest:1:1
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
KAFKA_CFG_ZOOKEEPER_CONNECT: localhost:2181
command: [kafka-topics.sh, --create, --topic, gosegtest, --bootstrap-server, localhost:9092]

steps:
- checkout
Expand Down Expand Up @@ -237,19 +258,6 @@ jobs:
paths:
- /tmp/librdkafka-v1.3.0

- run:
name: Enforce some dependencies
command: |
go get k8s.io/client-go@v0.17.0
go get k8s.io/apimachinery@v0.17.0
go get cloud.google.com/go/pubsub@v1.6.1
# Temporarily enforce this version. 1.9.0 is incompatible with go < 1.16
go get github.com/hashicorp/consul/api@v1.8.1
# github.com/hashicorp/vault/sdk > v0.2.0 doesn't compile with go1.14
go get github.com/hashicorp/vault/sdk@v0.2.0
# Shopify/sarama > v1.22 doesn't compile with go1.14
go get github.com/Shopify/sarama@v1.22.0
- run:
name: Wait for MySQL
command: dockerize -wait tcp://localhost:3306 -timeout 1m
Expand All @@ -258,6 +266,10 @@ jobs:
name: Wait for Postgres
command: dockerize -wait tcp://localhost:5432 -timeout 1m

- run:
name: Wait for MS SQL Server
command: dockerize -wait tcp://localhost:1433 -timeout 1m

- run:
name: Wait for Redis
command: dockerize -wait tcp://localhost:6379 -timeout 1m
Expand Down Expand Up @@ -294,13 +306,22 @@ jobs:
name: Wait for Consul
command: dockerize -wait http://localhost:8500 -timeout 1m

- run:
name: Go module graph (before)
command: go mod graph

- run:
name: Testing integrations
command: |
PACKAGE_NAMES=$(go list ./contrib/... | grep -v -e grpc.v12 -e google.golang.org/api | circleci tests split --split-by=timings --timings-type=classname)
export DD_APPSEC_ENABLED=$(test "<< parameters.build_tags >>" = appsec && echo -n true)
export INTEGRATION=true
gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v -race -coverprofile=coverage.txt -covermode=atomic -tags "<< parameters.build_tags >>"
gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES -v << parameters.goflags >> -coverprofile=coverage.txt -covermode=atomic -tags "<< parameters.build_tags >>"
- run:
name: Go module graph (after)
command: go mod graph
when: always

- store_artifacts: # upload test summary for display in Artifacts
path: /tmp/test-results
Expand Down Expand Up @@ -361,3 +382,27 @@ workflows:
matrix:
parameters:
build_tags: [ "", "appsec" ]
nightly:
triggers:
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- v1
jobs:
- go1_12-build:
matrix:
parameters:
build_tags: [ "", "appsec" ]
goflags: [ "-race" ]
- test-core:
matrix:
parameters:
build_tags: [ "", "appsec" ]
goflags: [ "-race" ]
- test-contrib:
matrix:
parameters:
build_tags: [ "", "appsec" ]
goflags: [ "-race" ]
10 changes: 9 additions & 1 deletion .github/workflows/system-tests.yml
@@ -1,6 +1,11 @@
name: System Tests

on:
push:
branches:
- v1
tags:
- "**"
pull_request:
branches:
- "**"
Expand All @@ -10,7 +15,6 @@ on:

jobs:
system-tests:
if: ${{ github.event.pull_request.head.repo.full_name == 'DataDog/dd-trace-go' }}
runs-on: ubuntu-latest
strategy:
matrix:
Expand All @@ -19,6 +23,10 @@ jobs:
weblog-variant: net-http
- library: golang
weblog-variant: gorilla
- library: golang
weblog-variant: echo
- library: golang
weblog-variant: chi
fail-fast: false
env:
TEST_LIBRARY: golang
Expand Down
18 changes: 18 additions & 0 deletions CODEOWNERS
Validating CODEOWNERS rules …
@@ -0,0 +1,18 @@
# Note: Later matches take precedence

# default owner
* @DataDog/apm-go

# tracing
/contrib @DataDog/tracing-go
/ddtrace @DataDog/tracing-go
/internal @DataDog/tracing-go

# profiling
/profiler @DataDog/profiling-go
/internal/traceprof @DataDog/profiling-go

# appsec
/appsec @DataDog/appsec-go
/internal/appsec @DataDog/appsec-go
/contrib/**/appsec.go @DataDog/appsec-go
10 changes: 2 additions & 8 deletions CONTRIBUTING.md
Expand Up @@ -40,15 +40,9 @@ Please view our contrib [README.md](contrib/README.md) for information on new in

### Go Modules

This repository currently takes an [idiosyncratic approach](https://github.com/DataDog/dd-trace-go/issues/810) to using Go modules which means that you should not commit modified versions of the `go.mod` or `go.sum` files.
When adding a new dependency, especially for `contrib/` packages, prefer the minimum secure versions of any modules rather than the latest versions. This is to avoid forcing upgrades on downstream users for modules such as `google.golang.org/grpc` which often introduce breaking changes within major versions.

The following git command can be used to permanently ignore modifications to these files:

```
git update-index --assume-unchanged go.*
```

If you need to undo this for any reason, you can run:
This repository used to omit many dependencies from the `go.mod` file due to concerns around version compatibility [(ref)](https://github.com/DataDog/dd-trace-go/issues/810). As such, you may have configured git to ignore changes to `go.mod` and `go.sum`. To undo this, run

```
git update-index --no-assume-unchanged go.*
Expand Down
32 changes: 32 additions & 0 deletions appsec/appsec.go
@@ -0,0 +1,32 @@
// Unless explicitly stated otherwise all files in this repository are licensed
// under the Apache License Version 2.0.
// This product includes software developed at Datadog (https://www.datadoghq.com/).
// Copyright 2022 Datadog, Inc.

// Package appsec provides application security features in the form of SDK
// functions that can be manually called to monitor specific code paths and data.
// Application Security is currently transparently integrated into the APM tracer
// and cannot be used nor started alone at the moment.
// You can read more on how to enable and start Application Security for Go at
// https://docs.datadoghq.com/security_platform/application_security/getting_started/go
package appsec

import (
"golang.org/x/net/context"

"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec"
"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo/instrumentation/httpsec"
)

// MonitorParsedHTTPBody runs the security monitoring rules on the given *parsed*
// HTTP request body. The given context must be the HTTP request context as returned
// by the Context() method of an HTTP request. Calls to this function are ignored if
// AppSec is disabled or the given context is incorrect.
// Note that passing the raw bytes of the HTTP request body is not expected and would
// result in inaccurate attack detection.
func MonitorParsedHTTPBody(ctx context.Context, body interface{}) {
if appsec.Enabled() {
httpsec.MonitorParsedBody(ctx, body)
}
// bonus: use sync.Once to log a debug message once if AppSec is disabled
}
62 changes: 62 additions & 0 deletions appsec/example_test.go
@@ -0,0 +1,62 @@
// Unless explicitly stated otherwise all files in this repository are licensed
// under the Apache License Version 2.0.
// This product includes software developed at Datadog (https://www.datadoghq.com/).
// Copyright 2022 Datadog, Inc.

package appsec_test

import (
"encoding/json"
"io"
"net/http"

"gopkg.in/DataDog/dd-trace-go.v1/appsec"
echotrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/labstack/echo.v4"
httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http"

"github.com/labstack/echo/v4"
)

type parsedBodyType struct {
Value string `json:"value"`
}

func customBodyParser(body io.ReadCloser) (*parsedBodyType, error) {
var parsedBody parsedBodyType
err := json.NewDecoder(body).Decode(&parsedBody)
return &parsedBody, err
}

// Monitor HTTP request parsed body
func ExampleMonitorParsedHTTPBody() {
mux := httptrace.NewServeMux()
mux.HandleFunc("/body", func(w http.ResponseWriter, r *http.Request) {
// Use the SDK to monitor the request's parsed body
body, err := customBodyParser(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
appsec.MonitorParsedHTTPBody(r.Context(), body)
w.Write([]byte("Body monitored using AppSec SDK\n"))
})
http.ListenAndServe(":8080", mux)
}

// Monitor HTTP request parsed body with a framework customized context type
func ExampleMonitorParsedHTTPBody_CustomContext() {
r := echo.New()
r.Use(echotrace.Middleware())
r.POST("/body", func(c echo.Context) (e error) {
req := c.Request()
body, err := customBodyParser(req.Body)
if err != nil {
return c.String(http.StatusInternalServerError, err.Error())
}
// Use the SDK to monitor the request's parsed body
appsec.MonitorParsedHTTPBody(c.Request().Context(), body)
return c.String(http.StatusOK, "Body monitored using AppSec SDK")
})

r.Start(":8080")
}

0 comments on commit cbf7f04

Please sign in to comment.