Merge branch 'master' into ali/fix-origin-detection

CHANGELOG.md

@@ -2,9 +2,15 @@
 
 [//]: # (comment: Don't forget to update statsd/telemetry.go:clientVersionTelemetryTag when releasing a new version)
 
-# 5.5.0 / xxxx-xx-xx
+# 5.6.0 / xxxx-xx-xx
 
 
+# 5.5.0 / 2024-01-23
+
+- [FEATURE] Added support for Origin Detection when the container-id is unavailable from inside the application container, only applies to `cgroupv2`. See [#291][].
+- [FEATURE] Added `DistributionSamples` which is similar to `Distribution`, but it lets the client handle the sampling. The rate is passed to the Agent and not used for further sampling. See [#296][].
+- [IMPROVEMENT] The Aggregator no longer allocates memory if there are no tags present. See [#297][].
+
 # 5.4.0 / 2023-12-07
 
 - [FEATURE] Add `WithMaxSamplesPerContext()` option to limit the number of samples per context. See [#292][].
@@ -419,6 +425,11 @@ Below, for reference, the latest improvements made in 07/2016 - 08/2016
 [#262]: https://github.com/DataDog/datadog-go/pull/262
 [#269]: https://github.com/DataDog/datadog-go/pull/269
 [#273]: https://github.com/DataDog/datadog-go/pull/273
+[#283]: https://github.com/DataDog/datadog-go/pull/283
+[#291]: https://github.com/DataDog/datadog-go/pull/291
+[#292]: https://github.com/DataDog/datadog-go/pull/292
+[#296]: https://github.com/DataDog/datadog-go/pull/296
+[#297]: https://github.com/DataDog/datadog-go/pull/297
 [@Aceeri]: https://github.com/Aceeri
 [@Jasrags]: https://github.com/Jasrags
 [@KJTsanaktsidis]: https://github.com/KJTsanaktsidis

statsd/container_linux.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"regexp"
 	"strings"
 	"syscall"
@@ -20,16 +21,25 @@ const (
 	// selfMountinfo is the path to the mountinfo path where we can find the container id in case cgroup namespace is preventing the use of /proc/self/cgroup
 	selfMountInfoPath = "/proc/self/mountinfo"
 
-	// mountsPath is the path to the file listing all the mount points
-	mountsPath = "/proc/mounts"
+	// defaultCgroupMountPath is the default path to the cgroup mount point.
+	defaultCgroupMountPath = "/sys/fs/cgroup"
+
+	// cgroupV1BaseController is the controller used to identify the container-id for cgroup v1
+	cgroupV1BaseController = "memory"
 
 	uuidSource      = "[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}"
 	containerSource = "[0-9a-f]{64}"
 	taskSource      = "[0-9a-f]{32}-\\d+"
 
 	containerdSandboxPrefix = "sandboxes"
-	containerRegexpStr      = "([0-9a-f]{64})|([0-9a-f]{8}(-[0-9a-f]{4}){4}$)"
-	cIDRegexpStr            = `([^\s/]+)/(` + containerRegexpStr + `)/[\S]*hostname`
+
+	// ContainerRegexpStr defines the regexp used to match container IDs
+	// ([0-9a-f]{64}) is standard container id used pretty much everywhere
+	// ([0-9a-f]{32}-[0-9]{10}) is container id used by AWS ECS
+	// ([0-9a-f]{8}(-[0-9a-f]{4}){4}$) is container id used by Garden
+	containerRegexpStr = "([0-9a-f]{64})|([0-9a-f]{32}-[0-9]{10})|([0-9a-f]{8}(-[0-9a-f]{4}){4}$)"
+	// cIDRegexpStr defines the regexp used to match container IDs in /proc/self/mountinfo
+	cIDRegexpStr = `.*/([^\s/]+)/(` + containerRegexpStr + `)/[\S]*hostname`
 
 	// From https://github.com/torvalds/linux/blob/5859a2b1991101d6b978f3feb5325dad39421f29/include/linux/proc_ns.h#L41-L49
 	// Currently, host namespace inode number are hardcoded, which can be used to detect
@@ -45,6 +55,9 @@ var (
 	expContainerID = regexp.MustCompile(fmt.Sprintf(`(%s|%s|%s)(?:.scope)?$`, uuidSource, containerSource, taskSource))
 
 	cIDMountInfoRegexp = regexp.MustCompile(cIDRegexpStr)
+
+	// initContainerID initializes the container ID.
+	initContainerID = internalInitContainerID
 )
 
 // parseContainerID finds the first container ID reading from r and returns it.
@@ -103,55 +116,90 @@ func readMountinfo(path string) string {
 	return parseMountinfo(f)
 }
 
-// isCgroupV1 checks if Cgroup V1 is used
-func isCgroupV1(mountsPath string) bool {
-	f, err := os.Open(mountsPath)
+func isHostCgroupNamespace() bool {
+	fi, err := os.Stat("/proc/self/ns/cgroup")
 	if err != nil {
 		return false
 	}
-	defer f.Close()
 
-	scn := bufio.NewScanner(f)
+	inode := fi.Sys().(*syscall.Stat_t).Ino
+
+	return inode == hostCgroupNamespaceInode
+}
+
+// parseCgroupNodePath parses /proc/self/cgroup and returns a map of controller to its associated cgroup node path.
+func parseCgroupNodePath(r io.Reader) map[string]string {
+	res := make(map[string]string)
+	scn := bufio.NewScanner(r)
 	for scn.Scan() {
 		line := scn.Text()
-
-		tokens := strings.Fields(line)
-		if len(tokens) >= 3 {
-			fsType := tokens[2]
-			if fsType == "cgroup" {
-				return true
-			}
+		tokens := strings.Split(line, ":")
+		if len(tokens) != 3 {
+			continue
+		}
+		if tokens[1] == cgroupV1BaseController || tokens[1] == "" {
+			res[tokens[1]] = tokens[2]
 		}
 	}
-
-	return false
+	return res
 }
 
-func isHostCgroupNamespace() bool {
-	fi, err := os.Stat("/proc/self/ns/cgroup")
+// getCgroupInode returns the cgroup controller inode if it exists otherwise an empty string.
+// The inode is prefixed by "in-" and is used by the agent to retrieve the container ID.
+// For cgroup v1, we use the memory controller.
+func getCgroupInode(cgroupMountPath, procSelfCgroupPath string) string {
+	// Parse /proc/self/cgroup to retrieve the paths to the memory controller (cgroupv1) and the cgroup node (cgroupv2)
+	f, err := os.Open(procSelfCgroupPath)
 	if err != nil {
-		return false
+		return ""
 	}
+	defer f.Close()
+	cgroupControllersPaths := parseCgroupNodePath(f)
+	// Retrieve the cgroup inode from /sys/fs/cgroup+controller+cgroupNodePath
+	for _, controller := range []string{cgroupV1BaseController, ""} {
+		cgroupNodePath, ok := cgroupControllersPaths[controller]
+		if !ok {
+			continue
+		}
+		inode := inodeForPath(path.Join(cgroupMountPath, controller, cgroupNodePath))
+		if inode != "" {
+			return inode
+		}
+	}
+	return ""
+}
 
-	inode := fi.Sys().(*syscall.Stat_t).Ino
-
-	return inode == hostCgroupNamespaceInode
+// inodeForPath returns the inode for the provided path or empty on failure.
+func inodeForPath(path string) string {
+	fi, err := os.Stat(path)
+	if err != nil {
+		return ""
+	}
+	stats, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return ""
+	}
+	return fmt.Sprintf("in-%d", stats.Ino)
 }
 
-// initContainerID initializes the container ID.
+// internalInitContainerID initializes the container ID.
 // It can either be provided by the user or read from cgroups.
-func initContainerID(userProvidedID string, cgroupFallback bool) {
+func internalInitContainerID(userProvidedID string, cgroupFallback bool) {
 	initOnce.Do(func() {
 		if userProvidedID != "" {
 			containerID = userProvidedID
 			return
 		}
 
 		if cgroupFallback {
-			if isCgroupV1(mountsPath) || isHostCgroupNamespace() {
+			isHostCgroupNs := isHostCgroupNamespace()
+			if isHostCgroupNs {
 				containerID = readContainerID(cgroupPath)
-			} else {
-				containerID = readMountinfo(selfMountInfoPath)
+				return
+			}
+			containerID = readMountinfo(selfMountInfoPath)
+			if containerID != "" {
+				containerID = getCgroupInode(defaultCgroupMountPath, cgroupPath)
 			}
 		}
 	})

statsd/container_stub.go

@@ -3,7 +3,7 @@
 
 package statsd
 
-func initContainerID(userProvidedID string, cgroupFallback bool) {
+var initContainerID = func(userProvidedID string, cgroupFallback bool) {
 	initOnce.Do(func() {
 		if userProvidedID != "" {
 			containerID = userProvidedID