From 9f82a1cfc8399d5a4db51e85512224d4351ddc05 Mon Sep 17 00:00:00 2001 From: Mike Dame Date: Mon, 14 Nov 2022 19:43:41 +0000 Subject: [PATCH] Allow extra resource attributes to be parsed to LogEntry labels --- exporter/collector/config.go | 8 +- .../testcases/testcases_logs.go | 12 + ...ogs_apache_access_resource_attributes.json | 1678 +++++++++++++++++ ...e_access_resource_attributes_expected.json | 409 ++++ exporter/collector/logs.go | 36 +- exporter/collector/logs_test.go | 3 +- exporter/collector/metrics.go | 2 +- exporter/collector/monitoredresource.go | 11 +- exporter/collector/monitoredresource_test.go | 4 +- 9 files changed, 2132 insertions(+), 31 deletions(-) create mode 100644 exporter/collector/integrationtest/testdata/fixtures/logs/logs_apache_access_resource_attributes.json create mode 100644 exporter/collector/integrationtest/testdata/fixtures/logs/logs_apache_access_resource_attributes_expected.json diff --git a/exporter/collector/config.go b/exporter/collector/config.go index 1823be1f5..5fb01c7e1 100644 --- a/exporter/collector/config.go +++ b/exporter/collector/config.go @@ -143,8 +143,12 @@ type ResourceFilter struct { type LogConfig struct { // DefaultLogName sets the fallback log name to use when one isn't explicitly set // for a log entry. If unset, logs without a log name will raise an error. - DefaultLogName string `mapstructure:"default_log_name"` - ClientConfig ClientConfig `mapstructure:",squash"` + DefaultLogName string `mapstructure:"default_log_name"` + // ResourceFilters, if provided, provides a list of resource filters. + // Resource attributes matching any filter will be included in LogEntry labels. + // Defaults to empty, which won't include any additional resource labels. + ResourceFilters []ResourceFilter `mapstructure:"resource_filters"` + ClientConfig ClientConfig `mapstructure:",squash"` } // Known metric domains. Note: This is now configurable for advanced usages. diff --git a/exporter/collector/integrationtest/testcases/testcases_logs.go b/exporter/collector/integrationtest/testcases/testcases_logs.go index b402bef12..a444c2f62 100644 --- a/exporter/collector/integrationtest/testcases/testcases_logs.go +++ b/exporter/collector/integrationtest/testcases/testcases_logs.go @@ -14,6 +14,8 @@ package testcases +import "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/collector" + var LogsTestCases = []TestCase{ { Name: "Apache access log with HTTPRequest", @@ -40,4 +42,14 @@ var LogsTestCases = []TestCase{ OTLPInputFixturePath: "testdata/fixtures/logs/logs_span_trace_id.json", ExpectFixturePath: "testdata/fixtures/logs/logs_span_trace_id_expected.json", }, + { + Name: "Logs with additional resource attributes", + OTLPInputFixturePath: "testdata/fixtures/logs/logs_apache_access_resource_attributes.json", + ExpectFixturePath: "testdata/fixtures/logs/logs_apache_access_resource_attributes_expected.json", + ConfigureCollector: func(cfg *collector.Config) { + cfg.LogConfig.ResourceFilters = []collector.ResourceFilter{ + {Prefix: "custom."}, + } + }, + }, } diff --git a/exporter/collector/integrationtest/testdata/fixtures/logs/logs_apache_access_resource_attributes.json b/exporter/collector/integrationtest/testdata/fixtures/logs/logs_apache_access_resource_attributes.json new file mode 100644 index 000000000..ada3e1681 --- /dev/null +++ b/exporter/collector/integrationtest/testdata/fixtures/logs/logs_apache_access_resource_attributes.json @@ -0,0 +1,1678 @@ +{ + "resourceLogs": [ + { + "resource": { + "attributes": [ + { + "key": "cloud.platform", + "value": { + "stringValue": "gcp_compute_engine" + } + }, + { + "key": "custom.foobar", + "value": { + "stringValue": "baz" + } + } + ] + }, + "scopeLogs": [ + { + "scope": {}, + "logRecords": [ + { + "timeUnixNano": "1650984816000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:36 +0800] \"GET / HTTP/1.1\" 200 1247" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "1247" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + }, + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:36 +0800" + } + }, + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984817000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:37 +0800] \"GET /lamp.png HTTP/1.1\" 200 51164" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:37 +0800" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/lamp.png" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "51164" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984817000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:37 +0800] \"GET /favicon.ico HTTP/1.1\" 200 3990" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "requestUrl", + "value": { + "stringValue": "/favicon.ico" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "3990" + } + }, + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:37 +0800" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984831000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:51 +0800] \"GET / HTTP/1.1\" 200 1247" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:51 +0800" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "1247" + } + }, + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984832000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:52 +0800] \"GET / HTTP/1.1\" 200 1247" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:52 +0800" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + }, + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "1247" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984833000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "1247" + } + }, + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:53 +0800" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/" + } + }, + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984833000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:53 +0800" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "1247" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984833000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "status", + "value": { + "stringValue": "200" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "1247" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + }, + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:53 +0800" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984833000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "responseSize", + "value": { + "stringValue": "1247" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:53 +0800" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984834000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:54 +0800" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "1247" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984834000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:54 +0800" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "1247" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984834000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:54 +0800" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "1247" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + }, + { + "key": "user", + "value": { + "stringValue": "-" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984834000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:54 +0800" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "1247" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + }, + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984834000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247" + }, + "attributes": [ + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "responseSize", + "value": { + "stringValue": "1247" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:54 +0800" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + }, + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + } + ] + } + } + }, + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984834000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "referer", + "value": { + "stringValue": "" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "1247" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:53:54 +0800" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984878000000000", + "body": { + "stringValue": "127.0.0.2 - - [26/Apr/2022:22:54:38 +0800] \"GET / HTTP/1.1\" 200 4429" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "referer", + "value": { + "stringValue": "" + } + }, + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "status", + "value": { + "stringValue": "200" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:54:38 +0800" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "/" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "HTTP/1.1" + } + }, + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.2" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "GET" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "4429" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + }, + { + "timeUnixNano": "1650984883000000000", + "body": { + "stringValue": "127.0.0.1 - - [26/Apr/2022:22:54:43 +0800] \"-\" 408 -" + }, + "attributes": [ + { + "key": "log.file.name", + "value": { + "stringValue": "test.log" + } + }, + { + "key": "gcp.http_request", + "value": { + "kvlistValue": { + "values": [ + { + "key": "userAgent", + "value": { + "stringValue": "" + } + }, + { + "key": "user", + "value": { + "stringValue": "-" + } + }, + { + "key": "time", + "value": { + "stringValue": "26/Apr/2022:22:54:43 +0800" + } + }, + { + "key": "responseSize", + "value": { + "stringValue": "-" + } + }, + { + "key": "remoteIp", + "value": { + "stringValue": "127.0.0.1" + } + }, + { + "key": "requestMethod", + "value": { + "stringValue": "-" + } + }, + { + "key": "protocol", + "value": { + "stringValue": "" + } + }, + { + "key": "status", + "value": { + "stringValue": "408" + } + }, + { + "key": "host", + "value": { + "stringValue": "-" + } + }, + { + "key": "requestUrl", + "value": { + "stringValue": "" + } + }, + { + "key": "referer", + "value": { + "stringValue": "" + } + } + ] + } + } + }, + { + "key": "gcp.log_name", + "value": { + "stringValue": "my-log-name-foo" + } + } + ], + "traceId": "", + "spanId": "" + } + ] + } + ] + } + ] +} diff --git a/exporter/collector/integrationtest/testdata/fixtures/logs/logs_apache_access_resource_attributes_expected.json b/exporter/collector/integrationtest/testdata/fixtures/logs/logs_apache_access_resource_attributes_expected.json new file mode 100644 index 000000000..78cf82017 --- /dev/null +++ b/exporter/collector/integrationtest/testdata/fixtures/logs/logs_apache_access_resource_attributes_expected.json @@ -0,0 +1,409 @@ +{ + "writeLogEntriesRequests": [ + { + "entries": [ + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:36 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:37 +0800] \"GET /lamp.png HTTP/1.1\" 200 51164", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/lamp.png", + "status": 200, + "responseSize": "51164", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:37 +0800] \"GET /favicon.ico HTTP/1.1\" 200 3990", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/favicon.ico", + "status": 200, + "responseSize": "3990", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:51 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:52 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.2 - - [26/Apr/2022:22:54:38 +0800] \"GET / HTTP/1.1\" 200 4429", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "4429", + "remoteIp": "127.0.0.2", + "protocol": "HTTP/1.1" + }, + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/my-log-name-foo", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:54:43 +0800] \"-\" 408 -", + "timestamp": "1970-01-01T00:00:00Z", + "labels": { + "custom_foobar": "baz", + "log.file.name": "test.log" + } + } + ], + "partialSuccess": true + } + ] +} diff --git a/exporter/collector/logs.go b/exporter/collector/logs.go index 4564a3852..c744bceb5 100644 --- a/exporter/collector/logs.go +++ b/exporter/collector/logs.go @@ -211,6 +211,7 @@ func (l logMapper) createEntries(ld plog.Logs) ([]*logpb.LogEntry, error) { for i := 0; i < ld.ResourceLogs().Len(); i++ { rl := ld.ResourceLogs().At(i) mr := defaultResourceToMonitoredResource(rl.Resource()) + extraResourceLabels := resourceToLabels(rl.Resource(), false, l.cfg.LogConfig.ResourceFilters) projectID := l.cfg.ProjectID // override project ID with gcp.project.id, if present if projectFromResource, found := rl.Resource().Attributes().Get(resourcemapping.ProjectIDAttributeKey); found { @@ -219,8 +220,7 @@ func (l logMapper) createEntries(ld plog.Logs) ([]*logpb.LogEntry, error) { for j := 0; j < rl.ScopeLogs().Len(); j++ { sl := rl.ScopeLogs().At(j) - instrumentationSource := sl.Scope().Name() - instrumentationVersion := sl.Scope().Version() + logLabels := mergeLogLabels(sl.Scope().Name(), sl.Scope().Version(), extraResourceLabels) for k := 0; k < sl.LogRecords().Len(); k++ { log := sl.LogRecords().At(k) @@ -237,8 +237,7 @@ func (l logMapper) createEntries(ld plog.Logs) ([]*logpb.LogEntry, error) { splitEntries, err := l.logToSplitEntries( log, mr, - instrumentationSource, - instrumentationVersion, + logLabels, time.Now(), logName, projectID, @@ -263,6 +262,18 @@ func (l logMapper) createEntries(ld plog.Logs) ([]*logpb.LogEntry, error) { return entries, multierr.Combine(errors...) } +func mergeLogLabels(instrumentationSource, instrumentationVersion string, resourceLabels map[string]string) map[string]string { + labelsMap := make(map[string]string) + // TODO(damemi): Make overwriting these labels (if they already exist) configurable + if len(instrumentationSource) > 0 { + labelsMap["instrumentation_source"] = instrumentationSource + } + if len(instrumentationVersion) > 0 { + labelsMap["instrumentation_version"] = instrumentationVersion + } + return mergeLabels(labelsMap, resourceLabels) +} + func (l logMapper) logEntryToInternal( entry logging.Entry, logName string, @@ -313,8 +324,7 @@ func (l logMapper) getLogName(log plog.LogRecord) (string, error) { func (l logMapper) logToSplitEntries( log plog.LogRecord, mr *monitoredres.MonitoredResource, - instrumentationSource string, - instrumentationVersion string, + logLabels map[string]string, processTime time.Time, logName string, projectID string, @@ -390,19 +400,7 @@ func (l logMapper) logToSplitEntries( } entry.Severity = severityMapping[severityNumber] - if entry.Labels == nil && - (len(instrumentationSource) > 0 || - len(instrumentationVersion) > 0) { - entry.Labels = make(map[string]string) - } - - // TODO(damemi): Make overwriting these labels (if they already exist) configurable - if len(instrumentationSource) > 0 { - entry.Labels["instrumentation_source"] = instrumentationSource - } - if len(instrumentationVersion) > 0 { - entry.Labels["instrumentation_version"] = instrumentationVersion - } + entry.Labels = logLabels // parse remaining OTel attributes to GCP labels for k, v := range attrsMap { diff --git a/exporter/collector/logs_test.go b/exporter/collector/logs_test.go index 9c060aa6a..b65814311 100644 --- a/exporter/collector/logs_test.go +++ b/exporter/collector/logs_test.go @@ -315,8 +315,7 @@ func TestLogMapping(t *testing.T) { entries, err := mapper.logToSplitEntries( log, mr, - "", - "", + nil, testObservedTime, logName, "fakeprojectid", diff --git a/exporter/collector/metrics.go b/exporter/collector/metrics.go index 8f04fda19..7216d1938 100644 --- a/exporter/collector/metrics.go +++ b/exporter/collector/metrics.go @@ -174,7 +174,7 @@ func (me *MetricsExporter) PushMetrics(ctx context.Context, m pmetric.Metrics) e for i := 0; i < rms.Len(); i++ { rm := rms.At(i) monitoredResource := me.cfg.MetricConfig.MapMonitoredResource(rm.Resource()) - extraResourceLabels := me.mapper.resourceToMetricLabels(rm.Resource()) + extraResourceLabels := resourceToLabels(rm.Resource(), me.cfg.MetricConfig.ServiceResourceLabels, me.cfg.MetricConfig.ResourceFilters) projectID := me.cfg.ProjectID // override project ID with gcp.project.id, if present if projectFromResource, found := rm.Resource().Attributes().Get(resourcemapping.ProjectIDAttributeKey); found { diff --git a/exporter/collector/monitoredresource.go b/exporter/collector/monitoredresource.go index 8106abe8d..a0e07c16f 100644 --- a/exporter/collector/monitoredresource.go +++ b/exporter/collector/monitoredresource.go @@ -53,15 +53,16 @@ func defaultResourceToMonitoredResource(resource pcommon.Resource) *monitoredres return mr } -// resourceToMetricLabels converts the Resource into metric labels needed to uniquely identify -// the timeseries. -func (m *metricMapper) resourceToMetricLabels( +// resourceToLabels converts the Resource attributes into labels. +func resourceToLabels( resource pcommon.Resource, + serviceResourceLabels bool, + resourceFilters []ResourceFilter, ) labels { attrs := pcommon.NewMap() resource.Attributes().Range(func(k string, v pcommon.Value) bool { // Is a service attribute and should be included - if m.cfg.MetricConfig.ServiceResourceLabels && + if serviceResourceLabels && (k == semconv.AttributeServiceName || k == semconv.AttributeServiceNamespace || k == semconv.AttributeServiceInstanceID) { @@ -69,7 +70,7 @@ func (m *metricMapper) resourceToMetricLabels( return true } // Matches one of the resource filters - for _, resourceFilter := range m.cfg.MetricConfig.ResourceFilters { + for _, resourceFilter := range resourceFilters { if strings.HasPrefix(k, resourceFilter.Prefix) { v.CopyTo(attrs.PutEmpty(k)) return true diff --git a/exporter/collector/monitoredresource_test.go b/exporter/collector/monitoredresource_test.go index d7cf50f4b..05f0f066c 100644 --- a/exporter/collector/monitoredresource_test.go +++ b/exporter/collector/monitoredresource_test.go @@ -468,7 +468,7 @@ func TestResourceToMetricLabels(t *testing.T) { for k, v := range test.resourceLabels { r.Attributes().PutStr(k, v) } - extraLabels := mapper.resourceToMetricLabels(r) + extraLabels := resourceToLabels(r, mapper.cfg.MetricConfig.ServiceResourceLabels, mapper.cfg.MetricConfig.ResourceFilters) assert.Equal(t, test.expectExtraLabels, extraLabels) }) } @@ -516,6 +516,6 @@ func TestResourceMetricsToMonitoredResourceUTF8(t *testing.T) { } mr := defaultResourceToMonitoredResource(r) assert.Equal(t, expectMr, mr) - extraLabels := mapper.resourceToMetricLabels(r) + extraLabels := resourceToLabels(r, mapper.cfg.MetricConfig.ServiceResourceLabels, mapper.cfg.MetricConfig.ResourceFilters) assert.Equal(t, expectExtraLabels, extraLabels) }