Implement aws.ecs.* resource attributes

CHANGELOG.md

@@ -7,7 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
-## Fixed
+### Added
+
+- `opentelemetry/sdk/extension/aws` Implement [`aws.ecs.*`](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/cloud_provider/aws/ecs.md) and [`aws.logs.*`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/cloud_provider/aws/logs/) resource attributes in the `AwsEcsResourceDetector` detector when the ECS Metadata v4 is available
+  ([#1212](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1212))
+
+### Fixed
 
 - Fix aiopg instrumentation to work with aiopg < 2.0.0
   ([#1473](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1473))

sdk-extension/opentelemetry-sdk-extension-aws/src/opentelemetry/sdk/extension/aws/resource/ecs.py

@@ -12,9 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import logging
 import os
+import re
 import socket
+from urllib.request import Request, urlopen
 
 from opentelemetry.sdk.resources import Resource, ResourceDetector
 from opentelemetry.semconv.resource import (
@@ -58,18 +61,127 @@ def detect(self) -> "Resource":
                     "Failed to get container ID on ECS: %s.", exception
                 )
 
-            return Resource(
+            base_resource = Resource(
                 {
                     ResourceAttributes.CLOUD_PROVIDER: CloudProviderValues.AWS.value,
                     ResourceAttributes.CLOUD_PLATFORM: CloudPlatformValues.AWS_ECS.value,
                     ResourceAttributes.CONTAINER_NAME: socket.gethostname(),
                     ResourceAttributes.CONTAINER_ID: container_id,
                 }
             )
+
+            metadata_v4_endpoint = os.environ.get(
+                "ECS_CONTAINER_METADATA_URI_V4"
+            )
+
+            if not metadata_v4_endpoint:
+                return base_resource
+
+            # Returns https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v4.html#task-metadata-endpoint-v4-response
+            metadata_container = json.loads(_http_get(metadata_v4_endpoint))
+            metadata_task = json.loads(
+                _http_get(f"{metadata_v4_endpoint}/task")
+            )
+
+            task_arn = metadata_task["TaskARN"]
+            base_arn = task_arn[0 : task_arn.rindex(":")]  # noqa
+            cluster: str = metadata_task["Cluster"]
+            cluster_arn = (
+                cluster
+                if cluster.startswith("arn:")
+                else f"{base_arn}:cluster/{cluster}"
+            )
+
+            logs_resource = _get_logs_resource(metadata_container)
+
+            return base_resource.merge(logs_resource).merge(
+                Resource(
+                    {
+                        ResourceAttributes.AWS_ECS_CONTAINER_ARN: metadata_container[
+                            "ContainerARN"
+                        ],
+                        ResourceAttributes.AWS_ECS_CLUSTER_ARN: cluster_arn,
+                        ResourceAttributes.AWS_ECS_LAUNCHTYPE: metadata_task[
+                            "LaunchType"
+                        ].lower(),
+                        ResourceAttributes.AWS_ECS_TASK_ARN: task_arn,
+                        ResourceAttributes.AWS_ECS_TASK_FAMILY: metadata_task[
+                            "Family"
+                        ],
+                        ResourceAttributes.AWS_ECS_TASK_REVISION: metadata_task[
+                            "Revision"
+                        ],
+                    }
+                )
+            )
         # pylint: disable=broad-except
         except Exception as exception:
             if self.raise_on_error:
                 raise exception
 
             logger.warning("%s failed: %s", self.__class__.__name__, exception)
             return Resource.get_empty()
+
+
+def _get_logs_resource(metadata_container):
+    if metadata_container.get("LogDriver") == "awslogs":
+        log_options = metadata_container.get("LogOptions")
+        if log_options:
+            logs_region = log_options.get("awslogs-region")
+            logs_group_name = log_options.get("awslogs-group")
+            logs_stream_name = log_options.get("awslogs-stream")
+
+            container_arn = metadata_container["ContainerARN"]
+
+            if not logs_region:
+                aws_region_match = re.match(
+                    r"arn:aws:ecs:([^:]+):.*", container_arn
+                )
+                if aws_region_match:
+                    logs_region = aws_region_match.group(1)
+
+                else:
+                    logger.warning("Cannot parse AWS region out of ECS ARN")
+
+            # We need to retrieve the account ID from some other ARN to create the
+            # log-group and log-stream ARNs
+            aws_account = None
+            aws_account_match = re.match(
+                r"arn:aws:ecs:[^:]+:([^:]+):.*", container_arn
+            )
+            if aws_account_match:
+                aws_account = aws_account_match.group(1)
+
+            logs_group_arn = None
+            logs_stream_arn = None
+            if logs_region and aws_account:
+                if logs_group_name:
+                    logs_group_arn = f"arn:aws:logs:{logs_region}:{aws_account}:log-group:{logs_group_name}"
+
+                if logs_stream_name:
+                    logs_stream_arn = f"arn:aws:logs:{logs_region}:{aws_account}:log-group:{logs_group_name}:log-stream:{logs_stream_name}"
+
+            return Resource(
+                {
+                    ResourceAttributes.AWS_LOG_GROUP_NAMES: [logs_group_name],
+                    ResourceAttributes.AWS_LOG_GROUP_ARNS: [logs_group_arn],
+                    ResourceAttributes.AWS_LOG_STREAM_NAMES: [
+                        logs_stream_name
+                    ],
+                    ResourceAttributes.AWS_LOG_STREAM_ARNS: [logs_stream_arn],
+                }
+            )
+
+        logger.warning(
+            "The metadata endpoint v4 has returned 'awslogs' as 'LogDriver', but there is no 'LogOptions' data"
+        )
+
+    return Resource.get_empty()
+
+
+def _http_get(url):
+    with urlopen(
+        Request(url, method="GET"),
+        timeout=5,
+    ) as response:
+        return response.read().decode("utf-8")

sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-container-ec2.json

@@ -0,0 +1,44 @@
+{
+    "DockerId": "ea32192c8553fbff06c9340478a2ff089b2bb5646fb718b4ee206641c9086d66",
+    "Name": "curl",
+    "DockerName": "ecs-curltest-24-curl-cca48e8dcadd97805600",
+    "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
+    "ImageID": "sha256:d691691e9652791a60114e67b365688d20d19940dde7c4736ea30e660d8d3553",
+    "Labels": {
+        "com.amazonaws.ecs.cluster": "default",
+        "com.amazonaws.ecs.container-name": "curl",
+        "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/8f03e41243824aea923aca126495f665",
+        "com.amazonaws.ecs.task-definition-family": "curltest",
+        "com.amazonaws.ecs.task-definition-version": "24"
+    },
+    "DesiredStatus": "RUNNING",
+    "KnownStatus": "RUNNING",
+    "Limits": {
+        "CPU": 10,
+        "Memory": 128
+    },
+    "CreatedAt": "2020-10-02T00:15:07.620912337Z",
+    "StartedAt": "2020-10-02T00:15:08.062559351Z",
+    "Type": "NORMAL",
+    "LogDriver": "awslogs",
+    "LogOptions": {
+        "awslogs-create-group": "true",
+        "awslogs-group": "/ecs/metadata",
+        "awslogs-region": "us-west-2",
+        "awslogs-stream": "ecs/curl/8f03e41243824aea923aca126495f665"
+    },
+    "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/0206b271-b33f-47ab-86c6-a0ba208a70a9",
+    "Networks": [
+        {
+            "NetworkMode": "awsvpc",
+            "IPv4Addresses": [
+                "10.0.2.100"
+            ],
+            "AttachmentIndex": 0,
+            "MACAddress": "0e:9e:32:c7:48:85",
+            "IPv4SubnetCIDRBlock": "10.0.2.0/24",
+            "PrivateDNSName": "ip-10-0-2-100.us-west-2.compute.internal",
+            "SubnetGatewayIpv4Address": "10.0.2.1/24"
+        }
+    ]
+}

sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-container-fargate.json

@@ -0,0 +1,50 @@
+{
+    "DockerId": "cd189a933e5849daa93386466019ab50-2495160603",
+    "Name": "curl",
+    "DockerName": "curl",
+    "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
+    "ImageID": "sha256:25f3695bedfb454a50f12d127839a68ad3caf91e451c1da073db34c542c4d2cb",
+    "Labels": {
+        "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default",
+        "com.amazonaws.ecs.container-name": "curl",
+        "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/cd189a933e5849daa93386466019ab50",
+        "com.amazonaws.ecs.task-definition-family": "curltest",
+        "com.amazonaws.ecs.task-definition-version": "2"
+    },
+    "DesiredStatus": "RUNNING",
+    "KnownStatus": "RUNNING",
+    "Limits": {
+        "CPU": 10,
+        "Memory": 128
+    },
+    "CreatedAt": "2020-10-08T20:09:11.44527186Z",
+    "StartedAt": "2020-10-08T20:09:11.44527186Z",
+    "Type": "NORMAL",
+    "Networks": [
+        {
+            "NetworkMode": "awsvpc",
+            "IPv4Addresses": [
+                "192.0.2.3"
+            ],
+            "AttachmentIndex": 0,
+            "MACAddress": "0a:de:f6:10:51:e5",
+            "IPv4SubnetCIDRBlock": "192.0.2.0/24",
+            "DomainNameServers": [
+                "192.0.2.2"
+            ],
+            "DomainNameSearchList": [
+                "us-west-2.compute.internal"
+            ],
+            "PrivateDNSName": "ip-10-0-0-222.us-west-2.compute.internal",
+            "SubnetGatewayIpv4Address": "192.0.2.0/24"
+        }
+    ],
+    "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/05966557-f16c-49cb-9352-24b3a0dcd0e1",
+    "LogOptions": {
+        "awslogs-create-group": "true",
+        "awslogs-group": "/ecs/containerlogs",
+        "awslogs-region": "us-west-2",
+        "awslogs-stream": "ecs/curl/cd189a933e5849daa93386466019ab50"
+    },
+    "LogDriver": "awslogs"
+}

sdk-extension/opentelemetry-sdk-extension-aws/tests/resource/ecs/metadatav4-response-task-ec2.json

@@ -0,0 +1,94 @@
+{
+    "Cluster": "default",
+    "TaskARN": "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c",
+    "Family": "curltest",
+    "Revision": "26",
+    "DesiredStatus": "RUNNING",
+    "KnownStatus": "RUNNING",
+    "PullStartedAt": "2020-10-02T00:43:06.202617438Z",
+    "PullStoppedAt": "2020-10-02T00:43:06.31288465Z",
+    "AvailabilityZone": "us-west-2d",
+    "LaunchType": "EC2",
+    "Containers": [
+        {
+            "DockerId": "598cba581fe3f939459eaba1e071d5c93bb2c49b7d1ba7db6bb19deeb70d8e38",
+            "Name": "~internal~ecs~pause",
+            "DockerName": "ecs-curltest-26-internalecspause-e292d586b6f9dade4a00",
+            "Image": "amazon/amazon-ecs-pause:0.1.0",
+            "ImageID": "",
+            "Labels": {
+                "com.amazonaws.ecs.cluster": "default",
+                "com.amazonaws.ecs.container-name": "~internal~ecs~pause",
+                "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c",
+                "com.amazonaws.ecs.task-definition-family": "curltest",
+                "com.amazonaws.ecs.task-definition-version": "26"
+            },
+            "DesiredStatus": "RESOURCES_PROVISIONED",
+            "KnownStatus": "RESOURCES_PROVISIONED",
+            "Limits": {
+                "CPU": 0,
+                "Memory": 0
+            },
+            "CreatedAt": "2020-10-02T00:43:05.602352471Z",
+            "StartedAt": "2020-10-02T00:43:06.076707576Z",
+            "Type": "CNI_PAUSE",
+            "Networks": [
+                {
+                    "NetworkMode": "awsvpc",
+                    "IPv4Addresses": [
+                        "10.0.2.61"
+                    ],
+                    "AttachmentIndex": 0,
+                    "MACAddress": "0e:10:e2:01:bd:91",
+                    "IPv4SubnetCIDRBlock": "10.0.2.0/24",
+                    "PrivateDNSName": "ip-10-0-2-61.us-west-2.compute.internal",
+                    "SubnetGatewayIpv4Address": "10.0.2.1/24"
+                }
+            ]
+        },
+        {
+            "DockerId": "ee08638adaaf009d78c248913f629e38299471d45fe7dc944d1039077e3424ca",
+            "Name": "curl",
+            "DockerName": "ecs-curltest-26-curl-a0e7dba5aca6d8cb2e00",
+            "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
+            "ImageID": "sha256:d691691e9652791a60114e67b365688d20d19940dde7c4736ea30e660d8d3553",
+            "Labels": {
+                "com.amazonaws.ecs.cluster": "default",
+                "com.amazonaws.ecs.container-name": "curl",
+                "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/158d1c8083dd49d6b527399fd6414f5c",
+                "com.amazonaws.ecs.task-definition-family": "curltest",
+                "com.amazonaws.ecs.task-definition-version": "26"
+            },
+            "DesiredStatus": "RUNNING",
+            "KnownStatus": "RUNNING",
+            "Limits": {
+                "CPU": 10,
+                "Memory": 128
+            },
+            "CreatedAt": "2020-10-02T00:43:06.326590752Z",
+            "StartedAt": "2020-10-02T00:43:06.767535449Z",
+            "Type": "NORMAL",
+            "LogDriver": "awslogs",
+            "LogOptions": {
+                "awslogs-create-group": "true",
+                "awslogs-group": "/ecs/metadata",
+                "awslogs-region": "us-west-2",
+                "awslogs-stream": "ecs/curl/158d1c8083dd49d6b527399fd6414f5c"
+            },
+            "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/abb51bdd-11b4-467f-8f6c-adcfe1fe059d",
+            "Networks": [
+                {
+                    "NetworkMode": "awsvpc",
+                    "IPv4Addresses": [
+                        "10.0.2.61"
+                    ],
+                    "AttachmentIndex": 0,
+                    "MACAddress": "0e:10:e2:01:bd:91",
+                    "IPv4SubnetCIDRBlock": "10.0.2.0/24",
+                    "PrivateDNSName": "ip-10-0-2-61.us-west-2.compute.internal",
+                    "SubnetGatewayIpv4Address": "10.0.2.1/24"
+                }
+            ]
+        }
+    ]
+}