Collect cloud metadata

apmtest/env.go

@@ -0,0 +1,25 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package apmtest
+
+import "os"
+
+func init() {
+	// Disable cloud metadata sniffing by default in tests.
+	os.Setenv("ELASTIC_APM_CLOUD_PROVIDER", "none")
+}

config.go

@@ -57,6 +57,7 @@ const (
 	envCentralConfig               = "ELASTIC_APM_CENTRAL_CONFIG"
 	envBreakdownMetrics            = "ELASTIC_APM_BREAKDOWN_METRICS"
 	envUseElasticTraceparentHeader = "ELASTIC_APM_USE_ELASTIC_TRACEPARENT_HEADER"
+	envCloudProvider               = "ELASTIC_APM_CLOUD_PROVIDER"
 
 	// NOTE(axw) profiling environment variables are experimental.
 	// They may be removed in a future minor version without being

docs/configuration.asciidoc

@@ -593,3 +593,20 @@ https://www.w3.org/TR/trace-context-1/[W3C Trace Context] specification.
 
 When this setting is `true`, the agent will also add the header `elastic-apm-traceparent`
 for backwards compatibility with older versions of Elastic APM agents.
+
+[float]
+[[config-cloud-provider]]
+==== `ELASTIC_APM_CLOUD_PROVIDER`
+
+[options="header"]
+|============
+| Environment                  | Default  | Example
+| `ELASTIC_APM_CLOUD_PROVIDER` | `"none"` | `"aws"`
+|============
+
+This config value allows you to specify which cloud provider should be assumed
+for metadata collection. By default, the agent will use trial and error to
+automatically collect the cloud metadata.
+
+Valid options are `"none"`, `"auto"`, `"aws"`, `"gcp"`, and `"azure"`
+If this config value is set to `"none"`, then no cloud metadata will be collected.

internal/apmcloudutil/aws.go

@@ -0,0 +1,100 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package apmcloudutil
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"net/http"
+
+	"go.elastic.co/apm/model"
+)
+
+const (
+	ec2TokenURL    = "http://169.254.169.254/latest/api/token"
+	ec2MetadataURL = "http://169.254.169.254/latest/dynamic/instance-identity/document"
+)
+
+// See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html
+func getAWSCloudMetadata(ctx context.Context, client *http.Client, out *model.Cloud) error {
+	token, err := getAWSToken(ctx, client)
+	if err != nil {
+		return err
+	}
+
+	req, err := http.NewRequest("GET", ec2MetadataURL, nil)
+	if err != nil {
+		return err
+	}
+	if token != "" {
+		req.Header.Set("X-aws-ec2-metadata-token", token)
+	}
+
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return errors.New(resp.Status)
+	}
+
+	var ec2Metadata struct {
+		AccountID        string `json:"accountId"`
+		AvailabilityZone string `json:"availabilityZone"`
+		Region           string `json:"region"`
+		InstanceID       string `json:"instanceId"`
+		InstanceType     string `json:"instanceType"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&ec2Metadata); err != nil {
+		return err
+	}
+
+	out.Region = ec2Metadata.Region
+	out.AvailabilityZone = ec2Metadata.AvailabilityZone
+	if ec2Metadata.InstanceID != "" {
+		out.Instance = &model.CloudInstance{ID: ec2Metadata.InstanceID}
+	}
+	if ec2Metadata.InstanceType != "" {
+		out.Machine = &model.CloudMachine{Type: ec2Metadata.InstanceType}
+	}
+	if ec2Metadata.AccountID != "" {
+		out.Account = &model.CloudAccount{ID: ec2Metadata.AccountID}
+	}
+	return nil
+}
+
+func getAWSToken(ctx context.Context, client *http.Client) (string, error) {
+	req, err := http.NewRequest("PUT", ec2TokenURL, nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "300")
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	token, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(token), nil
+}

internal/apmcloudutil/aws_test.go

@@ -0,0 +1,97 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package apmcloudutil
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"go.elastic.co/apm/model"
+)
+
+func TestAWSCloudMetadata(t *testing.T) {
+	srv, client := newAWSMetadataServer()
+	defer srv.Close()
+
+	for _, provider := range []Provider{Auto, AWS} {
+		var out model.Cloud
+		var logger testLogger
+		assert.True(t, provider.getCloudMetadata(context.Background(), client, &logger, &out))
+		assert.Zero(t, logger)
+		assert.Equal(t, model.Cloud{
+			Provider:         "aws",
+			Region:           "us-east-2",
+			AvailabilityZone: "us-east-2a",
+			Instance: &model.CloudInstance{
+				ID: "i-0ae894a7c1c4f2a75",
+			},
+			Machine: &model.CloudMachine{
+				Type: "t2.medium",
+			},
+			Account: &model.CloudAccount{
+				ID: "946960629917",
+			},
+		}, out)
+	}
+}
+
+func newAWSMetadataServer() (*httptest.Server, *http.Client) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/latest/api/token":
+			w.Write([]byte("topsecret"))
+			return
+		case "/latest/dynamic/instance-identity/document":
+			token := r.Header.Get("X-Aws-Ec2-Metadata-Token")
+			if token != "topsecret" {
+				w.WriteHeader(http.StatusBadRequest)
+				w.Write([]byte("invalid token"))
+				return
+			}
+			break
+		default:
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+
+		w.Write([]byte(`{
+    "accountId": "946960629917",
+    "architecture": "x86_64",
+    "availabilityZone": "us-east-2a",
+    "billingProducts": null,
+    "devpayProductCodes": null,
+    "marketplaceProductCodes": null,
+    "imageId": "ami-07c1207a9d40bc3bd",
+    "instanceId": "i-0ae894a7c1c4f2a75",
+    "instanceType": "t2.medium",
+    "kernelId": null,
+    "pendingTime": "2020-06-12T17:46:09Z",
+    "privateIp": "172.31.0.212",
+    "ramdiskId": null,
+    "region": "us-east-2",
+    "version": "2017-09-30"
+}`))
+	}))
+
+	client := &http.Client{Transport: newTargetedRoundTripper("169.254.169.254", srv.Listener.Addr().String())}
+	return srv, client
+}

internal/apmcloudutil/azure.go

@@ -0,0 +1,78 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package apmcloudutil
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+
+	"go.elastic.co/apm/model"
+)
+
+const (
+	azureMetadataURL = "http://169.254.169.254/metadata/instance/compute?api-version=2019-08-15"
+)
+
+// See: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service
+func getAzureCloudMetadata(ctx context.Context, client *http.Client, out *model.Cloud) error {
+	req, err := http.NewRequest("GET", azureMetadataURL, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Metadata", "true")
+
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return errors.New(resp.Status)
+	}
+
+	var azureMetadata struct {
+		Location          string `json:"location"`
+		Name              string `json:"name"`
+		ResourceGroupName string `json:"resourceGroupName"`
+		SubscriptionID    string `json:"subscriptionId"`
+		VMID              string `json:"vmId"`
+		VMSize            string `json:"vmSize"`
+		Zone              string `json:"zone"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&azureMetadata); err != nil {
+		return err
+	}
+
+	out.Region = azureMetadata.Location
+	out.AvailabilityZone = azureMetadata.Zone
+	if azureMetadata.VMID != "" || azureMetadata.Name != "" {
+		out.Instance = &model.CloudInstance{ID: azureMetadata.VMID, Name: azureMetadata.Name}
+	}
+	if azureMetadata.VMSize != "" {
+		out.Machine = &model.CloudMachine{Type: azureMetadata.VMSize}
+	}
+	if azureMetadata.ResourceGroupName != "" {
+		out.Project = &model.CloudProject{Name: azureMetadata.ResourceGroupName}
+	}
+	if azureMetadata.SubscriptionID != "" {
+		out.Account = &model.CloudAccount{ID: azureMetadata.SubscriptionID}
+	}
+	return nil
+}