add tracing to client-go requests

staging/src/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -122,14 +122,24 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	if err := o.CoreAPI.ApplyTo(config); err != nil {
 		return err
 	}
+	if err := o.EgressSelector.ApplyTo(&config.Config); err != nil {
+		return err
+	}
+	if feature.DefaultFeatureGate.Enabled(features.APIServerTracing) {
+		// Apply tracing config right after CoreAPI and EgressSelector so that the rest config is
+		// updated with a TracerProvider before anyone tries to use it.
+		if err := o.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil {
+			return err
+		}
+		if config.TracerProvider != nil {
+			config.ClientConfig.TracerProvider = config.TracerProvider
+		}
+	}
 	if initializers, err := o.ExtraAdmissionInitializers(config); err != nil {
 		return err
 	} else if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, config.ClientConfig, o.FeatureGate, initializers...); err != nil {
 		return err
 	}
-	if err := o.EgressSelector.ApplyTo(&config.Config); err != nil {
-		return err
-	}
 	if feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) {
 		if config.ClientConfig != nil {
 			if config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight <= 0 {
@@ -146,11 +156,6 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 			klog.Warningf("Neither kubeconfig is provided nor service-account is mounted, so APIPriorityAndFairness will be disabled")
 		}
 	}
-	if feature.DefaultFeatureGate.Enabled(features.APIServerTracing) {
-		if err := o.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil {
-			return err
-		}
-	}
 	return nil
 }
 

staging/src/k8s.io/apiserver/pkg/server/options/tracing.go

@@ -106,6 +106,7 @@ func (o *TracingOptions) ApplyTo(es *egressselector.EgressSelector, c *server.Co
 	}
 	tp := traces.NewProvider(context.Background(), sampler, resourceOpts, opts...)
 	c.TracerProvider = &tp
+	c.LoopbackClientConfig.TracerProvider = &tp
 	return nil
 }
 

staging/src/k8s.io/client-go/go.mod

@@ -24,6 +24,9 @@ require (
 	github.com/peterbourgon/diskv v2.0.1+incompatible
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0
+	go.opentelemetry.io/otel v0.20.0
+	go.opentelemetry.io/otel/trace v0.20.0
 	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect
 	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d

staging/src/k8s.io/client-go/rest/config.go

@@ -30,6 +30,8 @@ import (
 	"strings"
 	"time"
 
+	"go.opentelemetry.io/otel/trace"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -144,6 +146,10 @@ type Config struct {
 	// Version forces a specific version to be used (if registered)
 	// Do we need this?
 	// Version string
+
+	// TracerProvider can provide a tracer, which records distributed tracing spans.
+	// If unspecified, no spans will be recorded.
+	TracerProvider *trace.TracerProvider
 }
 
 var _ fmt.Stringer = new(Config)

staging/src/k8s.io/client-go/rest/transport.go

@@ -88,6 +88,7 @@ func (c *Config) TransportConfig() (*transport.Config, error) {
 		},
 		Dial:  c.Dial,
 		Proxy: c.Proxy,
+		TracerProvider: c.TracerProvider,
 	}
 
 	if c.ExecProvider != nil && c.AuthProvider != nil {

staging/src/k8s.io/client-go/transport/config.go

@@ -22,6 +22,8 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+
+	"go.opentelemetry.io/otel/trace"
 )
 
 // Config holds various options for establishing a transport.
@@ -76,6 +78,10 @@ type Config struct {
 	//
 	// socks5 proxying does not currently support spdy streaming endpoints.
 	Proxy func(*http.Request) (*url.URL, error)
+
+	// TracerProvider can provide a tracer, which records distributed tracing spans.
+	// If unspecified, no spans will be recorded.
+	TracerProvider *trace.TracerProvider
 }
 
 // ImpersonationConfig has all the available impersonation options

staging/src/k8s.io/client-go/transport/round_trippers.go

@@ -22,6 +22,9 @@ import (
 	"strings"
 	"time"
 
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/oauth2"
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
@@ -61,9 +64,38 @@ func HTTPWrappersForConfig(config *Config, rt http.RoundTripper) (http.RoundTrip
 		len(config.Impersonate.Extra) > 0 {
 		rt = NewImpersonatingRoundTripper(config.Impersonate, rt)
 	}
+	rt = NewOtelRoundTripper(rt, config.TracerProvider)
 	return rt, nil
 }
 
+type otelWrapper struct {
+	otelRT http.RoundTripper
+	baseRT http.RoundTripper
+}
+
+// NewOtelRoundTripper returns an otelhttp-wrapped round tripper that implements
+// WrappedRoundTripper
+func NewOtelRoundTripper(rt http.RoundTripper, tp *trace.TracerProvider) http.RoundTripper {
+	opts := []otelhttp.Option{
+		otelhttp.WithPropagators(propagation.NewCompositeTextMapPropagator(
+			propagation.TraceContext{},
+			propagation.Baggage{},
+		)),
+	}
+	if tp != nil {
+		opts = append(opts, otelhttp.WithTracerProvider(*tp))
+	}
+	// Even if there is no TracerProvider, the otelhttp still handles context propagation.
+	// See https://github.com/open-telemetry/opentelemetry-go/tree/main/example/passthrough
+	return &otelWrapper{otelRT: otelhttp.NewTransport(rt, opts...), baseRT: rt}
+}
+
+func (o *otelWrapper) WrappedRoundTripper() http.RoundTripper { return o.baseRT }
+
+func (o *otelWrapper) RoundTrip(req *http.Request) (*http.Response, error) {
+	return o.otelRT.RoundTrip(req)
+}
+
 // DebugWrappers wraps a round tripper and logs based on the current log level.
 func DebugWrappers(rt http.RoundTripper) http.RoundTripper {
 	switch {