migrate server tracing to OpenTelemetry

Signed-off-by: Saeid Akbari <saeidakbari.work@gmail.com>

go.mod

@@ -69,6 +69,10 @@ require (
 	github.com/lib/pq v1.10.6
 	github.com/prometheus/prometheus v0.37.0 // indirect
 	go.opencensus.io v0.23.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.32.0
+	go.opentelemetry.io/otel v1.7.0
+	go.opentelemetry.io/otel/sdk v1.7.0
+	go.opentelemetry.io/otel/trace v1.7.0
 	go.uber.org/multierr v1.8.0 // indirect
 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
 	golang.org/x/net v0.0.0-20220802222814-0bcc04d9c69b

go.sum

@@ -539,6 +539,7 @@ github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGE
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
@@ -583,8 +584,10 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY=
 github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
@@ -1430,16 +1433,19 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0=
 go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.31.0/go.mod h1:PFmBsWbldL1kiWZk9+0LBZz2brhByaGsvp6pRICMlPE=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.32.0 h1:mac9BKRqwaX6zxHPDe3pvmWpwuuIM0vuXv2juCnQevE=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.32.0/go.mod h1:5eCOqeGphOyz6TsY3ZDNjE33SM/TFAK3RGuCL2naTgY=
 go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
 go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs=
 go.opentelemetry.io/otel v1.6.0/go.mod h1:bfJD2DZVw0LBxghOTlgnlI0CV3hLDu9XF/QKOUXMTQQ=
 go.opentelemetry.io/otel v1.6.1/go.mod h1:blzUabWHkX6LJewxvadmzafgh/wnvBSDBdOuwkAtrWQ=
+go.opentelemetry.io/otel v1.7.0 h1:Z2lA3Tdch0iDcrhJXDIlC94XE+bxok1F9B+4Lz/lGsM=
 go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk=
 go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=
 go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4=
@@ -1456,18 +1462,21 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.6.1/go.mod h1
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.7.0/go.mod h1:aFXT9Ng2seM9eizF+LfKiyPBGy8xIZKwhusC1gIu3hA=
 go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
 go.opentelemetry.io/otel/metric v0.28.0/go.mod h1:TrzsfQAmQaB1PDcdhBauLMk7nyyg9hm+GoQq/ekE9Iw=
+go.opentelemetry.io/otel/metric v0.30.0 h1:Hs8eQZ8aQgs0U49diZoaS6Uaxw3+bBE3lcMUKBFIk3c=
 go.opentelemetry.io/otel/metric v0.30.0/go.mod h1:/ShZ7+TS4dHzDFmfi1kSXMhMVubNoP0oIaBp70J6UXU=
 go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
 go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
 go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs=
 go.opentelemetry.io/otel/sdk v1.6.1/go.mod h1:IVYrddmFZ+eJqu2k38qD3WezFR2pymCzm8tdxyh3R4E=
+go.opentelemetry.io/otel/sdk v1.7.0 h1:4OmStpcKVOfvDOgCt7UriAPtKolwIhxpnSNI/yK+1B0=
 go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU=
 go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
 go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
 go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
 go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk=
 go.opentelemetry.io/otel/trace v1.6.0/go.mod h1:qs7BrU5cZ8dXQHBGxHMOxwME/27YH2qEp4/+tZLLwJE=
 go.opentelemetry.io/otel/trace v1.6.1/go.mod h1:RkFRM1m0puWIq10oxImnGEduNBzxiN7TXluRBtE+5j0=
+go.opentelemetry.io/otel/trace v1.7.0 h1:O37Iogk1lEkMRXewVtZ1BBTVn5JEp8GrJvP92bJqC6o=
 go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ=
@@ -1526,7 +1535,6 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=

internal/trace/trace.go

@@ -18,27 +18,29 @@ package trace
 import (
 	"context"
 
-	"go.opencensus.io/trace"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 	"gocloud.dev/gcerrors"
 )
 
 // StartSpan adds a span to the trace with the given name.
 func StartSpan(ctx context.Context, name string) context.Context {
-	ctx, _ = trace.StartSpan(ctx, name)
-	return ctx
+	span := trace.SpanFromContext(ctx)
+	span.SetName(name)
+	return trace.ContextWithSpan(ctx, span)
 }
 
 // EndSpan ends a span with the given error.
 func EndSpan(ctx context.Context, err error) {
-	span := trace.FromContext(ctx)
+	span := trace.SpanFromContext(ctx)
 	if err != nil {
-		span.SetStatus(toStatus(err))
+		span.SetStatus(toStatus(err), err.Error())
 	}
 	span.End()
 }
 
 // toStatus interrogates an error and converts it to an appropriate
 // OpenCensus status.
-func toStatus(err error) trace.Status {
-	return trace.Status{Code: int32(gcerrors.Code(err)), Message: err.Error()}
+func toStatus(err error) codes.Code {
+	return codes.Code(gcerrors.Code(err))
 }

server/requestlog/requestlog.go

@@ -25,7 +25,7 @@ import (
 	"net/http"
 	"time"
 
-	"go.opencensus.io/trace"
+	"go.opentelemetry.io/otel/trace"
 )
 
 // Logger wraps the Log method.  Log must be safe to call from multiple
@@ -56,7 +56,7 @@ func NewHandler(log Logger, h http.Handler) *Handler {
 // even if the underlying handler does not.
 func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	start := time.Now()
-	sc := trace.FromContext(r.Context()).SpanContext()
+	sc := trace.SpanContextFromContext(r.Context())
 	ent := &Entry{
 		Request:           cloneRequestWithoutBody(r),
 		ReceivedTime:      start,
@@ -67,8 +67,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		Referer:           r.Referer(),
 		Proto:             r.Proto,
 		RemoteIP:          ipFromHostPort(r.RemoteAddr),
-		TraceID:           sc.TraceID,
-		SpanID:            sc.SpanID,
+		TraceID:           sc.TraceID(),
+		SpanID:            sc.SpanID(),
 	}
 	if addr, ok := r.Context().Value(http.LocalAddrContextKey).(net.Addr); ok {
 		ent.ServerIP = ipFromHostPort(addr.String())

server/requestlog/requestlog_test.go

@@ -24,7 +24,8 @@ import (
 	"strings"
 	"testing"
 
-	"go.opencensus.io/trace"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/trace"
 )
 
 func TestHandler(t *testing.T) {
@@ -84,11 +85,11 @@ func TestHandler(t *testing.T) {
 	if ent.ResponseBodySize != int64(len(responseMsg)) {
 		t.Errorf("ResponseBodySize = %d; want %d", ent.ResponseBodySize, len(responseMsg))
 	}
-	if ent.TraceID != spanCtx.TraceID {
-		t.Errorf("TraceID = %v; want %v", ent.TraceID, spanCtx.TraceID)
+	if ent.TraceID != spanCtx.TraceID() {
+		t.Errorf("TraceID = %v; want %v", ent.TraceID, spanCtx.TraceID())
 	}
-	if ent.SpanID != spanCtx.SpanID {
-		t.Errorf("SpanID = %v; want %v", ent.SpanID, spanCtx.SpanID)
+	if ent.SpanID != spanCtx.SpanID() {
+		t.Errorf("SpanID = %v; want %v", ent.SpanID, spanCtx.SpanID())
 	}
 }
 
@@ -98,10 +99,11 @@ type testSpanHandler struct {
 }
 
 func (sh *testSpanHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	ctx, span := trace.StartSpan(r.Context(), "test")
+	tracer := otel.Tracer("test")
+	ctx, span := tracer.Start(r.Context(), "test")
 	defer span.End()
 	r = r.WithContext(ctx)
-	sc := trace.FromContext(ctx).SpanContext()
+	sc := trace.SpanContextFromContext(ctx)
 	sh.spanCtx = &sc
 	sh.h.ServeHTTP(w, r)
 }

server/requestlog/stackdriver_test.go

@@ -27,7 +27,8 @@ import (
 	"testing"
 	"time"
 
-	"go.opencensus.io/trace"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/trace"
 )
 
 func TestStackdriverLog(t *testing.T) {
@@ -41,9 +42,10 @@ func TestStackdriverLog(t *testing.T) {
 		endTime      = startTime + latencySec
 		endTimeNanos = startTimeNanos + latencyNanos
 	)
-	ctx, span := trace.StartSpan(context.Background(), "test")
+	tracer := otel.Tracer("test")
+	ctx, span := tracer.Start(context.Background(), "test")
 	defer span.End()
-	sc := trace.FromContext(ctx).SpanContext()
+	sc := trace.SpanContextFromContext(ctx)
 	buf := new(bytes.Buffer)
 	var logErr error
 	l := NewStackdriverLogger(buf, func(e error) { logErr = e })
@@ -62,8 +64,8 @@ func TestStackdriverLog(t *testing.T) {
 		ResponseHeaderSize: 555,
 		ResponseBodySize:   789000,
 		Latency:            latencySec*time.Second + latencyNanos*time.Nanosecond,
-		TraceID:            sc.TraceID,
-		SpanID:             sc.SpanID,
+		TraceID:            sc.TraceID(),
+		SpanID:             sc.SpanID(),
 	}
 	ent := *want // copy in case Log accidentally mutates
 	l.Log(&ent)

server/sdserver/server.go

@@ -28,7 +28,7 @@ import (
 
 	"contrib.go.opencensus.io/exporter/stackdriver"
 	"contrib.go.opencensus.io/exporter/stackdriver/monitoredresource"
-	"go.opencensus.io/trace"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	"golang.org/x/oauth2"
 	"google.golang.org/api/option"
 )
@@ -39,7 +39,7 @@ var Set = wire.NewSet(
 	server.Set,
 	NewExporter,
 	monitoredresource.Autodetect,
-	wire.Bind(new(trace.Exporter), new(*stackdriver.Exporter)),
+	wire.Bind(new(sdktrace.SpanExporter), new(*stackdriver.Exporter)),
 	NewRequestLogger,
 	wire.Bind(new(requestlog.Logger), new(*requestlog.StackdriverLogger)),
 )

server/server.go

@@ -27,8 +27,8 @@ import (
 	"gocloud.dev/server/health"
 	"gocloud.dev/server/requestlog"
 
-	"go.opencensus.io/plugin/ochttp"
-	"go.opencensus.io/trace"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 )
 
 // Set is a Wire provider set that produces a *Server given the fields of
@@ -47,14 +47,19 @@ type Server struct {
 	handler        http.Handler
 	wrappedHandler http.Handler
 	healthHandler  health.Handler
-	te             trace.Exporter
-	sampler        trace.Sampler
+	te             sdktrace.SpanExporter
+	sampler        sdktrace.Sampler
+	tp             *sdktrace.TracerProvider
 	once           sync.Once
 	driver         driver.Server
+	handlerName    string
 }
 
 // Options is the set of optional parameters.
 type Options struct {
+	// HandlerName is the name of the server.
+	HandlerName string
+
 	// RequestLogger specifies the logger that will be used to log requests.
 	RequestLogger requestlog.Logger
 
@@ -63,12 +68,12 @@ type Options struct {
 	HealthChecks []health.Checker
 
 	// TraceExporter exports sampled trace spans.
-	TraceExporter trace.Exporter
+	TraceExporter sdktrace.SpanExporter
 
 	// DefaultSamplingPolicy is a function that takes a
-	// trace.SamplingParameters struct and returns a true or false decision about
+	// sdktrace.SamplingParameters struct and returns a true or false decision about
 	// whether it should be sampled and exported.
-	DefaultSamplingPolicy trace.Sampler
+	DefaultSamplingPolicy sdktrace.Sampler
 
 	// Driver serves HTTP requests.
 	Driver driver.Server
@@ -78,6 +83,7 @@ type Options struct {
 func New(h http.Handler, opts *Options) *Server {
 	srv := &Server{handler: h}
 	if opts != nil {
+		srv.handlerName = opts.HandlerName
 		srv.reqlog = opts.RequestLogger
 		srv.te = opts.TraceExporter
 		for _, c := range opts.HealthChecks {
@@ -92,10 +98,13 @@ func New(h http.Handler, opts *Options) *Server {
 func (srv *Server) init() {
 	srv.once.Do(func() {
 		if srv.te != nil {
-			trace.RegisterExporter(srv.te)
+			srv.tp = sdktrace.NewTracerProvider(
+				sdktrace.WithSampler(srv.sampler),
+				sdktrace.WithBatcher(srv.te),
+			)
 		}
 		if srv.sampler != nil {
-			trace.ApplyConfig(trace.Config{DefaultSampler: srv.sampler})
+			sdktrace.WithSampler(srv.sampler)
 		}
 		if srv.driver == nil {
 			srv.driver = NewDefaultDriver()
@@ -115,10 +124,7 @@ func (srv *Server) init() {
 		if srv.reqlog != nil {
 			h = requestlog.NewHandler(srv.reqlog, h)
 		}
-		h = &ochttp.Handler{
-			Handler:          h,
-			IsPublicEndpoint: true,
-		}
+		h = otelhttp.NewHandler(h, srv.handlerName)
 		mux.Handle("/", h)
 		srv.wrappedHandler = mux
 	})
@@ -149,6 +155,10 @@ func (srv *Server) ListenAndServeTLS(addr, certFile, keyFile string) error {
 
 // Shutdown gracefully shuts down the server without interrupting any active connections.
 func (srv *Server) Shutdown(ctx context.Context) error {
+	if srv.tp != nil {
+		srv.tp.Shutdown(ctx)
+	}
+
 	if srv.driver == nil {
 		return nil
 	}

server/xrayserver/server.go

@@ -25,7 +25,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/xray"
 	"github.com/aws/aws-sdk-go/service/xray/xrayiface"
 	"github.com/google/wire"
-	"go.opencensus.io/trace"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	"gocloud.dev/server"
 	"gocloud.dev/server/requestlog"
 )
@@ -36,7 +36,7 @@ var Set = wire.NewSet(
 	server.Set,
 	ServiceSet,
 	NewExporter,
-	wire.Bind(new(trace.Exporter), new(*exporter.Exporter)),
+	wire.Bind(new(sdktrace.SpanExporter), new(*exporter.Exporter)),
 	NewRequestLogger,
 	wire.Bind(new(requestlog.Logger), new(*requestlog.NCSALogger)),
 )