e2e test and bug fix

test/xds/xds_client_outlier_detection_test.go

@@ -20,19 +20,31 @@ package xds_test
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"strings"
 	"testing"
+	"time"
 
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/stubserver"
 	"google.golang.org/grpc/internal/testutils/xds/e2e"
-
 	testgrpc "google.golang.org/grpc/test/grpc_testing"
 	testpb "google.golang.org/grpc/test/grpc_testing"
+	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
 )
 
+// TestOutlierDetection tests an xDS configured ClientConn with an Outlier
+// Detection present in the system which is a logical no-op. An RPC should
+// proceed as normal.
 func (s) TestOutlierDetection(t *testing.T) {
 	oldOD := envconfig.XDSOutlierDetection
 	envconfig.XDSOutlierDetection = true
@@ -74,3 +86,132 @@ func (s) TestOutlierDetection(t *testing.T) {
 		t.Fatalf("rpc EmptyCall() failed: %v", err)
 	}
 }
+
+// defaultClientResourcesSpecifyingMultipleBackendsAndOutlierDetection returns
+// xDS resources which correspond to multiple upstreams, corresponding different
+// backends listening on different localhost:port combinations. The resources
+// also configure an Outlier Detection Balancer set up with Failure Percentage
+// Algorithm, which ejects endpoints based on failure rate.
+func defaultClientResourcesSpecifyingMultipleBackendsAndOutlierDetection(params e2e.ResourceParams, ports []uint32) e2e.UpdateOptions {
+	routeConfigName := "route-" + params.DialTarget
+	clusterName := "cluster-" + params.DialTarget
+	endpointsName := "endpoints-" + params.DialTarget
+	return e2e.UpdateOptions{
+		NodeID:    params.NodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(params.DialTarget, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, params.DialTarget, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{defaultClusterWithOutlierDetection(clusterName, endpointsName, params.SecLevel)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, params.Host, ports)},
+	}
+}
+
+func defaultClusterWithOutlierDetection(clusterName, edsServiceName string, secLevel e2e.SecurityLevel) *v3clusterpb.Cluster {
+	cluster := e2e.DefaultCluster(clusterName, edsServiceName, secLevel)
+	cluster.OutlierDetection = &v3clusterpb.OutlierDetection{
+		Interval: &durationpb.Duration{
+			Nanos: 500000000,
+		},
+		BaseEjectionTime:               &durationpb.Duration{Seconds: 30},
+		MaxEjectionTime:                &durationpb.Duration{Seconds: 300},
+		MaxEjectionPercent:             &wrapperspb.UInt32Value{Value: 1},
+		FailurePercentageThreshold:     &wrapperspb.UInt32Value{Value: 50},
+		EnforcingFailurePercentage:     &wrapperspb.UInt32Value{Value: 100},
+		FailurePercentageRequestVolume: &wrapperspb.UInt32Value{Value: 1},
+		FailurePercentageMinimumHosts:  &wrapperspb.UInt32Value{Value: 1},
+	}
+	return cluster
+}
+
+// TestOutlierDetectionWithOutlier tests the Outlier Detection Balancer e2e. It
+// spins up three backends, one which consistently errors, and configures the
+// ClientConn using xDS to connect to all three of those backends. The Outlier
+// Detection Balancer should eject the connection to the backend which
+// constantly errors, and thus RPC's should mainly go to backend 1 and 2.
+func (s) TestOutlierDetectionWithOutlier(t *testing.T) {
+	oldOD := envconfig.XDSOutlierDetection
+	envconfig.XDSOutlierDetection = true
+	internal.RegisterOutlierDetectionBalancerForTesting()
+	defer func() {
+		envconfig.XDSOutlierDetection = oldOD
+		internal.UnregisterOutlierDetectionBalancerForTesting()
+	}()
+
+	managementServer, nodeID, _, resolver, cleanup := e2e.SetupManagementServer(t)
+	defer cleanup()
+
+	// counters for how many times backends got called
+	var count1, count2, count3 int
+
+	// Working backend 1.
+	port1, cleanup1 := startTestService(t, &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			count1++
+			return &testpb.Empty{}, nil
+		},
+		Address: "localhost:0",
+	})
+	defer cleanup1()
+
+	// Working backend 2.
+	port2, cleanup2 := startTestService(t, &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			count2++
+			return &testpb.Empty{}, nil
+		},
+		Address: "localhost:0",
+	})
+	defer cleanup2()
+	// Backend 3 that will always return an error and eventually ejected.
+	port3, cleanup3 := startTestService(t, &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			count3++
+			return nil, errors.New("some error")
+		},
+		Address: "localhost:0",
+	})
+	defer cleanup3()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := defaultClientResourcesSpecifyingMultipleBackendsAndOutlierDetection(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		SecLevel:   e2e.SecurityLevelNone,
+	}, []uint32{port1, port2, port3})
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*100)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	for i := 0; i < 2000; i++ {
+		// Can either error or not depending on the backend called.
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil && !strings.Contains(err.Error(), "some error") {
+			t.Fatalf("rpc EmptyCall() failed: %v", err)
+		}
+		time.Sleep(time.Millisecond)
+	}
+
+	// Backend 1 should've gotten more than 1/3rd of the load as backend 3
+	// should get ejected, leaving only 1 and 2.
+	if count1 < 700 {
+		t.Fatalf("backend 1 should've gotten more than 1/3rd of the load")
+	}
+	// Backend 2 should've gotten more than 1/3rd of the load as backend 3
+	// should get ejected, leaving only 1 and 2.
+	if count2 < 700 {
+		t.Fatalf("backend 2 should've gotten more than 1/3rd of the load")
+	}
+	// Backend 3 should've gotten less than 1/3rd of the load since it gets
+	// ejected.
+	if count3 > 650 {
+		t.Fatalf("backend 1 should've gotten more than 1/3rd of the load")
+	}
+}