Skip to content

Commit

Permalink
xds/federation: e2e tests (#5103)
Browse files Browse the repository at this point in the history
  • Loading branch information
menghanl committed Jan 7, 2022
1 parent 907a202 commit 77b478d
Show file tree
Hide file tree
Showing 6 changed files with 214 additions and 45 deletions.
31 changes: 25 additions & 6 deletions internal/xds/bootstrap.go
Expand Up @@ -55,6 +55,14 @@ type BootstrapOptions struct {
ServerListenerResourceNameTemplate string
// CertificateProviders is the certificate providers configuration.
CertificateProviders map[string]json.RawMessage
// Authorities is a list of non-default authorities.
//
// In the config, an authority contains {ServerURI, xds-version, creds,
// features, etc}. Note that this fields only has ServerURI (it's a
// map[authority-name]ServerURI). The other fields (version, creds,
// features) are assumed to be the same as the default authority (they can
// be added later if needed).
Authorities map[string]string
}

// SetupBootstrapFile creates a temporary file with bootstrap contents, based on
Expand Down Expand Up @@ -94,12 +102,8 @@ func BootstrapContents(opts BootstrapOptions) ([]byte, error) {
cfg := &bootstrapConfig{
XdsServers: []server{
{
ServerURI: opts.ServerURI,
ChannelCreds: []creds{
{
Type: "insecure",
},
},
ServerURI: opts.ServerURI,
ChannelCreds: []creds{{Type: "insecure"}},
},
},
Node: node{
Expand All @@ -117,6 +121,16 @@ func BootstrapContents(opts BootstrapOptions) ([]byte, error) {
return nil, fmt.Errorf("unsupported xDS transport protocol version: %v", opts.Version)
}

auths := make(map[string]authority)
for n, auURI := range opts.Authorities {
auths[n] = authority{XdsServers: []server{{
ServerURI: auURI,
ChannelCreds: []creds{{Type: "insecure"}},
ServerFeatures: cfg.XdsServers[0].ServerFeatures,
}}}
}
cfg.Authorities = auths

bootstrapContents, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
return nil, fmt.Errorf("failed to created bootstrap file: %v", err)
Expand All @@ -129,6 +143,11 @@ type bootstrapConfig struct {
Node node `json:"node,omitempty"`
CertificateProviders map[string]json.RawMessage `json:"certificate_providers,omitempty"`
ServerListenerResourceNameTemplate string `json:"server_listener_resource_name_template,omitempty"`
Authorities map[string]authority `json:"authorities,omitempty"`
}

type authority struct {
XdsServers []server `json:"xds_servers,omitempty"`
}

type server struct {
Expand Down
142 changes: 142 additions & 0 deletions xds/internal/test/xds_client_federation_test.go
@@ -0,0 +1,142 @@
//go:build !386
// +build !386

/*
*
* Copyright 2021 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package xds_test

import (
"context"
"fmt"
"testing"

v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
"github.com/google/uuid"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/internal/envconfig"
xdsinternal "google.golang.org/grpc/internal/xds"
testpb "google.golang.org/grpc/test/grpc_testing"
"google.golang.org/grpc/xds"
"google.golang.org/grpc/xds/internal/testutils"
"google.golang.org/grpc/xds/internal/testutils/e2e"
"google.golang.org/grpc/xds/internal/xdsclient/xdsresource"
)

// TestClientSideFederation tests that federation is supported.
//
// In this test, some xDS responses contain resource names in another authority
// (in the new resource name style):
// - LDS: old style, no authority (default authority)
// - RDS: new style, in a different authority
// - CDS: old style, no authority (default authority)
// - EDS: new style, in a different authority
func (s) TestClientSideFederation(t *testing.T) {
oldXDSFederation := envconfig.XDSFederation
envconfig.XDSFederation = true
defer func() { envconfig.XDSFederation = oldXDSFederation }()

// Start a management server as the default authority.
serverDefaultAuth, err := e2e.StartManagementServer()
if err != nil {
t.Fatalf("Failed to spin up the xDS management server: %v", err)
}
t.Cleanup(serverDefaultAuth.Stop)

// Start another management server as the other authority.
const nonDefaultAuth = "non-default-auth"
serverAnotherAuth, err := e2e.StartManagementServer()
if err != nil {
t.Fatalf("Failed to spin up the xDS management server: %v", err)
}
t.Cleanup(serverAnotherAuth.Stop)

// Create a bootstrap file in a temporary directory.
nodeID := uuid.New().String()
bootstrapContents, err := xdsinternal.BootstrapContents(xdsinternal.BootstrapOptions{
Version: xdsinternal.TransportV3,
NodeID: nodeID,
ServerURI: serverDefaultAuth.Address,
ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
// Specify the address of the non-default authority.
Authorities: map[string]string{nonDefaultAuth: serverAnotherAuth.Address},
})
if err != nil {
t.Fatalf("Failed to create bootstrap file: %v", err)
}

resolver, err := xds.NewXDSResolverWithConfigForTesting(bootstrapContents)
if err != nil {
t.Fatalf("Failed to create xDS resolver for testing: %v", err)
}
port, cleanup := clientSetup(t, &testService{})
defer cleanup()

const serviceName = "my-service-client-side-xds"
// LDS is old style name.
ldsName := serviceName
// RDS is new style, with the non default authority.
rdsName := testutils.BuildResourceName(xdsresource.RouteConfigResource, nonDefaultAuth, "route-"+serviceName, nil)
// CDS is old style name.
cdsName := "cluster-" + serviceName
// EDS is new style, with the non default authority.
edsName := testutils.BuildResourceName(xdsresource.EndpointsResource, nonDefaultAuth, "endpoints-"+serviceName, nil)

// Split resources, put LDS/CDS in the default authority, and put RDS/EDS in
// the other authority.
resourcesDefault := e2e.UpdateOptions{
NodeID: nodeID,
// This has only LDS and CDS.
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)},
Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone)},
SkipValidation: true,
}
resourcesAnother := e2e.UpdateOptions{
NodeID: nodeID,
// This has only RDS and EDS.
Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, cdsName)},
Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsName, "localhost", []uint32{port})},
SkipValidation: true,
}

ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
// This has only LDS and CDS.
if err := serverDefaultAuth.Update(ctx, resourcesDefault); err != nil {
t.Fatal(err)
}
// This has only RDS and EDS.
if err := serverAnotherAuth.Update(ctx, resourcesAnother); err != nil {
t.Fatal(err)
}

// Create a ClientConn and make a successful RPC.
cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
if err != nil {
t.Fatalf("failed to dial local test server: %v", err)
}
defer cc.Close()

client := testpb.NewTestServiceClient(cc)
if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
t.Fatalf("rpc EmptyCall() failed: %v", err)
}
}
28 changes: 28 additions & 0 deletions xds/internal/testutils/testutils.go
Expand Up @@ -17,3 +17,31 @@

// Package testutils provides utility types, for use in xds tests.
package testutils

import (
"google.golang.org/grpc/xds/internal/xdsclient/xdsresource"
"google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version"
)

// BuildResourceName returns the resource name in the format of an xdstp://
// resource.
func BuildResourceName(typ xdsresource.ResourceType, auth, id string, ctxParams map[string]string) string {
var typS string
switch typ {
case xdsresource.ListenerResource:
typS = version.V3ListenerType
case xdsresource.RouteConfigResource:
typS = version.V3RouteConfigType
case xdsresource.ClusterResource:
typS = version.V3ClusterType
case xdsresource.EndpointsResource:
typS = version.V3EndpointsType
}
return (&xdsresource.Name{
Scheme: "xdstp",
Authority: auth,
Type: typS,
ID: id,
ContextParams: ctxParams,
}).String()
}
18 changes: 9 additions & 9 deletions xds/internal/xdsclient/authority_test.go
Expand Up @@ -150,7 +150,7 @@ func (s) TestAuthorityNoneDefaultAuthority(t *testing.T) {
}
t.Cleanup(client.Close)

resourceName := buildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
resourceName := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
ctrl, ok, _ := watchAndFetchNewController(t, client, resourceName, ctrlCh)
if !ok {
t.Fatalf("want a new controller to be built, got none")
Expand Down Expand Up @@ -182,7 +182,7 @@ func (s) TestAuthorityShare(t *testing.T) {
}
t.Cleanup(client.Close)

resourceName := buildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
resourceName := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
ctrl1, ok1, _ := watchAndFetchNewController(t, client, resourceName, ctrlCh)
if !ok1 {
t.Fatalf("want a new controller to be built, got none")
Expand All @@ -195,15 +195,15 @@ func (s) TestAuthorityShare(t *testing.T) {

// Call the watch with the same authority name. This shouldn't create a new
// controller.
resourceNameSameAuthority := buildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName+"1", nil)
resourceNameSameAuthority := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName+"1", nil)
ctrl2, ok2, _ := watchAndFetchNewController(t, client, resourceNameSameAuthority, ctrlCh)
if ok2 {
t.Fatalf("an unexpected controller is built with config: %v", ctrl2.config)
}

// Call the watch with a different authority name, but the same server
// config. This shouldn't create a new controller.
resourceNameSameConfig := buildResourceName(xdsresource.ClusterResource, testAuthority2, testCDSName+"1", nil)
resourceNameSameConfig := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority2, testCDSName+"1", nil)
if ctrl, ok, _ := watchAndFetchNewController(t, client, resourceNameSameConfig, ctrlCh); ok {
t.Fatalf("an unexpected controller is built with config: %v", ctrl.config)
}
Expand All @@ -230,7 +230,7 @@ func (s) TestAuthorityIdleTimeout(t *testing.T) {
}
t.Cleanup(client.Close)

resourceName := buildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
resourceName := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
ctrl1, ok1, cancelWatch1 := watchAndFetchNewController(t, client, resourceName, ctrlCh)
if !ok1 {
t.Fatalf("want a new controller to be built, got none")
Expand All @@ -239,7 +239,7 @@ func (s) TestAuthorityIdleTimeout(t *testing.T) {
var cancelWatch2 func()
// Call the watch with the same authority name. This shouldn't create a new
// controller.
resourceNameSameAuthority := buildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName+"1", nil)
resourceNameSameAuthority := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName+"1", nil)
ctrl2, ok2, cancelWatch2 := watchAndFetchNewController(t, client, resourceNameSameAuthority, ctrlCh)
if ok2 {
t.Fatalf("an unexpected controller is built with config: %v", ctrl2.config)
Expand Down Expand Up @@ -285,7 +285,7 @@ func (s) TestAuthorityClientClose(t *testing.T) {
t.Fatalf("want a new controller to be built, got none")
}

resourceNameWithAuthority := buildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
resourceNameWithAuthority := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
ctrl2, ok2, _ := watchAndFetchNewController(t, client, resourceNameWithAuthority, ctrlCh)
if !ok2 {
t.Fatalf("want a new controller to be built, got none")
Expand Down Expand Up @@ -329,7 +329,7 @@ func (s) TestAuthorityRevive(t *testing.T) {

// Start a watch on the authority, and cancel it. This puts the authority in
// the idle cache.
resourceName := buildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
resourceName := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName, nil)
ctrl1, ok1, cancelWatch1 := watchAndFetchNewController(t, client, resourceName, ctrlCh)
if !ok1 {
t.Fatalf("want a new controller to be built, got none")
Expand All @@ -338,7 +338,7 @@ func (s) TestAuthorityRevive(t *testing.T) {

// Start another watch on this authority, it should retrieve the authority
// from the cache, instead of creating a new one.
resourceNameWithAuthority := buildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName+"1", nil)
resourceNameWithAuthority := xdstestutils.BuildResourceName(xdsresource.ClusterResource, testAuthority, testCDSName+"1", nil)
ctrl2, ok2, _ := watchAndFetchNewController(t, client, resourceNameWithAuthority, ctrlCh)
if ok2 {
t.Fatalf("an unexpected controller is built with config: %v", ctrl2.config)
Expand Down
5 changes: 3 additions & 2 deletions xds/internal/xdsclient/watchers_federation_test.go
Expand Up @@ -22,6 +22,7 @@ import (
"testing"

"google.golang.org/grpc/internal/envconfig"
"google.golang.org/grpc/xds/internal/testutils"
"google.golang.org/grpc/xds/internal/xdsclient/xdsresource"
)

Expand All @@ -35,8 +36,8 @@ func testFedTwoWatchDifferentContextParameterOrder(t *testing.T, typ xdsresource
overrideFedEnvVar(t)
var (
// Two resource names only differ in context parameter __order__.
resourceName1 = buildResourceName(typ, testAuthority, "test-resource-name", nil) + "?a=1&b=2"
resourceName2 = buildResourceName(typ, testAuthority, "test-resource-name", nil) + "?b=2&a=1"
resourceName1 = testutils.BuildResourceName(typ, testAuthority, "test-resource-name", nil) + "?a=1&b=2"
resourceName2 = testutils.BuildResourceName(typ, testAuthority, "test-resource-name", nil) + "?b=2&a=1"
)

ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
Expand Down

0 comments on commit 77b478d

Please sign in to comment.