/
cdsbalancer_security_test.go
727 lines (651 loc) · 29.1 KB
/
cdsbalancer_security_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
// +build go1.12
/*
* Copyright 2020 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cdsbalancer
import (
"context"
"errors"
"fmt"
"regexp"
"testing"
"github.com/google/go-cmp/cmp"
"google.golang.org/grpc/attributes"
"google.golang.org/grpc/balancer"
"google.golang.org/grpc/credentials/local"
"google.golang.org/grpc/credentials/tls/certprovider"
"google.golang.org/grpc/credentials/xds"
"google.golang.org/grpc/internal"
xdscredsinternal "google.golang.org/grpc/internal/credentials/xds"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/xds/matcher"
"google.golang.org/grpc/resolver"
xdstestutils "google.golang.org/grpc/xds/internal/testutils"
"google.golang.org/grpc/xds/internal/testutils/fakeclient"
"google.golang.org/grpc/xds/internal/xdsclient"
"google.golang.org/grpc/xds/internal/xdsclient/bootstrap"
)
const (
fakeProvider1Name = "fake-certificate-provider-1"
fakeProvider2Name = "fake-certificate-provider-2"
fakeConfig = "my fake config"
testSAN = "test-san"
)
var (
testSANMatchers = []matcher.StringMatcher{
matcher.StringMatcherForTesting(newStringP(testSAN), nil, nil, nil, nil, true),
matcher.StringMatcherForTesting(nil, newStringP(testSAN), nil, nil, nil, false),
matcher.StringMatcherForTesting(nil, nil, newStringP(testSAN), nil, nil, false),
matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(testSAN), false),
matcher.StringMatcherForTesting(nil, nil, nil, newStringP(testSAN), nil, false),
}
fpb1, fpb2 *fakeProviderBuilder
bootstrapConfig *bootstrap.Config
cdsUpdateWithGoodSecurityCfg = xdsclient.ClusterUpdate{
ClusterName: serviceName,
SecurityCfg: &xdsclient.SecurityConfig{
RootInstanceName: "default1",
IdentityInstanceName: "default2",
SubjectAltNameMatchers: testSANMatchers,
},
}
cdsUpdateWithMissingSecurityCfg = xdsclient.ClusterUpdate{
ClusterName: serviceName,
SecurityCfg: &xdsclient.SecurityConfig{
RootInstanceName: "not-default",
},
}
)
func newStringP(s string) *string {
return &s
}
func init() {
fpb1 = &fakeProviderBuilder{name: fakeProvider1Name}
fpb2 = &fakeProviderBuilder{name: fakeProvider2Name}
cfg1, _ := fpb1.ParseConfig(fakeConfig + "1111")
cfg2, _ := fpb2.ParseConfig(fakeConfig + "2222")
bootstrapConfig = &bootstrap.Config{
CertProviderConfigs: map[string]*certprovider.BuildableConfig{
"default1": cfg1,
"default2": cfg2,
},
}
certprovider.Register(fpb1)
certprovider.Register(fpb2)
}
// fakeProviderBuilder builds new instances of fakeProvider and interprets the
// config provided to it as a string.
type fakeProviderBuilder struct {
name string
}
func (b *fakeProviderBuilder) ParseConfig(config interface{}) (*certprovider.BuildableConfig, error) {
s, ok := config.(string)
if !ok {
return nil, fmt.Errorf("providerBuilder %s received config of type %T, want string", b.name, config)
}
return certprovider.NewBuildableConfig(b.name, []byte(s), func(certprovider.BuildOptions) certprovider.Provider {
return &fakeProvider{
Distributor: certprovider.NewDistributor(),
config: s,
}
}), nil
}
func (b *fakeProviderBuilder) Name() string {
return b.name
}
// fakeProvider is an implementation of the Provider interface which provides a
// method for tests to invoke to push new key materials.
type fakeProvider struct {
*certprovider.Distributor
config string
}
// Close helps implement the Provider interface.
func (p *fakeProvider) Close() {
p.Distributor.Stop()
}
// setupWithXDSCreds performs all the setup steps required for tests which use
// xDSCredentials.
func setupWithXDSCreds(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, func()) {
t.Helper()
xdsC := fakeclient.NewClient()
builder := balancer.Get(cdsName)
if builder == nil {
t.Fatalf("balancer.Get(%q) returned nil", cdsName)
}
// Create and pass xdsCredentials while building the CDS balancer.
creds, err := xds.NewClientCredentials(xds.ClientOptions{
FallbackCreds: local.NewCredentials(), // Placeholder fallback credentials.
})
if err != nil {
t.Fatalf("Failed to create xDS client creds: %v", err)
}
// Create a new CDS balancer and pass it a fake balancer.ClientConn which we
// can use to inspect the different calls made by the balancer.
tcc := xdstestutils.NewTestClientConn(t)
cdsB := builder.Build(tcc, balancer.BuildOptions{DialCreds: creds})
// Override the creation of the EDS balancer to return a fake EDS balancer
// implementation.
edsB := newTestEDSBalancer()
oldEDSBalancerBuilder := newChildBalancer
newChildBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.Balancer, error) {
edsB.parentCC = cc
return edsB, nil
}
// Push a ClientConnState update to the CDS balancer with a cluster name.
if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil {
t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err)
}
// Make sure the CDS balancer registers a Cluster watch with the xDS client
// passed via attributes in the above update.
ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer ctxCancel()
gotCluster, err := xdsC.WaitForWatchCluster(ctx)
if err != nil {
t.Fatalf("xdsClient.WatchCDS failed with error: %v", err)
}
if gotCluster != clusterName {
t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, clusterName)
}
return xdsC, cdsB.(*cdsBalancer), edsB, tcc, func() {
newChildBalancer = oldEDSBalancerBuilder
xdsC.Close()
}
}
// makeNewSubConn invokes the NewSubConn() call on the balancer.ClientConn
// passed to the EDS balancer, and verifies that the CDS balancer forwards the
// call appropriately to its parent balancer.ClientConn with or without
// attributes bases on the value of wantFallback.
func makeNewSubConn(ctx context.Context, edsCC balancer.ClientConn, parentCC *xdstestutils.TestClientConn, wantFallback bool) (balancer.SubConn, error) {
dummyAddr := "foo-address"
addrs := []resolver.Address{{Addr: dummyAddr}}
sc, err := edsCC.NewSubConn(addrs, balancer.NewSubConnOptions{})
if err != nil {
return nil, fmt.Errorf("NewSubConn(%+v) on parent ClientConn failed: %v", addrs, err)
}
select {
case <-ctx.Done():
return nil, errors.New("timeout when waiting for new SubConn")
case gotAddrs := <-parentCC.NewSubConnAddrsCh:
if len(gotAddrs) != 1 {
return nil, fmt.Errorf("NewSubConn expected 1 address, got %d", len(gotAddrs))
}
if got, want := gotAddrs[0].Addr, addrs[0].Addr; got != want {
return nil, fmt.Errorf("resolver.Address passed to parent ClientConn has address %q, want %q", got, want)
}
getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xdscredsinternal.HandshakeInfo)
hi := getHI(gotAddrs[0].Attributes)
if hi == nil {
return nil, errors.New("resolver.Address passed to parent ClientConn doesn't contain attributes")
}
if gotFallback := hi.UseFallbackCreds(); gotFallback != wantFallback {
return nil, fmt.Errorf("resolver.Address HandshakeInfo uses fallback creds? %v, want %v", gotFallback, wantFallback)
}
if !wantFallback {
if diff := cmp.Diff(testSANMatchers, hi.GetSANMatchersForTesting(), cmp.AllowUnexported(regexp.Regexp{})); diff != "" {
return nil, fmt.Errorf("unexpected diff in the list of SAN matchers (-got, +want):\n%s", diff)
}
}
}
return sc, nil
}
// TestSecurityConfigWithoutXDSCreds tests the case where xdsCredentials are not
// in use, but the CDS balancer receives a Cluster update with security
// configuration. Verifies that no certificate providers are created, and that
// the address attributes added as part of the intercepted NewSubConn() method
// indicate the use of fallback credentials.
func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) {
// This creates a CDS balancer, pushes a ClientConnState update with a fake
// xdsClient, and makes sure that the CDS balancer registers a watch on the
// provided xdsClient.
xdsC, cdsB, edsB, tcc, cancel := setupWithWatch(t)
defer func() {
cancel()
cdsB.Close()
}()
// Override the provider builder function to push on a channel. We do not
// expect this function to be called as part of this test.
providerCh := testutils.NewChannel()
origBuildProvider := buildProvider
buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) {
p, err := origBuildProvider(c, id, cert, wi, wr)
providerCh.Send(nil)
return p, err
}
defer func() { buildProvider = origBuildProvider }()
// Here we invoke the watch callback registered on the fake xdsClient. This
// will trigger the watch handler on the CDS balancer, which will attempt to
// create a new EDS balancer. The fake EDS balancer created above will be
// returned to the CDS balancer, because we have overridden the
// newChildBalancer function as part of test setup.
cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
wantCCS := edsCCS(serviceName, nil, false)
ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer ctxCancel()
if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
t.Fatal(err)
}
// Make a NewSubConn and verify that the HandshakeInfo does not contain any
// certificate providers, forcing the credentials implementation to use
// fallback creds.
if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil {
t.Fatal(err)
}
// Again, since xdsCredentials are not in use, no certificate providers
// should have been initialized by the CDS balancer.
sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
defer sCancel()
if _, err := providerCh.Receive(sCtx); err != context.DeadlineExceeded {
t.Fatal("cds balancer created certificate providers when not using xds credentials")
}
}
// TestNoSecurityConfigWithXDSCreds tests the case where xdsCredentials are in
// use, but the CDS balancer receives a Cluster update without security
// configuration. Verifies that no certificate providers are created, and that
// the address attributes added as part of the intercepted NewSubConn() method
// indicate the use of fallback credentials.
func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) {
// This creates a CDS balancer which uses xdsCredentials, pushes a
// ClientConnState update with a fake xdsClient, and makes sure that the CDS
// balancer registers a watch on the provided xdsClient.
xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
defer func() {
cancel()
cdsB.Close()
}()
// Override the provider builder function to push on a channel. We do not
// expect this function to be called as part of this test.
providerCh := testutils.NewChannel()
origBuildProvider := buildProvider
buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) {
p, err := origBuildProvider(c, id, cert, wi, wr)
providerCh.Send(nil)
return p, err
}
defer func() { buildProvider = origBuildProvider }()
// Here we invoke the watch callback registered on the fake xdsClient. This
// will trigger the watch handler on the CDS balancer, which will attempt to
// create a new EDS balancer. The fake EDS balancer created above will be
// returned to the CDS balancer, because we have overridden the
// newChildBalancer function as part of test setup. No security config is
// passed to the CDS balancer as part of this update.
cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
wantCCS := edsCCS(serviceName, nil, false)
ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer ctxCancel()
if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
t.Fatal(err)
}
// Make a NewSubConn and verify that the HandshakeInfo does not contain any
// certificate providers, forcing the credentials implementation to use
// fallback creds.
if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil {
t.Fatal(err)
}
// Again, since no security configuration was received, no certificate
// providers should have been initialized by the CDS balancer.
sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
defer sCancel()
if _, err := providerCh.Receive(sCtx); err != context.DeadlineExceeded {
t.Fatal("cds balancer created certificate providers when not using xds credentials")
}
}
// TestSecurityConfigNotFoundInBootstrap tests the case where the security
// config returned by the xDS server cannot be resolved based on the contents of
// the bootstrap config. Verifies that the balancer puts the channel in a failed
// state, and returns an error picker.
func (s) TestSecurityConfigNotFoundInBootstrap(t *testing.T) {
// We test two cases here:
// 0: Bootstrap contains security config. But received plugin instance name
// is not found in the bootstrap config.
// 1: Bootstrap contains no security config.
for i := 0; i < 2; i++ {
// This creates a CDS balancer which uses xdsCredentials, pushes a
// ClientConnState update with a fake xdsClient, and makes sure that the CDS
// balancer registers a watch on the provided xdsClient.
xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
defer func() {
cancel()
cdsB.Close()
}()
if i == 0 {
// Set the bootstrap config used by the fake client.
xdsC.SetBootstrapConfig(bootstrapConfig)
}
// Here we invoke the watch callback registered on the fake xdsClient. A bad
// security config is passed here. So, we expect the CDS balancer to not
// create an EDS balancer and instead reject this update and put the channel
// in a bad state.
xdsC.InvokeWatchClusterCallback(cdsUpdateWithMissingSecurityCfg, nil)
// The CDS balancer has not yet created an EDS balancer. So, this bad
// watcher update should not be forwarded forwarded to our fake EDS balancer
// as an error.
sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
defer sCancel()
if err := edsB.waitForResolverError(sCtx, nil); err != context.DeadlineExceeded {
t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
}
// Make sure the CDS balancer reports an error picker.
ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer ctxCancel()
if err := tcc.WaitForErrPicker(ctx); err != nil {
t.Fatal(err)
}
}
}
// TestCertproviderStoreError tests the case where the certprovider.Store
// returns an error when the CDS balancer attempts to create a provider.
func (s) TestCertproviderStoreError(t *testing.T) {
// This creates a CDS balancer which uses xdsCredentials, pushes a
// ClientConnState update with a fake xdsClient, and makes sure that the CDS
// balancer registers a watch on the provided xdsClient.
xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
defer func() {
cancel()
cdsB.Close()
}()
// Override the provider builder function to return an error.
origBuildProvider := buildProvider
buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) {
return nil, errors.New("certprovider store error")
}
defer func() { buildProvider = origBuildProvider }()
// Set the bootstrap config used by the fake client.
xdsC.SetBootstrapConfig(bootstrapConfig)
// Here we invoke the watch callback registered on the fake xdsClient. Even
// though the received update is good, the certprovider.Store is configured
// to return an error. So, CDS balancer should reject this config and report
// an error.
xdsC.InvokeWatchClusterCallback(cdsUpdateWithGoodSecurityCfg, nil)
// The CDS balancer has not yet created an EDS balancer. So, this bad
// watcher update should not be forwarded forwarded to our fake EDS balancer
// as an error.
sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
defer sCancel()
if err := edsB.waitForResolverError(sCtx, nil); err != context.DeadlineExceeded {
t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
}
// Make sure the CDS balancer reports an error picker.
ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer ctxCancel()
if err := tcc.WaitForErrPicker(ctx); err != nil {
t.Fatal(err)
}
}
func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) {
// This creates a CDS balancer which uses xdsCredentials, pushes a
// ClientConnState update with a fake xdsClient, and makes sure that the CDS
// balancer registers a watch on the provided xdsClient.
xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
defer func() {
cancel()
cdsB.Close()
}()
// Set the bootstrap config used by the fake client.
xdsC.SetBootstrapConfig(bootstrapConfig)
// Here we invoke the watch callback registered on the fake xdsClient. A bad
// security config is passed here. So, we expect the CDS balancer to not
// create an EDS balancer and instead reject this update and put the channel
// in a bad state.
xdsC.InvokeWatchClusterCallback(cdsUpdateWithMissingSecurityCfg, nil)
// The CDS balancer has not yet created an EDS balancer. So, this bad
// watcher update should not be forwarded forwarded to our fake EDS balancer
// as an error.
sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
defer sCancel()
if err := edsB.waitForResolverError(sCtx, nil); err != context.DeadlineExceeded {
t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)")
}
// Make sure the CDS balancer reports an error picker.
ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer ctxCancel()
if err := tcc.WaitForErrPicker(ctx); err != nil {
t.Fatal(err)
}
// Here we invoke the watch callback registered on the fake xdsClient. This
// will trigger the watch handler on the CDS balancer, which will attempt to
// create a new EDS balancer. The fake EDS balancer created above will be
// returned to the CDS balancer, because we have overridden the
// newChildBalancer function as part of test setup.
wantCCS := edsCCS(serviceName, nil, false)
if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
t.Fatal(err)
}
// Make a NewSubConn and verify that attributes are added.
if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
t.Fatal(err)
}
}
// TestGoodSecurityConfig tests the case where the CDS balancer receives
// security configuration as part of the Cluster resource which can be
// successfully resolved using the bootstrap file contents. Verifies that
// certificate providers are created, and that the NewSubConn() call adds
// appropriate address attributes.
func (s) TestGoodSecurityConfig(t *testing.T) {
// This creates a CDS balancer which uses xdsCredentials, pushes a
// ClientConnState update with a fake xdsClient, and makes sure that the CDS
// balancer registers a watch on the provided xdsClient.
xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
defer func() {
cancel()
cdsB.Close()
}()
// Set the bootstrap config used by the fake client.
xdsC.SetBootstrapConfig(bootstrapConfig)
// Here we invoke the watch callback registered on the fake xdsClient. This
// will trigger the watch handler on the CDS balancer, which will attempt to
// create a new EDS balancer. The fake EDS balancer created above will be
// returned to the CDS balancer, because we have overridden the
// newChildBalancer function as part of test setup.
wantCCS := edsCCS(serviceName, nil, false)
ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer ctxCancel()
if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
t.Fatal(err)
}
// Make a NewSubConn and verify that attributes are added.
sc, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false)
if err != nil {
t.Fatal(err)
}
// Invoke UpdateAddresses and verify that attributes are added.
dummyAddr := "bar-address"
addrs := []resolver.Address{{Addr: dummyAddr}}
edsB.parentCC.UpdateAddresses(sc, addrs)
select {
case <-ctx.Done():
t.Fatal("timeout when waiting for addresses to be updated on the subConn")
case gotAddrs := <-tcc.UpdateAddressesAddrsCh:
if len(gotAddrs) != 1 {
t.Fatalf("UpdateAddresses expected 1 address, got %d", len(gotAddrs))
}
if got, want := gotAddrs[0].Addr, addrs[0].Addr; got != want {
t.Fatalf("resolver.Address passed to parent ClientConn through UpdateAddresses() has address %q, want %q", got, want)
}
getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xdscredsinternal.HandshakeInfo)
hi := getHI(gotAddrs[0].Attributes)
if hi == nil {
t.Fatal("resolver.Address passed to parent ClientConn through UpdateAddresses() doesn't contain attributes")
}
}
}
func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
// This creates a CDS balancer which uses xdsCredentials, pushes a
// ClientConnState update with a fake xdsClient, and makes sure that the CDS
// balancer registers a watch on the provided xdsClient.
xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
defer func() {
cancel()
cdsB.Close()
}()
// Set the bootstrap config used by the fake client.
xdsC.SetBootstrapConfig(bootstrapConfig)
// Here we invoke the watch callback registered on the fake xdsClient. This
// will trigger the watch handler on the CDS balancer, which will attempt to
// create a new EDS balancer. The fake EDS balancer created above will be
// returned to the CDS balancer, because we have overridden the
// newChildBalancer function as part of test setup.
wantCCS := edsCCS(serviceName, nil, false)
ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer ctxCancel()
if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
t.Fatal(err)
}
// Make a NewSubConn and verify that attributes are added.
if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
t.Fatal(err)
}
// Here we invoke the watch callback registered on the fake xdsClient with
// an update which contains bad security config. So, we expect the CDS
// balancer to forward this error to the EDS balancer and eventually the
// channel needs to be put in a bad state.
cdsUpdate := xdsclient.ClusterUpdate{ClusterName: serviceName}
if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
t.Fatal(err)
}
// Make a NewSubConn and verify that fallback creds are used.
if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, true); err != nil {
t.Fatal(err)
}
}
// TestSecurityConfigUpdate_GoodToBad tests the case where the first security
// config returned by the xDS server is successful, but the second update cannot
// be resolved based on the contents of the bootstrap config. Verifies that the
// error is forwarded to the EDS balancer (which was created as part of the
// first successful update).
func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) {
// This creates a CDS balancer which uses xdsCredentials, pushes a
// ClientConnState update with a fake xdsClient, and makes sure that the CDS
// balancer registers a watch on the provided xdsClient.
xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
defer func() {
cancel()
cdsB.Close()
}()
// Set the bootstrap config used by the fake client.
xdsC.SetBootstrapConfig(bootstrapConfig)
// Here we invoke the watch callback registered on the fake xdsClient. This
// will trigger the watch handler on the CDS balancer, which will attempt to
// create a new EDS balancer. The fake EDS balancer created above will be
// returned to the CDS balancer, because we have overridden the
// newChildBalancer function as part of test setup.
wantCCS := edsCCS(serviceName, nil, false)
ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer ctxCancel()
if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdateWithGoodSecurityCfg, nil}, wantCCS, edsB); err != nil {
t.Fatal(err)
}
// Make a NewSubConn and verify that attributes are added.
if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
t.Fatal(err)
}
// Here we invoke the watch callback registered on the fake xdsClient with
// an update which contains bad security config. So, we expect the CDS
// balancer to forward this error to the EDS balancer and eventually the
// channel needs to be put in a bad state.
xdsC.InvokeWatchClusterCallback(cdsUpdateWithMissingSecurityCfg, nil)
// We manually check that an error is forwarded to the EDS balancer instead
// of using one of the helper methods on the testEDSBalancer, because all we
// care here is whether an error is sent to it or not. We don't care about
// the exact error.
gotErr, err := edsB.resolverErrCh.Receive(ctx)
if err != nil {
t.Fatal("timeout waiting for CDS balancer to forward error to EDS balancer upon receipt of bad security config")
}
if gotErr == nil {
t.Fatal("CDS balancer did not forward error to EDS balancer upon receipt of bad security config")
}
// Since the error being pushed here is not a resource-not-found-error, the
// registered watch should not be cancelled.
sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
defer sCancel()
if _, err := xdsC.WaitForCancelClusterWatch(sCtx); err != context.DeadlineExceeded {
t.Fatal("cluster watch cancelled for a non-resource-not-found-error")
}
}
// TestSecurityConfigUpdate_GoodToGood tests the case where the CDS balancer
// receives two different but successful updates with security configuration.
// Verifies that appropriate providers are created, and that address attributes
// are added.
func (s) TestSecurityConfigUpdate_GoodToGood(t *testing.T) {
// This creates a CDS balancer which uses xdsCredentials, pushes a
// ClientConnState update with a fake xdsClient, and makes sure that the CDS
// balancer registers a watch on the provided xdsClient.
xdsC, cdsB, edsB, tcc, cancel := setupWithXDSCreds(t)
defer func() {
cancel()
cdsB.Close()
}()
// Override the provider builder function to push on a channel.
providerCh := testutils.NewChannel()
origBuildProvider := buildProvider
buildProvider = func(c map[string]*certprovider.BuildableConfig, id, cert string, wi, wr bool) (certprovider.Provider, error) {
p, err := origBuildProvider(c, id, cert, wi, wr)
providerCh.Send(nil)
return p, err
}
defer func() { buildProvider = origBuildProvider }()
// Set the bootstrap config used by the fake client.
xdsC.SetBootstrapConfig(bootstrapConfig)
// Here we invoke the watch callback registered on the fake xdsClient. This
// will trigger the watch handler on the CDS balancer, which will attempt to
// create a new EDS balancer. The fake EDS balancer created above will be
// returned to the CDS balancer, because we have overridden the
// newChildBalancer function as part of test setup.
cdsUpdate := xdsclient.ClusterUpdate{
ClusterName: serviceName,
SecurityCfg: &xdsclient.SecurityConfig{
RootInstanceName: "default1",
SubjectAltNameMatchers: testSANMatchers,
},
}
wantCCS := edsCCS(serviceName, nil, false)
ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer ctxCancel()
if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
t.Fatal(err)
}
// We specified only the root provider. So, expect only one provider here.
if _, err := providerCh.Receive(ctx); err != nil {
t.Fatalf("Failed to create certificate provider upon receipt of security config")
}
// Make a NewSubConn and verify that attributes are added.
if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
t.Fatal(err)
}
// Push another update with a new security configuration.
cdsUpdate = xdsclient.ClusterUpdate{
ClusterName: serviceName,
SecurityCfg: &xdsclient.SecurityConfig{
RootInstanceName: "default2",
SubjectAltNameMatchers: testSANMatchers,
},
}
if err := invokeWatchCbAndWait(ctx, xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil {
t.Fatal(err)
}
// We specified only the root provider. So, expect only one provider here.
if _, err := providerCh.Receive(ctx); err != nil {
t.Fatalf("Failed to create certificate provider upon receipt of security config")
}
// Make a NewSubConn and verify that attributes are added.
if _, err := makeNewSubConn(ctx, edsB.parentCC, tcc, false); err != nil {
t.Fatal(err)
}
// The HandshakeInfo type does not expose its internals. So, we cannot
// verify that the HandshakeInfo carried by the attributes have actually
// been changed. This will be covered in e2e/interop tests.
// TODO(easwars): Remove this TODO once appropriate e2e/intertop tests have
// been added.
}