From b876bea0980a1e8af4d3c107c616b01e5d2b6da5 Mon Sep 17 00:00:00 2001 From: Terry Wilson Date: Thu, 18 Aug 2022 09:38:34 -0700 Subject: [PATCH 1/2] xds: Configure outlier detection. --- .../util/OutlierDetectionLoadBalancer.java | 34 +- .../OutlierDetectionLoadBalancerProvider.java | 2 + .../java/io/grpc/xds/CdsLoadBalancer2.java | 2 +- .../java/io/grpc/xds/ClientXdsClient.java | 78 ++++- .../grpc/xds/ClusterResolverLoadBalancer.java | 115 ++++++- .../ClusterResolverLoadBalancerProvider.java | 14 +- .../io/grpc/xds/EnvoyServerProtoData.java | 145 +++++++++ xds/src/main/java/io/grpc/xds/XdsClient.java | 16 +- .../io/grpc/xds/CdsLoadBalancer2Test.java | 82 ++--- .../io/grpc/xds/ClientXdsClientTestBase.java | 292 ++++++++++++++++-- .../io/grpc/xds/ClientXdsClientV2Test.java | 12 +- .../io/grpc/xds/ClientXdsClientV3Test.java | 12 +- .../xds/ClusterResolverLoadBalancerTest.java | 102 +++++- 13 files changed, 789 insertions(+), 117 deletions(-) diff --git a/core/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java b/core/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java index 3fb32fdb078..66380b9631b 100644 --- a/core/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java +++ b/core/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java @@ -31,6 +31,7 @@ import io.grpc.ConnectivityState; import io.grpc.ConnectivityStateInfo; import io.grpc.EquivalentAddressGroup; +import io.grpc.Internal; import io.grpc.LoadBalancer; import io.grpc.Metadata; import io.grpc.Status; @@ -58,7 +59,8 @@ *

This implements the outlier detection gRFC: * https://github.com/grpc/proposal/blob/master/A50-xds-outlier-detection.md */ -public class OutlierDetectionLoadBalancer extends LoadBalancer { +@Internal +public final class OutlierDetectionLoadBalancer extends LoadBalancer { @VisibleForTesting final AddressTrackerMap trackerMap; @@ -837,13 +839,13 @@ private static boolean hasSingleAddress(List addressGrou */ public static final class OutlierDetectionLoadBalancerConfig { - final Long intervalNanos; - final Long baseEjectionTimeNanos; - final Long maxEjectionTimeNanos; - final Integer maxEjectionPercent; - final SuccessRateEjection successRateEjection; - final FailurePercentageEjection failurePercentageEjection; - final PolicySelection childPolicy; + public final Long intervalNanos; + public final Long baseEjectionTimeNanos; + public final Long maxEjectionTimeNanos; + public final Integer maxEjectionPercent; + public final SuccessRateEjection successRateEjection; + public final FailurePercentageEjection failurePercentageEjection; + public final PolicySelection childPolicy; private OutlierDetectionLoadBalancerConfig(Long intervalNanos, Long baseEjectionTimeNanos, @@ -932,10 +934,10 @@ public OutlierDetectionLoadBalancerConfig build() { /** The configuration for success rate ejection. */ public static class SuccessRateEjection { - final Integer stdevFactor; - final Integer enforcementPercentage; - final Integer minimumHosts; - final Integer requestVolume; + public final Integer stdevFactor; + public final Integer enforcementPercentage; + public final Integer minimumHosts; + public final Integer requestVolume; SuccessRateEjection(Integer stdevFactor, Integer enforcementPercentage, Integer minimumHosts, Integer requestVolume) { @@ -996,10 +998,10 @@ public SuccessRateEjection build() { /** The configuration for failure percentage ejection. */ public static class FailurePercentageEjection { - final Integer threshold; - final Integer enforcementPercentage; - final Integer minimumHosts; - final Integer requestVolume; + public final Integer threshold; + public final Integer enforcementPercentage; + public final Integer minimumHosts; + public final Integer requestVolume; FailurePercentageEjection(Integer threshold, Integer enforcementPercentage, Integer minimumHosts, Integer requestVolume) { diff --git a/core/src/main/java/io/grpc/util/OutlierDetectionLoadBalancerProvider.java b/core/src/main/java/io/grpc/util/OutlierDetectionLoadBalancerProvider.java index a92f49bd1d2..e52c7414653 100644 --- a/core/src/main/java/io/grpc/util/OutlierDetectionLoadBalancerProvider.java +++ b/core/src/main/java/io/grpc/util/OutlierDetectionLoadBalancerProvider.java @@ -16,6 +16,7 @@ package io.grpc.util; +import io.grpc.Internal; import io.grpc.LoadBalancer; import io.grpc.LoadBalancer.Helper; import io.grpc.LoadBalancerProvider; @@ -33,6 +34,7 @@ import java.util.List; import java.util.Map; +@Internal public final class OutlierDetectionLoadBalancerProvider extends LoadBalancerProvider { @Override diff --git a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java index 4a599ef232c..4fa701c3c42 100644 --- a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java +++ b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java @@ -159,7 +159,7 @@ private void handleClusterDiscovered() { instance = DiscoveryMechanism.forEds( clusterState.name, clusterState.result.edsServiceName(), clusterState.result.lrsServerInfo(), clusterState.result.maxConcurrentRequests(), - clusterState.result.upstreamTlsContext()); + clusterState.result.upstreamTlsContext(), clusterState.result.outlierDetection()); } else { // logical DNS instance = DiscoveryMechanism.forLogicalDns( clusterState.name, clusterState.result.dnsHostName(), diff --git a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java index 9a4c366f84f..208b9a2852f 100644 --- a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java +++ b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java @@ -90,6 +90,7 @@ import io.grpc.xds.EnvoyServerProtoData.ConnectionSourceType; import io.grpc.xds.EnvoyServerProtoData.FilterChain; import io.grpc.xds.EnvoyServerProtoData.FilterChainMatch; +import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.Filter.ClientInterceptorBuilder; import io.grpc.xds.Filter.FilterConfig; @@ -166,6 +167,10 @@ final class ClientXdsClient extends XdsClient implements XdsResponseHandler, Res static boolean enableCustomLbConfig = Strings.isNullOrEmpty(System.getenv("GRPC_EXPERIMENTAL_XDS_CUSTOM_LB_CONFIG")) || Boolean.parseBoolean(System.getenv("GRPC_EXPERIMENTAL_XDS_CUSTOM_LB_CONFIG")); + @VisibleForTesting + static boolean enableOutlierDetection = + !Strings.isNullOrEmpty(System.getenv("GRPC_EXPERIMENTAL_XDS_OUTLIER_DETECTION")) + || Boolean.parseBoolean(System.getenv("GRPC_EXPERIMENTAL_XDS_OUTLIER_DETECTION")); private static final String TYPE_URL_HTTP_CONNECTION_MANAGER_V2 = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2" + ".HttpConnectionManager"; @@ -632,6 +637,65 @@ static void validateCommonTlsContext( } } + static io.envoyproxy.envoy.config.cluster.v3.OutlierDetection validateOutlierDetection( + io.envoyproxy.envoy.config.cluster.v3.OutlierDetection outlierDetection) + throws ResourceInvalidException { + if (outlierDetection.hasInterval()) { + if (!Durations.isValid(outlierDetection.getInterval())) { + throw new ResourceInvalidException("outlier_detection interval is not a valid Duration"); + } + if (hasNegativeValues(outlierDetection.getInterval())) { + throw new ResourceInvalidException("outlier_detection interval has a negative value"); + } + } + if (outlierDetection.hasBaseEjectionTime()) { + if (!Durations.isValid(outlierDetection.getBaseEjectionTime())) { + throw new ResourceInvalidException( + "outlier_detection base_ejection_time is not a valid Duration"); + } + if (hasNegativeValues(outlierDetection.getBaseEjectionTime())) { + throw new ResourceInvalidException( + "outlier_detection base_ejection_time has a negative value"); + } + } + if (outlierDetection.hasMaxEjectionTime()) { + if (!Durations.isValid(outlierDetection.getMaxEjectionTime())) { + throw new ResourceInvalidException( + "outlier_detection max_ejection_time is not a valid Duration"); + } + if (hasNegativeValues(outlierDetection.getMaxEjectionTime())) { + throw new ResourceInvalidException( + "outlier_detection max_ejection_time has a negative value"); + } + } + if (outlierDetection.hasMaxEjectionPercent() + && outlierDetection.getMaxEjectionPercent().getValue() > 100) { + throw new ResourceInvalidException( + "outlier_detection max_ejection_percent is > 100"); + } + if (outlierDetection.hasEnforcingSuccessRate() + && outlierDetection.getEnforcingSuccessRate().getValue() > 100) { + throw new ResourceInvalidException( + "outlier_detection enforcing_success_rate is > 100"); + } + if (outlierDetection.hasFailurePercentageThreshold() + && outlierDetection.getFailurePercentageThreshold().getValue() > 100) { + throw new ResourceInvalidException( + "outlier_detection failure_percentage_threshold is > 100"); + } + if (outlierDetection.hasEnforcingFailurePercentage() + && outlierDetection.getEnforcingFailurePercentage().getValue() > 100) { + throw new ResourceInvalidException( + "outlier_detection enforcing_failure_percentage is > 100"); + } + + return outlierDetection; + } + + static boolean hasNegativeValues(Duration duration) { + return duration.getSeconds() < 0 || duration.getNanos() < 0; + } + private static String getIdentityCertInstanceName(CommonTlsContext commonTlsContext) { if (commonTlsContext.hasTlsCertificateProviderInstance()) { return commonTlsContext.getTlsCertificateProviderInstance().getInstanceName(); @@ -1704,6 +1768,7 @@ private static StructOrError parseNonAggregateCluster( ServerInfo lrsServerInfo = null; Long maxConcurrentRequests = null; UpstreamTlsContext upstreamTlsContext = null; + OutlierDetection outlierDetection = null; if (cluster.hasLrsServer()) { if (!cluster.getLrsServer().hasSelf()) { return StructOrError.fromError( @@ -1743,6 +1808,16 @@ private static StructOrError parseNonAggregateCluster( "Cluster " + clusterName + ": malformed UpstreamTlsContext: " + e); } } + if (cluster.hasOutlierDetection() && enableOutlierDetection) { + try { + outlierDetection = OutlierDetection.fromEnvoyOutlierDetection( + validateOutlierDetection(cluster.getOutlierDetection())); + } catch (ResourceInvalidException e) { + return StructOrError.fromError( + "Cluster " + clusterName + ": malformed outlier_detection: " + e); + } + } + DiscoveryType type = cluster.getType(); if (type == DiscoveryType.EDS) { @@ -1763,7 +1838,8 @@ private static StructOrError parseNonAggregateCluster( edsResources.add(clusterName); } return StructOrError.fromStruct(CdsUpdate.forEds( - clusterName, edsServiceName, lrsServerInfo, maxConcurrentRequests, upstreamTlsContext)); + clusterName, edsServiceName, lrsServerInfo, maxConcurrentRequests, upstreamTlsContext, + outlierDetection)); } else if (type.equals(DiscoveryType.LOGICAL_DNS)) { if (!cluster.hasLoadAssignment()) { return StructOrError.fromError( diff --git a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java index 91da43eb24a..96e2cf441a0 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java @@ -38,6 +38,7 @@ import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.util.ForwardingLoadBalancerHelper; import io.grpc.util.GracefulSwitchLoadBalancer; +import io.grpc.util.OutlierDetectionLoadBalancer.OutlierDetectionLoadBalancerConfig; import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.ClusterImplLoadBalancerProvider.ClusterImplConfig; import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig; @@ -45,6 +46,9 @@ import io.grpc.xds.Endpoints.DropOverload; import io.grpc.xds.Endpoints.LbEndpoint; import io.grpc.xds.Endpoints.LocalityLbEndpoints; +import io.grpc.xds.EnvoyServerProtoData.FailurePercentageEjection; +import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; +import io.grpc.xds.EnvoyServerProtoData.SuccessRateEjection; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig.PriorityChildConfig; @@ -176,7 +180,8 @@ public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { ClusterState state; if (instance.type == DiscoveryMechanism.Type.EDS) { state = new EdsClusterState(instance.cluster, instance.edsServiceName, - instance.lrsServerInfo, instance.maxConcurrentRequests, instance.tlsContext); + instance.lrsServerInfo, instance.maxConcurrentRequests, instance.tlsContext, + instance.outlierDetection); } else { // logical DNS state = new LogicalDnsClusterState(instance.cluster, instance.dnsHostName, instance.lrsServerInfo, instance.maxConcurrentRequests, instance.tlsContext); @@ -316,6 +321,8 @@ private abstract class ClusterState { protected final Long maxConcurrentRequests; @Nullable protected final UpstreamTlsContext tlsContext; + @Nullable + protected final OutlierDetection outlierDetection; // Resolution status, may contain most recent error encountered. protected Status status = Status.OK; // True if has received resolution result. @@ -327,11 +334,13 @@ private abstract class ClusterState { protected boolean shutdown; private ClusterState(String name, @Nullable ServerInfo lrsServerInfo, - @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext) { + @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext, + @Nullable OutlierDetection outlierDetection) { this.name = name; this.lrsServerInfo = lrsServerInfo; this.maxConcurrentRequests = maxConcurrentRequests; this.tlsContext = tlsContext; + this.outlierDetection = outlierDetection; } abstract void start(); @@ -349,8 +358,8 @@ private final class EdsClusterState extends ClusterState implements EdsResourceW private EdsClusterState(String name, @Nullable String edsServiceName, @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, - @Nullable UpstreamTlsContext tlsContext) { - super(name, lrsServerInfo, maxConcurrentRequests, tlsContext); + @Nullable UpstreamTlsContext tlsContext, @Nullable OutlierDetection outlierDetection) { + super(name, lrsServerInfo, maxConcurrentRequests, tlsContext, outlierDetection); this.edsServiceName = edsServiceName; } @@ -434,7 +443,8 @@ public void run() { Map priorityChildConfigs = generateEdsBasedPriorityChildConfigs( name, edsServiceName, lrsServerInfo, maxConcurrentRequests, tlsContext, - endpointLbPolicy, lbRegistry, prioritizedLocalityWeights, dropOverloads); + outlierDetection, endpointLbPolicy, lbRegistry, prioritizedLocalityWeights, + dropOverloads); status = Status.OK; resolved = true; result = new ClusterResolutionResult(addresses, priorityChildConfigs, @@ -530,7 +540,7 @@ private final class LogicalDnsClusterState extends ClusterState { private LogicalDnsClusterState(String name, String dnsHostName, @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext) { - super(name, lrsServerInfo, maxConcurrentRequests, tlsContext); + super(name, lrsServerInfo, maxConcurrentRequests, tlsContext, null); this.dnsHostName = checkNotNull(dnsHostName, "dnsHostName"); nameResolverFactory = checkNotNull(helper.getNameResolverRegistry().asFactory(), "nameResolverFactory"); @@ -730,9 +740,9 @@ private static PriorityChildConfig generateDnsBasedPriorityChildConfig( private static Map generateEdsBasedPriorityChildConfigs( String cluster, @Nullable String edsServiceName, @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext, - PolicySelection endpointLbPolicy, LoadBalancerRegistry lbRegistry, - Map> prioritizedLocalityWeights, - List dropOverloads) { + @Nullable OutlierDetection outlierDetection, PolicySelection endpointLbPolicy, + LoadBalancerRegistry lbRegistry, Map> prioritizedLocalityWeights, List dropOverloads) { Map configs = new HashMap<>(); for (String priority : prioritizedLocalityWeights.keySet()) { ClusterImplConfig clusterImplConfig = @@ -740,15 +750,98 @@ private static Map generateEdsBasedPriorityChildCon dropOverloads, endpointLbPolicy, tlsContext); LoadBalancerProvider clusterImplLbProvider = lbRegistry.getProvider(XdsLbPolicies.CLUSTER_IMPL_POLICY_NAME); - PolicySelection clusterImplPolicy = + PolicySelection priorityChildPolicy = new PolicySelection(clusterImplLbProvider, clusterImplConfig); + + // If outlier detection has been configured we wrap the child policy in the outlier detection + // load balancer. + if (outlierDetection != null) { + LoadBalancerProvider outlierDetectionProvider = lbRegistry.getProvider( + "outlier_detection_experimental"); + priorityChildPolicy = new PolicySelection(outlierDetectionProvider, + buildOutlierDetectionLbConfig(outlierDetection, priorityChildPolicy)); + } + PriorityChildConfig priorityChildConfig = - new PriorityChildConfig(clusterImplPolicy, true /* ignoreReresolution */); + new PriorityChildConfig(priorityChildPolicy, true /* ignoreReresolution */); configs.put(priority, priorityChildConfig); } return configs; } + /** + * Converts {@link OutlierDetection} that represents the xDS configuration to {@link + * OutlierDetectionLoadBalancerConfig} that the {@link io.grpc.util.OutlierDetectionLoadBalancer} + * understands. + */ + private static OutlierDetectionLoadBalancerConfig buildOutlierDetectionLbConfig( + OutlierDetection outlierDetection, PolicySelection childPolicy) { + OutlierDetectionLoadBalancerConfig.Builder configBuilder + = new OutlierDetectionLoadBalancerConfig.Builder(); + + configBuilder.setChildPolicy(childPolicy); + + if (outlierDetection.intervalNanos() != null) { + configBuilder.setIntervalNanos(outlierDetection.intervalNanos()); + } + if (outlierDetection.baseEjectionTimeNanos() != null) { + configBuilder.setBaseEjectionTimeNanos(outlierDetection.baseEjectionTimeNanos()); + } + if (outlierDetection.maxEjectionTimeNanos() != null) { + configBuilder.setMaxEjectionTimeNanos(outlierDetection.maxEjectionTimeNanos()); + } + if (outlierDetection.maxEjectionPercent() != null) { + configBuilder.setMaxEjectionPercent(outlierDetection.maxEjectionPercent()); + } + + SuccessRateEjection successRate = outlierDetection.successRateEjection(); + if (successRate != null) { + OutlierDetectionLoadBalancerConfig.SuccessRateEjection.Builder + successRateConfigBuilder = new OutlierDetectionLoadBalancerConfig + .SuccessRateEjection.Builder(); + + if (successRate.stdevFactor() != null) { + successRateConfigBuilder.setStdevFactor(successRate.stdevFactor()); + } + if (successRate.enforcementPercentage() != null) { + successRateConfigBuilder.setEnforcementPercentage(successRate.enforcementPercentage()); + } + if (successRate.minimumHosts() != null) { + successRateConfigBuilder.setMinimumHosts(successRate.minimumHosts()); + } + if (successRate.requestVolume() != null) { + successRateConfigBuilder.setRequestVolume(successRate.requestVolume()); + } + + configBuilder.setSuccessRateEjection(successRateConfigBuilder.build()); + } + + FailurePercentageEjection failurePercentage = outlierDetection.failurePercentageEjection(); + if (failurePercentage != null) { + OutlierDetectionLoadBalancerConfig.FailurePercentageEjection.Builder + failurePercentageConfigBuilder = new OutlierDetectionLoadBalancerConfig + .FailurePercentageEjection.Builder(); + + if (failurePercentage.threshold() != null) { + failurePercentageConfigBuilder.setThreshold(failurePercentage.threshold()); + } + if (failurePercentage.enforcementPercentage() != null) { + failurePercentageConfigBuilder.setEnforcementPercentage( + failurePercentage.enforcementPercentage()); + } + if (failurePercentage.minimumHosts() != null) { + failurePercentageConfigBuilder.setMinimumHosts(failurePercentage.minimumHosts()); + } + if (failurePercentage.requestVolume() != null) { + failurePercentageConfigBuilder.setRequestVolume(failurePercentage.requestVolume()); + } + + configBuilder.setFailurePercentageEjection(failurePercentageConfigBuilder.build()); + } + + return configBuilder.build(); + } + /** * Generates a string that represents the priority in the LB policy config. The string is unique * across priorities in all clusters and priorityName(c, p1) < priorityName(c, p2) iff p1 < p2. diff --git a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancerProvider.java b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancerProvider.java index 6f6f887e925..38da1f465c1 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancerProvider.java +++ b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancerProvider.java @@ -26,6 +26,7 @@ import io.grpc.NameResolver.ConfigOrError; import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.xds.Bootstrapper.ServerInfo; +import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import java.util.List; import java.util.Map; @@ -124,6 +125,8 @@ static final class DiscoveryMechanism { // Hostname for resolving endpoints via DNS. Only valid for LOGICAL_DNS clusters. @Nullable final String dnsHostName; + @Nullable + final OutlierDetection outlierDetection; enum Type { EDS, @@ -132,7 +135,8 @@ enum Type { private DiscoveryMechanism(String cluster, Type type, @Nullable String edsServiceName, @Nullable String dnsHostName, @Nullable ServerInfo lrsServerInfo, - @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext) { + @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext, + @Nullable OutlierDetection outlierDetection) { this.cluster = checkNotNull(cluster, "cluster"); this.type = checkNotNull(type, "type"); this.edsServiceName = edsServiceName; @@ -140,20 +144,22 @@ private DiscoveryMechanism(String cluster, Type type, @Nullable String edsServic this.lrsServerInfo = lrsServerInfo; this.maxConcurrentRequests = maxConcurrentRequests; this.tlsContext = tlsContext; + this.outlierDetection = outlierDetection; } static DiscoveryMechanism forEds(String cluster, @Nullable String edsServiceName, @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, - @Nullable UpstreamTlsContext tlsContext) { + @Nullable UpstreamTlsContext tlsContext, + OutlierDetection outlierDetection) { return new DiscoveryMechanism(cluster, Type.EDS, edsServiceName, null, lrsServerInfo, - maxConcurrentRequests, tlsContext); + maxConcurrentRequests, tlsContext, outlierDetection); } static DiscoveryMechanism forLogicalDns(String cluster, String dnsHostName, @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext) { return new DiscoveryMechanism(cluster, Type.LOGICAL_DNS, null, dnsHostName, - lrsServerInfo, maxConcurrentRequests, tlsContext); + lrsServerInfo, maxConcurrentRequests, tlsContext, null); } @Override diff --git a/xds/src/main/java/io/grpc/xds/EnvoyServerProtoData.java b/xds/src/main/java/io/grpc/xds/EnvoyServerProtoData.java index e53439755be..fad5700f5b1 100644 --- a/xds/src/main/java/io/grpc/xds/EnvoyServerProtoData.java +++ b/xds/src/main/java/io/grpc/xds/EnvoyServerProtoData.java @@ -19,6 +19,7 @@ import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; +import com.google.protobuf.util.Durations; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; import io.grpc.Internal; import io.grpc.xds.internal.sds.SslContextProviderSupplier; @@ -254,4 +255,148 @@ static Listener create( defaultFilterChain); } } + + /** + * Corresponds to Envoy proto message {@link + * io.envoyproxy.envoy.config.cluster.v3.OutlierDetection}. Only the fields supported by gRPC are + * included. + * + *

Protobuf Duration fields are represented in their string format (e.g. "10s"). + */ + @AutoValue + abstract static class OutlierDetection { + + @Nullable + abstract Long intervalNanos(); + + @Nullable + abstract Long baseEjectionTimeNanos(); + + @Nullable + abstract Long maxEjectionTimeNanos(); + + @Nullable + abstract Integer maxEjectionPercent(); + + @Nullable + abstract SuccessRateEjection successRateEjection(); + + @Nullable + abstract FailurePercentageEjection failurePercentageEjection(); + + static OutlierDetection create( + @Nullable Long intervalNanos, + @Nullable Long baseEjectionTimeNanos, + @Nullable Long maxEjectionTimeNanos, + @Nullable Integer maxEjectionPercentage, + @Nullable SuccessRateEjection successRateEjection, + @Nullable FailurePercentageEjection failurePercentageEjection) { + return new AutoValue_EnvoyServerProtoData_OutlierDetection(intervalNanos, + baseEjectionTimeNanos, maxEjectionTimeNanos, maxEjectionPercentage, successRateEjection, + failurePercentageEjection); + } + + static OutlierDetection fromEnvoyOutlierDetection( + io.envoyproxy.envoy.config.cluster.v3.OutlierDetection envoyOutlierDetection) { + + Long intervalNanos = envoyOutlierDetection.hasInterval() + ? Durations.toNanos(envoyOutlierDetection.getInterval()) : null; + Long baseEjectionTimeNanos = envoyOutlierDetection.hasBaseEjectionTime() + ? Durations.toNanos(envoyOutlierDetection.getBaseEjectionTime()) : null; + Long maxEjectionTimeNanos = envoyOutlierDetection.hasMaxEjectionTime() + ? Durations.toNanos(envoyOutlierDetection.getMaxEjectionTime()) : null; + Integer maxEjectionPercentage = envoyOutlierDetection.hasMaxEjectionPercent() + ? envoyOutlierDetection.getMaxEjectionPercent().getValue() : null; + + SuccessRateEjection successRateEjection; + // If success rate enforcement has been turned completely off, don't configure this ejection. + if (envoyOutlierDetection.hasEnforcingSuccessRate() + && envoyOutlierDetection.getEnforcingSuccessRate().getValue() == 0) { + successRateEjection = null; + } else { + Integer stdevFactor = envoyOutlierDetection.hasSuccessRateStdevFactor() + ? envoyOutlierDetection.getSuccessRateStdevFactor().getValue() : null; + Integer enforcementPercentage = envoyOutlierDetection.hasEnforcingSuccessRate() + ? envoyOutlierDetection.getEnforcingSuccessRate().getValue() : null; + Integer minimumHosts = envoyOutlierDetection.hasSuccessRateMinimumHosts() + ? envoyOutlierDetection.getSuccessRateMinimumHosts().getValue() : null; + Integer requestVolume = envoyOutlierDetection.hasSuccessRateRequestVolume() + ? envoyOutlierDetection.getSuccessRateMinimumHosts().getValue() : null; + + successRateEjection = SuccessRateEjection.create(stdevFactor, enforcementPercentage, + minimumHosts, requestVolume); + } + + FailurePercentageEjection failurePercentageEjection; + if (envoyOutlierDetection.hasEnforcingFailurePercentage() + && envoyOutlierDetection.getEnforcingFailurePercentage().getValue() == 0) { + failurePercentageEjection = null; + } else { + Integer threshold = envoyOutlierDetection.hasFailurePercentageThreshold() + ? envoyOutlierDetection.getFailurePercentageThreshold().getValue() : null; + Integer enforcementPercentage = envoyOutlierDetection.hasEnforcingFailurePercentage() + ? envoyOutlierDetection.getEnforcingFailurePercentage().getValue() : null; + Integer minimumHosts = envoyOutlierDetection.hasFailurePercentageMinimumHosts() + ? envoyOutlierDetection.getFailurePercentageMinimumHosts().getValue() : null; + Integer requestVolume = envoyOutlierDetection.hasFailurePercentageRequestVolume() + ? envoyOutlierDetection.getFailurePercentageRequestVolume().getValue() : null; + + failurePercentageEjection = FailurePercentageEjection.create(threshold, + enforcementPercentage, minimumHosts, requestVolume); + } + + return create(intervalNanos, baseEjectionTimeNanos, maxEjectionTimeNanos, + maxEjectionPercentage, successRateEjection, failurePercentageEjection); + } + } + + @AutoValue + abstract static class SuccessRateEjection { + + @Nullable + abstract Integer stdevFactor(); + + @Nullable + abstract Integer enforcementPercentage(); + + @Nullable + abstract Integer minimumHosts(); + + @Nullable + abstract Integer requestVolume(); + + static SuccessRateEjection create( + @Nullable Integer stdevFactor, + @Nullable Integer enforcementPercentage, + @Nullable Integer minimumHosts, + @Nullable Integer requestVolume) { + return new AutoValue_EnvoyServerProtoData_SuccessRateEjection(stdevFactor, + enforcementPercentage, minimumHosts, requestVolume); + } + } + + @AutoValue + abstract static class FailurePercentageEjection { + + @Nullable + abstract Integer threshold(); + + @Nullable + abstract Integer enforcementPercentage(); + + @Nullable + abstract Integer minimumHosts(); + + @Nullable + abstract Integer requestVolume(); + + static FailurePercentageEjection create( + @Nullable Integer threshold, + @Nullable Integer enforcementPercentage, + @Nullable Integer minimumHosts, + @Nullable Integer requestVolume) { + return new AutoValue_EnvoyServerProtoData_FailurePercentageEjection(threshold, + enforcementPercentage, minimumHosts, requestVolume); + } + } } diff --git a/xds/src/main/java/io/grpc/xds/XdsClient.java b/xds/src/main/java/io/grpc/xds/XdsClient.java index bdffc361193..028ae155ba6 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/XdsClient.java @@ -34,6 +34,7 @@ import io.grpc.xds.Endpoints.DropOverload; import io.grpc.xds.Endpoints.LocalityLbEndpoints; import io.grpc.xds.EnvoyServerProtoData.Listener; +import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.LoadStatsManager2.ClusterDropStats; import io.grpc.xds.LoadStatsManager2.ClusterLocalityStats; @@ -221,6 +222,10 @@ abstract static class CdsUpdate implements ResourceUpdate { @Nullable abstract ImmutableList prioritizedClusterNames(); + // Outlier detection configuration. + @Nullable + abstract OutlierDetection outlierDetection(); + static Builder forAggregate(String clusterName, List prioritizedClusterNames) { checkNotNull(prioritizedClusterNames, "prioritizedClusterNames"); return new AutoValue_XdsClient_CdsUpdate.Builder() @@ -234,7 +239,8 @@ static Builder forAggregate(String clusterName, List prioritizedClusterN static Builder forEds(String clusterName, @Nullable String edsServiceName, @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, - @Nullable UpstreamTlsContext upstreamTlsContext) { + @Nullable UpstreamTlsContext upstreamTlsContext, + @Nullable OutlierDetection outlierDetection) { return new AutoValue_XdsClient_CdsUpdate.Builder() .clusterName(clusterName) .clusterType(ClusterType.EDS) @@ -244,7 +250,8 @@ static Builder forEds(String clusterName, @Nullable String edsServiceName, .edsServiceName(edsServiceName) .lrsServerInfo(lrsServerInfo) .maxConcurrentRequests(maxConcurrentRequests) - .upstreamTlsContext(upstreamTlsContext); + .upstreamTlsContext(upstreamTlsContext) + .outlierDetection(outlierDetection); } static Builder forLogicalDns(String clusterName, String dnsHostName, @@ -284,8 +291,9 @@ public final String toString() { .add("dnsHostName", dnsHostName()) .add("lrsServerInfo", lrsServerInfo()) .add("maxConcurrentRequests", maxConcurrentRequests()) - // Exclude upstreamTlsContext as its string representation is cumbersome. .add("prioritizedClusterNames", prioritizedClusterNames()) + // Exclude upstreamTlsContext and outlierDetection as their string representations are + // cumbersome. .toString(); } @@ -341,6 +349,8 @@ Builder leastRequestLbPolicy(Integer choiceCount) { // Private, use CdsUpdate.forAggregate() instead. protected abstract Builder prioritizedClusterNames(List prioritizedClusterNames); + protected abstract Builder outlierDetection(OutlierDetection outlierDetection); + abstract CdsUpdate build(); } } diff --git a/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java b/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java index a90df303e04..128d7bd395e 100644 --- a/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java +++ b/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java @@ -50,6 +50,8 @@ import io.grpc.xds.CdsLoadBalancerProvider.CdsConfig; import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig; import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig.DiscoveryMechanism; +import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; +import io.grpc.xds.EnvoyServerProtoData.SuccessRateEjection; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.LeastRequestLoadBalancer.LeastRequestConfig; import io.grpc.xds.RingHashLoadBalancer.RingHashConfig; @@ -85,6 +87,9 @@ public class CdsLoadBalancer2Test { ServerInfo.create("lrs.googleapis.com", InsecureChannelCredentials.create(), true); private final UpstreamTlsContext upstreamTlsContext = CommonTlsContextTestsUtil.buildUpstreamTlsContext("google_cloud_private_spiffe", true); + private final OutlierDetection outlierDetection = OutlierDetection.create( + null, null, null, null, SuccessRateEjection.create(null, null, null, null), null); + private final SynchronizationContext syncContext = new SynchronizationContext( new Thread.UncaughtExceptionHandler() { @@ -154,7 +159,8 @@ public void tearDown() { @Test public void discoverTopLevelEdsCluster() { CdsUpdate update = - CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext) + CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext, + outlierDetection) .roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(CLUSTER, update); assertThat(childBalancers).hasSize(1); @@ -164,7 +170,7 @@ public void discoverTopLevelEdsCluster() { assertThat(childLbConfig.discoveryMechanisms).hasSize(1); DiscoveryMechanism instance = Iterables.getOnlyElement(childLbConfig.discoveryMechanisms); assertDiscoveryMechanism(instance, CLUSTER, DiscoveryMechanism.Type.EDS, EDS_SERVICE_NAME, - null, LRS_SERVER_INFO, 100L, upstreamTlsContext); + null, LRS_SERVER_INFO, 100L, upstreamTlsContext, outlierDetection); assertThat(childLbConfig.lbPolicy.getProvider().getPolicyName()).isEqualTo("round_robin"); } @@ -181,7 +187,7 @@ public void discoverTopLevelLogicalDnsCluster() { assertThat(childLbConfig.discoveryMechanisms).hasSize(1); DiscoveryMechanism instance = Iterables.getOnlyElement(childLbConfig.discoveryMechanisms); assertDiscoveryMechanism(instance, CLUSTER, DiscoveryMechanism.Type.LOGICAL_DNS, null, - DNS_HOST_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext); + DNS_HOST_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext, null); assertThat(childLbConfig.lbPolicy.getProvider().getPolicyName()) .isEqualTo("least_request_experimental"); assertThat(((LeastRequestConfig) childLbConfig.lbPolicy.getConfig()).choiceCount).isEqualTo(3); @@ -201,7 +207,7 @@ public void nonAggregateCluster_resourceNotExist_returnErrorPicker() { @Test public void nonAggregateCluster_resourceUpdate() { CdsUpdate update = - CdsUpdate.forEds(CLUSTER, null, null, 100L, upstreamTlsContext) + CdsUpdate.forEds(CLUSTER, null, null, 100L, upstreamTlsContext, outlierDetection) .roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(CLUSTER, update); assertThat(childBalancers).hasSize(1); @@ -209,15 +215,15 @@ public void nonAggregateCluster_resourceUpdate() { ClusterResolverConfig childLbConfig = (ClusterResolverConfig) childBalancer.config; DiscoveryMechanism instance = Iterables.getOnlyElement(childLbConfig.discoveryMechanisms); assertDiscoveryMechanism(instance, CLUSTER, DiscoveryMechanism.Type.EDS, null, null, null, - 100L, upstreamTlsContext); + 100L, upstreamTlsContext, outlierDetection); - update = CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, null) - .roundRobinLbPolicy().build(); + update = CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, null, + outlierDetection).roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(CLUSTER, update); childLbConfig = (ClusterResolverConfig) childBalancer.config; instance = Iterables.getOnlyElement(childLbConfig.discoveryMechanisms); assertDiscoveryMechanism(instance, CLUSTER, DiscoveryMechanism.Type.EDS, EDS_SERVICE_NAME, - null, LRS_SERVER_INFO, 200L, null); + null, LRS_SERVER_INFO, 200L, null, outlierDetection); } @Test @@ -231,7 +237,7 @@ public void nonAggregateCluster_resourceRevoked() { ClusterResolverConfig childLbConfig = (ClusterResolverConfig) childBalancer.config; DiscoveryMechanism instance = Iterables.getOnlyElement(childLbConfig.discoveryMechanisms); assertDiscoveryMechanism(instance, CLUSTER, DiscoveryMechanism.Type.LOGICAL_DNS, null, - DNS_HOST_NAME, null, 100L, upstreamTlsContext); + DNS_HOST_NAME, null, 100L, upstreamTlsContext, null); xdsClient.deliverResourceNotExist(CLUSTER); assertThat(childBalancer.shutdown).isTrue(); @@ -265,9 +271,8 @@ public void discoverAggregateCluster() { assertThat(xdsClient.watchers.keySet()).containsExactly( CLUSTER, cluster1, cluster2, cluster3, cluster4); assertThat(childBalancers).isEmpty(); - CdsUpdate update3 = - CdsUpdate.forEds(cluster3, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, upstreamTlsContext) - .roundRobinLbPolicy().build(); + CdsUpdate update3 = CdsUpdate.forEds(cluster3, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, + upstreamTlsContext, outlierDetection).roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(cluster3, update3); assertThat(childBalancers).isEmpty(); CdsUpdate update2 = @@ -276,7 +281,7 @@ public void discoverAggregateCluster() { xdsClient.deliverCdsUpdate(cluster2, update2); assertThat(childBalancers).isEmpty(); CdsUpdate update4 = - CdsUpdate.forEds(cluster4, null, LRS_SERVER_INFO, 300L, null) + CdsUpdate.forEds(cluster4, null, LRS_SERVER_INFO, 300L, null, outlierDetection) .roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(cluster4, update4); assertThat(childBalancers).hasSize(1); // all non-aggregate clusters discovered @@ -286,12 +291,12 @@ public void discoverAggregateCluster() { assertThat(childLbConfig.discoveryMechanisms).hasSize(3); // Clusters on higher level has higher priority: [cluster2, cluster3, cluster4] assertDiscoveryMechanism(childLbConfig.discoveryMechanisms.get(0), cluster2, - DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, null, 100L, null); + DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, null, 100L, null, null); assertDiscoveryMechanism(childLbConfig.discoveryMechanisms.get(1), cluster3, DiscoveryMechanism.Type.EDS, EDS_SERVICE_NAME, null, LRS_SERVER_INFO, 200L, - upstreamTlsContext); + upstreamTlsContext, outlierDetection); assertDiscoveryMechanism(childLbConfig.discoveryMechanisms.get(2), cluster4, - DiscoveryMechanism.Type.EDS, null, null, LRS_SERVER_INFO, 300L, null); + DiscoveryMechanism.Type.EDS, null, null, LRS_SERVER_INFO, 300L, null, outlierDetection); assertThat(childLbConfig.lbPolicy.getProvider().getPolicyName()) .isEqualTo("ring_hash_experimental"); // dominated by top-level cluster's config assertThat(((RingHashConfig) childLbConfig.lbPolicy.getConfig()).minRingSize).isEqualTo(100L); @@ -326,9 +331,8 @@ public void aggregateCluster_descendantClustersRevoked() { .roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(CLUSTER, update); assertThat(xdsClient.watchers.keySet()).containsExactly(CLUSTER, cluster1, cluster2); - CdsUpdate update1 = - CdsUpdate.forEds(cluster1, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, upstreamTlsContext) - .roundRobinLbPolicy().build(); + CdsUpdate update1 = CdsUpdate.forEds(cluster1, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, + upstreamTlsContext, outlierDetection).roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(cluster1, update1); CdsUpdate update2 = CdsUpdate.forLogicalDns(cluster2, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null) @@ -339,9 +343,10 @@ public void aggregateCluster_descendantClustersRevoked() { assertThat(childLbConfig.discoveryMechanisms).hasSize(2); assertDiscoveryMechanism(childLbConfig.discoveryMechanisms.get(0), cluster1, DiscoveryMechanism.Type.EDS, EDS_SERVICE_NAME, null, LRS_SERVER_INFO, 200L, - upstreamTlsContext); + upstreamTlsContext, outlierDetection); assertDiscoveryMechanism(childLbConfig.discoveryMechanisms.get(1), cluster2, - DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null); + DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null, + null); // Revoke cluster1, should still be able to proceed with cluster2. xdsClient.deliverResourceNotExist(cluster1); @@ -349,7 +354,8 @@ public void aggregateCluster_descendantClustersRevoked() { childLbConfig = (ClusterResolverConfig) childBalancer.config; assertThat(childLbConfig.discoveryMechanisms).hasSize(1); assertDiscoveryMechanism(Iterables.getOnlyElement(childLbConfig.discoveryMechanisms), cluster2, - DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null); + DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null, + null); verify(helper, never()).updateBalancingState( eq(ConnectivityState.TRANSIENT_FAILURE), any(SubchannelPicker.class)); @@ -374,9 +380,8 @@ public void aggregateCluster_rootClusterRevoked() { .roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(CLUSTER, update); assertThat(xdsClient.watchers.keySet()).containsExactly(CLUSTER, cluster1, cluster2); - CdsUpdate update1 = - CdsUpdate.forEds(cluster1, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, upstreamTlsContext) - .roundRobinLbPolicy().build(); + CdsUpdate update1 = CdsUpdate.forEds(cluster1, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, + upstreamTlsContext, outlierDetection).roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(cluster1, update1); CdsUpdate update2 = CdsUpdate.forLogicalDns(cluster2, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null) @@ -387,9 +392,10 @@ public void aggregateCluster_rootClusterRevoked() { assertThat(childLbConfig.discoveryMechanisms).hasSize(2); assertDiscoveryMechanism(childLbConfig.discoveryMechanisms.get(0), cluster1, DiscoveryMechanism.Type.EDS, EDS_SERVICE_NAME, null, LRS_SERVER_INFO, 200L, - upstreamTlsContext); + upstreamTlsContext, outlierDetection); assertDiscoveryMechanism(childLbConfig.discoveryMechanisms.get(1), cluster2, - DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null); + DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null, + null); xdsClient.deliverResourceNotExist(CLUSTER); assertThat(xdsClient.watchers.keySet()) @@ -428,16 +434,15 @@ public void aggregateCluster_intermediateClusterChanges() { .roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(cluster2, update2); assertThat(xdsClient.watchers.keySet()).containsExactly(CLUSTER, cluster2, cluster3); - CdsUpdate update3 = - CdsUpdate.forEds(cluster3, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext) - .roundRobinLbPolicy().build(); + CdsUpdate update3 = CdsUpdate.forEds(cluster3, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, + upstreamTlsContext, outlierDetection).roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(cluster3, update3); FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); ClusterResolverConfig childLbConfig = (ClusterResolverConfig) childBalancer.config; assertThat(childLbConfig.discoveryMechanisms).hasSize(1); DiscoveryMechanism instance = Iterables.getOnlyElement(childLbConfig.discoveryMechanisms); assertDiscoveryMechanism(instance, cluster3, DiscoveryMechanism.Type.EDS, EDS_SERVICE_NAME, - null, LRS_SERVER_INFO, 100L, upstreamTlsContext); + null, LRS_SERVER_INFO, 100L, upstreamTlsContext, outlierDetection); // cluster2 revoked xdsClient.deliverResourceNotExist(cluster2); @@ -506,9 +511,8 @@ public void handleNameResolutionErrorFromUpstream_beforeChildLbCreated_returnErr @Test public void handleNameResolutionErrorFromUpstream_afterChildLbCreated_fallThrough() { - CdsUpdate update = - CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext) - .roundRobinLbPolicy().build(); + CdsUpdate update = CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, + upstreamTlsContext, outlierDetection).roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(CLUSTER, update); FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); assertThat(childBalancer.shutdown).isFalse(); @@ -523,7 +527,8 @@ public void handleNameResolutionErrorFromUpstream_afterChildLbCreated_fallThroug public void unknownLbProvider() { try { xdsClient.deliverCdsUpdate(CLUSTER, - CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext) + CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext, + outlierDetection) .lbPolicyConfig(ImmutableMap.of("unknown", ImmutableMap.of("foo", "bar"))).build()); } catch (Exception e) { assertThat(e).hasCauseThat().hasMessageThat().contains("No provider available"); @@ -536,8 +541,8 @@ public void unknownLbProvider() { public void invalidLbConfig() { try { xdsClient.deliverCdsUpdate(CLUSTER, - CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext) - .lbPolicyConfig( + CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext, + outlierDetection).lbPolicyConfig( ImmutableMap.of("ring_hash_experimental", ImmutableMap.of("minRingSize", "-1"))) .build()); } catch (Exception e) { @@ -561,7 +566,7 @@ private static void assertPicker(SubchannelPicker picker, Status expectedStatus, private static void assertDiscoveryMechanism(DiscoveryMechanism instance, String name, DiscoveryMechanism.Type type, @Nullable String edsServiceName, @Nullable String dnsHostName, @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, - @Nullable UpstreamTlsContext tlsContext) { + @Nullable UpstreamTlsContext tlsContext, @Nullable OutlierDetection outlierDetection) { assertThat(instance.cluster).isEqualTo(name); assertThat(instance.type).isEqualTo(type); assertThat(instance.edsServiceName).isEqualTo(edsServiceName); @@ -569,6 +574,7 @@ private static void assertDiscoveryMechanism(DiscoveryMechanism instance, String assertThat(instance.lrsServerInfo).isEqualTo(lrsServerInfo); assertThat(instance.maxConcurrentRequests).isEqualTo(maxConcurrentRequests); assertThat(instance.tlsContext).isEqualTo(tlsContext); + assertThat(instance.outlierDetection).isEqualTo(outlierDetection); } private final class FakeLoadBalancerProvider extends LoadBalancerProvider { diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java index 53b4027768f..97f892a0c27 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java @@ -37,8 +37,12 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.protobuf.Any; +import com.google.protobuf.Duration; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; +import com.google.protobuf.UInt32Value; +import com.google.protobuf.util.Durations; +import io.envoyproxy.envoy.config.cluster.v3.OutlierDetection; import io.envoyproxy.envoy.config.route.v3.FilterConfig; import io.envoyproxy.envoy.extensions.filters.http.router.v3.Router; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CertificateProviderPluginInstance; @@ -66,12 +70,15 @@ import io.grpc.xds.Bootstrapper.AuthorityInfo; import io.grpc.xds.Bootstrapper.CertificateProviderInfo; import io.grpc.xds.Bootstrapper.ServerInfo; +import io.grpc.xds.ClientXdsClient.ResourceInvalidException; import io.grpc.xds.ClientXdsClient.XdsChannelFactory; import io.grpc.xds.Endpoints.DropOverload; import io.grpc.xds.Endpoints.LbEndpoint; import io.grpc.xds.Endpoints.LocalityLbEndpoints; import io.grpc.xds.EnvoyProtoData.Node; +import io.grpc.xds.EnvoyServerProtoData.FailurePercentageEjection; import io.grpc.xds.EnvoyServerProtoData.FilterChain; +import io.grpc.xds.EnvoyServerProtoData.SuccessRateEjection; import io.grpc.xds.FaultConfig.FractionalPercent.DenominatorType; import io.grpc.xds.LoadStatsManager2.ClusterDropStats; import io.grpc.xds.XdsClient.CdsResourceWatcher; @@ -211,7 +218,7 @@ public long currentTimeNanos() { // CDS test resources. private final Any testClusterRoundRobin = Any.pack(mf.buildEdsCluster(CDS_RESOURCE, null, "round_robin", null, - null, false, null, "envoy.transport_sockets.tls", null + null, false, null, "envoy.transport_sockets.tls", null, null )); // EDS test resources. @@ -1021,7 +1028,7 @@ public void cdsResourceUpdated_withXdstpResourceName_withWrongType() { "xdstp://authority.xds.com/envoy.config.listener.v3.Listener/cluster1"; Any testClusterConfig = Any.pack(mf.buildEdsCluster( cdsResourceNameWithWrongType, null, "round_robin", null, null, false, null, - "envoy.transport_sockets.tls", null)); + "envoy.transport_sockets.tls", null, null)); call.sendResponse(CDS, testClusterConfig, VERSION_1, "0000"); call.verifyRequestNack( CDS, cdsResourceName, "", "0000", NODE, @@ -1615,9 +1622,9 @@ public void cdsResourceNotFound() { List clusters = ImmutableList.of( Any.pack(mf.buildEdsCluster("cluster-bar.googleapis.com", null, "round_robin", null, - null, false, null, "envoy.transport_sockets.tls", null)), + null, false, null, "envoy.transport_sockets.tls", null, null)), Any.pack(mf.buildEdsCluster("cluster-baz.googleapis.com", null, "round_robin", null, - null, false, null, "envoy.transport_sockets.tls", null))); + null, false, null, "envoy.transport_sockets.tls", null, null))); call.sendResponse(CDS, clusters, VERSION_1, "0000"); // Client sent an ACK CDS request. @@ -1692,13 +1699,13 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { // CDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( "A", Any.pack(mf.buildEdsCluster("A", "A.1", "round_robin", null, null, false, null, - "envoy.transport_sockets.tls", null + "envoy.transport_sockets.tls", null, null )), "B", Any.pack(mf.buildEdsCluster("B", "B.1", "round_robin", null, null, false, null, - "envoy.transport_sockets.tls", null + "envoy.transport_sockets.tls", null, null )), "C", Any.pack(mf.buildEdsCluster("C", "C.1", "round_robin", null, null, false, null, - "envoy.transport_sockets.tls", null + "envoy.transport_sockets.tls", null, null ))); call.sendResponse(CDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 @@ -1711,7 +1718,7 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { // Failed to parse endpoint B ImmutableMap resourcesV2 = ImmutableMap.of( "A", Any.pack(mf.buildEdsCluster("A", "A.2", "round_robin", null, null, false, null, - "envoy.transport_sockets.tls", null + "envoy.transport_sockets.tls", null, null )), "B", Any.pack(mf.buildClusterInvalid("B"))); call.sendResponse(CDS, resourcesV2.values().asList(), VERSION_2, "0001"); @@ -1733,10 +1740,10 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid() { // CDS -> {B, C} version 3 ImmutableMap resourcesV3 = ImmutableMap.of( "B", Any.pack(mf.buildEdsCluster("B", "B.3", "round_robin", null, null, false, null, - "envoy.transport_sockets.tls", null + "envoy.transport_sockets.tls", null, null )), "C", Any.pack(mf.buildEdsCluster("C", "C.3", "round_robin", null, null, false, null, - "envoy.transport_sockets.tls", null + "envoy.transport_sockets.tls", null, null ))); call.sendResponse(CDS, resourcesV3.values().asList(), VERSION_3, "0002"); // {A} -> does not exit @@ -1774,13 +1781,13 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti // CDS -> {A, B, C}, version 1 ImmutableMap resourcesV1 = ImmutableMap.of( "A", Any.pack(mf.buildEdsCluster("A", "A.1", "round_robin", null, null, false, null, - "envoy.transport_sockets.tls", null + "envoy.transport_sockets.tls", null, null )), "B", Any.pack(mf.buildEdsCluster("B", "B.1", "round_robin", null, null, false, null, - "envoy.transport_sockets.tls", null + "envoy.transport_sockets.tls", null, null )), "C", Any.pack(mf.buildEdsCluster("C", "C.1", "round_robin", null, null, false, null, - "envoy.transport_sockets.tls", null + "envoy.transport_sockets.tls", null, null ))); call.sendResponse(CDS, resourcesV1.values().asList(), VERSION_1, "0000"); // {A, B, C} -> ACK, version 1 @@ -1806,7 +1813,7 @@ public void cdsResponseErrorHandling_subscribedResourceInvalid_withEdsSubscripti // Failed to parse endpoint B ImmutableMap resourcesV2 = ImmutableMap.of( "A", Any.pack(mf.buildEdsCluster("A", "A.2", "round_robin", null, null, false, null, - "envoy.transport_sockets.tls", null + "envoy.transport_sockets.tls", null, null )), "B", Any.pack(mf.buildClusterInvalid("B"))); call.sendResponse(CDS, resourcesV2.values().asList(), VERSION_2, "0001"); @@ -1876,7 +1883,7 @@ public void cdsResourceFound_leastRequestLbPolicy() { Message leastRequestConfig = mf.buildLeastRequestLbConfig(3); Any clusterRingHash = Any.pack( mf.buildEdsCluster(CDS_RESOURCE, null, "least_request_experimental", null, - leastRequestConfig, false, null, "envoy.transport_sockets.tls", null + leastRequestConfig, false, null, "envoy.transport_sockets.tls", null, null )); call.sendResponse(ResourceType.CDS, clusterRingHash, VERSION_1, "0000"); @@ -1907,7 +1914,7 @@ public void cdsResourceFound_ringHashLbPolicy() { Message ringHashConfig = mf.buildRingHashLbConfig("xx_hash", 10L, 100L); Any clusterRingHash = Any.pack( mf.buildEdsCluster(CDS_RESOURCE, null, "ring_hash_experimental", ringHashConfig, null, - false, null, "envoy.transport_sockets.tls", null + false, null, "envoy.transport_sockets.tls", null, null )); call.sendResponse(ResourceType.CDS, clusterRingHash, VERSION_1, "0000"); @@ -1962,7 +1969,7 @@ public void cdsResponseWithCircuitBreakers() { DiscoveryRpcCall call = startResourceWatcher(CDS, CDS_RESOURCE, cdsResourceWatcher); Any clusterCircuitBreakers = Any.pack( mf.buildEdsCluster(CDS_RESOURCE, null, "round_robin", null, null, false, null, - "envoy.transport_sockets.tls", mf.buildCircuitBreakers(50, 200))); + "envoy.transport_sockets.tls", mf.buildCircuitBreakers(50, 200), null)); call.sendResponse(CDS, clusterCircuitBreakers, VERSION_1, "0000"); // Client sent an ACK CDS request. @@ -1999,13 +2006,13 @@ public void cdsResponseWithUpstreamTlsContext() { Any.pack(mf.buildEdsCluster(CDS_RESOURCE, "eds-cluster-foo.googleapis.com", "round_robin", null, null, true, mf.buildUpstreamTlsContext("cert-instance-name", "cert1"), - "envoy.transport_sockets.tls", null)); + "envoy.transport_sockets.tls", null, null)); List clusters = ImmutableList.of( Any.pack(mf.buildLogicalDnsCluster("cluster-bar.googleapis.com", "dns-service-bar.googleapis.com", 443, "round_robin", null, null,false, null, null)), clusterEds, Any.pack(mf.buildEdsCluster("cluster-baz.googleapis.com", null, "round_robin", null, null, - false, null, "envoy.transport_sockets.tls", null))); + false, null, "envoy.transport_sockets.tls", null, null))); call.sendResponse(CDS, clusters, VERSION_1, "0000"); // Client sent an ACK CDS request. @@ -2035,13 +2042,13 @@ public void cdsResponseWithNewUpstreamTlsContext() { Any.pack(mf.buildEdsCluster(CDS_RESOURCE, "eds-cluster-foo.googleapis.com", "round_robin", null, null,true, mf.buildNewUpstreamTlsContext("cert-instance-name", "cert1"), - "envoy.transport_sockets.tls", null)); + "envoy.transport_sockets.tls", null, null)); List clusters = ImmutableList.of( Any.pack(mf.buildLogicalDnsCluster("cluster-bar.googleapis.com", "dns-service-bar.googleapis.com", 443, "round_robin", null, null, false, null, null)), clusterEds, Any.pack(mf.buildEdsCluster("cluster-baz.googleapis.com", null, "round_robin", null, null, - false, null, "envoy.transport_sockets.tls", null))); + false, null, "envoy.transport_sockets.tls", null, null))); call.sendResponse(CDS, clusters, VERSION_1, "0000"); // Client sent an ACK CDS request. @@ -2069,7 +2076,7 @@ public void cdsResponseErrorHandling_badUpstreamTlsContext() { List clusters = ImmutableList.of(Any .pack(mf.buildEdsCluster(CDS_RESOURCE, "eds-cluster-foo.googleapis.com", "round_robin", null, null, true, - mf.buildUpstreamTlsContext(null, null), "envoy.transport_sockets.tls", null))); + mf.buildUpstreamTlsContext(null, null), "envoy.transport_sockets.tls", null, null))); call.sendResponse(CDS, clusters, VERSION_1, "0000"); // The response NACKed with errors indicating indices of the failed resources. @@ -2082,6 +2089,224 @@ public void cdsResponseErrorHandling_badUpstreamTlsContext() { verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); } + /** + * CDS response containing OutlierDetection for a cluster. + */ + @Test + @SuppressWarnings("deprecation") + public void cdsResponseWithOutlierDetection() { + Assume.assumeTrue(useProtocolV3()); + ClientXdsClient.enableOutlierDetection = true; + + DiscoveryRpcCall call = startResourceWatcher(CDS, CDS_RESOURCE, cdsResourceWatcher); + + OutlierDetection outlierDetectionXds = OutlierDetection.newBuilder() + .setInterval(Durations.fromNanos(100)) + .setBaseEjectionTime(Durations.fromNanos(100)) + .setMaxEjectionTime(Durations.fromNanos(100)) + .setMaxEjectionPercent(UInt32Value.of(100)) + .setSuccessRateStdevFactor(UInt32Value.of(100)) + .setEnforcingSuccessRate(UInt32Value.of(100)) + .setSuccessRateMinimumHosts(UInt32Value.of(100)) + .setSuccessRateRequestVolume(UInt32Value.of(100)) + .setFailurePercentageThreshold(UInt32Value.of(100)) + .setEnforcingFailurePercentage(UInt32Value.of(100)) + .setFailurePercentageMinimumHosts(UInt32Value.of(100)) + .setFailurePercentageRequestVolume(UInt32Value.of(100)).build(); + + // Management server sends back CDS response with UpstreamTlsContext. + Any clusterEds = + Any.pack(mf.buildEdsCluster(CDS_RESOURCE, "eds-cluster-foo.googleapis.com", "round_robin", + null, null, true, + mf.buildUpstreamTlsContext("cert-instance-name", "cert1"), + "envoy.transport_sockets.tls", null, outlierDetectionXds)); + List clusters = ImmutableList.of( + Any.pack(mf.buildLogicalDnsCluster("cluster-bar.googleapis.com", + "dns-service-bar.googleapis.com", 443, "round_robin", null, null,false, null, null)), + clusterEds, + Any.pack(mf.buildEdsCluster("cluster-baz.googleapis.com", null, "round_robin", null, null, + false, null, "envoy.transport_sockets.tls", null, outlierDetectionXds))); + call.sendResponse(CDS, clusters, VERSION_1, "0000"); + + // Client sent an ACK CDS request. + call.verifyRequest(CDS, CDS_RESOURCE, VERSION_1, "0000", NODE); + verify(cdsResourceWatcher, times(1)).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + + // The outlier detection config in CdsUpdate should match what we get from xDS. + EnvoyServerProtoData.OutlierDetection outlierDetection = cdsUpdate.outlierDetection(); + assertThat(outlierDetection).isNotNull(); + assertThat(outlierDetection.intervalNanos()).isEqualTo(100); + assertThat(outlierDetection.baseEjectionTimeNanos()).isEqualTo(100); + assertThat(outlierDetection.maxEjectionTimeNanos()).isEqualTo(100); + assertThat(outlierDetection.maxEjectionPercent()).isEqualTo(100); + + SuccessRateEjection successRateEjection = outlierDetection.successRateEjection(); + assertThat(successRateEjection).isNotNull(); + assertThat(successRateEjection.stdevFactor()).isEqualTo(100); + assertThat(successRateEjection.enforcementPercentage()).isEqualTo(100); + assertThat(successRateEjection.minimumHosts()).isEqualTo(100); + assertThat(successRateEjection.requestVolume()).isEqualTo(100); + + FailurePercentageEjection failurePercentageEjection + = outlierDetection.failurePercentageEjection(); + assertThat(failurePercentageEjection).isNotNull(); + assertThat(failurePercentageEjection.threshold()).isEqualTo(100); + assertThat(failurePercentageEjection.enforcementPercentage()).isEqualTo(100); + assertThat(failurePercentageEjection.minimumHosts()).isEqualTo(100); + assertThat(failurePercentageEjection.requestVolume()).isEqualTo(100); + + verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusterEds, VERSION_1, TIME_INCREMENT); + verifySubscribedResourcesMetadataSizes(0, 1, 0, 0); + } + + /** + * CDS response containing OutlierDetection for a cluster, but support has not been enabled. + */ + @Test + @SuppressWarnings("deprecation") + public void cdsResponseWithOutlierDetection_supportDisabled() { + Assume.assumeTrue(useProtocolV3()); + ClientXdsClient.enableOutlierDetection = false; + + DiscoveryRpcCall call = startResourceWatcher(CDS, CDS_RESOURCE, cdsResourceWatcher); + + OutlierDetection outlierDetectionXds = OutlierDetection.newBuilder() + .setInterval(Durations.fromNanos(100)).build(); + + // Management server sends back CDS response with UpstreamTlsContext. + Any clusterEds = + Any.pack(mf.buildEdsCluster(CDS_RESOURCE, "eds-cluster-foo.googleapis.com", "round_robin", + null, null, true, + mf.buildUpstreamTlsContext("cert-instance-name", "cert1"), + "envoy.transport_sockets.tls", null, outlierDetectionXds)); + List clusters = ImmutableList.of( + Any.pack(mf.buildLogicalDnsCluster("cluster-bar.googleapis.com", + "dns-service-bar.googleapis.com", 443, "round_robin", null, null,false, null, null)), + clusterEds, + Any.pack(mf.buildEdsCluster("cluster-baz.googleapis.com", null, "round_robin", null, null, + false, null, "envoy.transport_sockets.tls", null, outlierDetectionXds))); + call.sendResponse(CDS, clusters, VERSION_1, "0000"); + + // Client sent an ACK CDS request. + call.verifyRequest(CDS, CDS_RESOURCE, VERSION_1, "0000", NODE); + verify(cdsResourceWatcher, times(1)).onChanged(cdsUpdateCaptor.capture()); + CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); + + assertThat(cdsUpdate.outlierDetection()).isNull(); + + verifyResourceMetadataAcked(CDS, CDS_RESOURCE, clusterEds, VERSION_1, TIME_INCREMENT); + verifySubscribedResourcesMetadataSizes(0, 1, 0, 0); + } + + /** + * CDS response containing OutlierDetection for a cluster. + */ + @Test + @SuppressWarnings("deprecation") + public void cdsResponseWithInvalidOutlierDetectionNacks() { + Assume.assumeTrue(useProtocolV3()); + ClientXdsClient.enableOutlierDetection = true; + + DiscoveryRpcCall call = startResourceWatcher(CDS, CDS_RESOURCE, cdsResourceWatcher); + + OutlierDetection outlierDetectionXds = OutlierDetection.newBuilder() + .setMaxEjectionPercent(UInt32Value.of(101)).build(); + + // Management server sends back CDS response with UpstreamTlsContext. + Any clusterEds = + Any.pack(mf.buildEdsCluster(CDS_RESOURCE, "eds-cluster-foo.googleapis.com", "round_robin", + null, null, true, + mf.buildUpstreamTlsContext("cert-instance-name", "cert1"), + "envoy.transport_sockets.tls", null, outlierDetectionXds)); + List clusters = ImmutableList.of( + Any.pack(mf.buildLogicalDnsCluster("cluster-bar.googleapis.com", + "dns-service-bar.googleapis.com", 443, "round_robin", null, null,false, null, null)), + clusterEds, + Any.pack(mf.buildEdsCluster("cluster-baz.googleapis.com", null, "round_robin", null, null, + false, null, "envoy.transport_sockets.tls", null, outlierDetectionXds))); + call.sendResponse(CDS, clusters, VERSION_1, "0000"); + + String errorMsg = "CDS response Cluster 'cluster.googleapis.com' validation error: " + + "Cluster cluster.googleapis.com: malformed outlier_detection: " + + "io.grpc.xds.ClientXdsClient$ResourceInvalidException: outlier_detection " + + "max_ejection_percent is > 100"; + call.verifyRequestNack(CDS, CDS_RESOURCE, "", "0000", NODE, ImmutableList.of(errorMsg)); + verify(cdsResourceWatcher).onError(errorCaptor.capture()); + verifyStatusWithNodeId(errorCaptor.getValue(), Code.UNAVAILABLE, errorMsg); + } + + @Test(expected = ResourceInvalidException.class) + public void validateOutlierDetection_invalidInterval() throws ResourceInvalidException { + ClientXdsClient.validateOutlierDetection( + OutlierDetection.newBuilder().setInterval(Duration.newBuilder().setSeconds(Long.MAX_VALUE)) + .build()); + } + + @Test(expected = ResourceInvalidException.class) + public void validateOutlierDetection_negativeInterval() throws ResourceInvalidException { + ClientXdsClient.validateOutlierDetection( + OutlierDetection.newBuilder().setInterval(Duration.newBuilder().setSeconds(-1)) + .build()); + } + + @Test(expected = ResourceInvalidException.class) + public void validateOutlierDetection_invalidBaseEjectionTime() throws ResourceInvalidException { + ClientXdsClient.validateOutlierDetection( + OutlierDetection.newBuilder() + .setBaseEjectionTime(Duration.newBuilder().setSeconds(Long.MAX_VALUE)) + .build()); + } + + @Test(expected = ResourceInvalidException.class) + public void validateOutlierDetection_negativeBaseEjectionTime() throws ResourceInvalidException { + ClientXdsClient.validateOutlierDetection( + OutlierDetection.newBuilder().setBaseEjectionTime(Duration.newBuilder().setSeconds(-1)) + .build()); + } + + @Test(expected = ResourceInvalidException.class) + public void validateOutlierDetection_invalidMaxEjectionTime() throws ResourceInvalidException { + ClientXdsClient.validateOutlierDetection( + OutlierDetection.newBuilder() + .setMaxEjectionTime(Duration.newBuilder().setSeconds(Long.MAX_VALUE)) + .build()); + } + + @Test(expected = ResourceInvalidException.class) + public void validateOutlierDetection_negativeMaxEjectionTime() throws ResourceInvalidException { + ClientXdsClient.validateOutlierDetection( + OutlierDetection.newBuilder().setMaxEjectionTime(Duration.newBuilder().setSeconds(-1)) + .build()); + } + + @Test(expected = ResourceInvalidException.class) + public void validateOutlierDetection_maxEjectionPercentTooHigh() throws ResourceInvalidException { + ClientXdsClient.validateOutlierDetection( + OutlierDetection.newBuilder().setMaxEjectionPercent(UInt32Value.of(101)).build()); + } + + @Test(expected = ResourceInvalidException.class) + public void validateOutlierDetection_enforcingSuccessRateTooHigh() + throws ResourceInvalidException { + ClientXdsClient.validateOutlierDetection( + OutlierDetection.newBuilder().setEnforcingSuccessRate(UInt32Value.of(101)).build()); + } + + @Test(expected = ResourceInvalidException.class) + public void validateOutlierDetection_failurePercentageThresholdTooHigh() + throws ResourceInvalidException { + ClientXdsClient.validateOutlierDetection( + OutlierDetection.newBuilder().setFailurePercentageThreshold(UInt32Value.of(101)).build()); + } + + @Test(expected = ResourceInvalidException.class) + public void validateOutlierDetection_enforcingFailurePercentageTooHigh() + throws ResourceInvalidException { + ClientXdsClient.validateOutlierDetection( + OutlierDetection.newBuilder().setEnforcingFailurePercentage(UInt32Value.of(101)).build()); + } + /** * CDS response containing UpstreamTlsContext with bad transportSocketName for a cluster. */ @@ -2094,7 +2319,8 @@ public void cdsResponseErrorHandling_badTransportSocketName() { List clusters = ImmutableList.of(Any .pack(mf.buildEdsCluster(CDS_RESOURCE, "eds-cluster-foo.googleapis.com", "round_robin", null, null, true, - mf.buildUpstreamTlsContext("secret1", "cert1"), "envoy.transport_sockets.bad", null))); + mf.buildUpstreamTlsContext("secret1", "cert1"), "envoy.transport_sockets.bad", null, + null))); call.sendResponse(CDS, clusters, VERSION_1, "0000"); // The response NACKed with errors indicating indices of the failed resources. @@ -2169,7 +2395,7 @@ public void cdsResourceUpdated() { String edsService = "eds-service-bar.googleapis.com"; Any clusterEds = Any.pack( mf.buildEdsCluster(CDS_RESOURCE, edsService, "round_robin", null, null, true, null, - "envoy.transport_sockets.tls", null + "envoy.transport_sockets.tls", null, null )); call.sendResponse(CDS, clusterEds, VERSION_2, "0001"); call.verifyRequest(CDS, CDS_RESOURCE, VERSION_2, "0001", NODE); @@ -2199,17 +2425,17 @@ public void cdsResourceUpdatedWithDuplicate() { String transportSocketName = "envoy.transport_sockets.tls"; Any roundRobinConfig = Any.pack( mf.buildEdsCluster(CDS_RESOURCE, edsService, "round_robin", null, null, true, null, - transportSocketName, null + transportSocketName, null, null )); Any ringHashConfig = Any.pack( mf.buildEdsCluster(CDS_RESOURCE, edsService, "ring_hash_experimental", mf.buildRingHashLbConfig("xx_hash", 1, 2), null, true, null, - transportSocketName, null + transportSocketName, null, null )); Any leastRequestConfig = Any.pack( mf.buildEdsCluster(CDS_RESOURCE, edsService, "least_request_experimental", null, mf.buildLeastRequestLbConfig(2), true, null, - transportSocketName, null + transportSocketName, null, null )); // Configure with round robin, the update should be sent to the watcher. @@ -2332,7 +2558,7 @@ public void multipleCdsWatchers() { Any.pack(mf.buildLogicalDnsCluster(CDS_RESOURCE, dnsHostAddr, dnsHostPort, "round_robin", null, null, false, null, null)), Any.pack(mf.buildEdsCluster(cdsResourceTwo, edsService, "round_robin", null, null, true, - null, "envoy.transport_sockets.tls", null))); + null, "envoy.transport_sockets.tls", null, null))); call.sendResponse(CDS, clusters, VERSION_1, "0000"); verify(cdsResourceWatcher).onChanged(cdsUpdateCaptor.capture()); CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); @@ -2643,10 +2869,10 @@ public void edsResourceDeletedByCds() { DiscoveryRpcCall call = resourceDiscoveryCalls.poll(); List clusters = ImmutableList.of( Any.pack(mf.buildEdsCluster(resource, null, "round_robin", null, null, true, null, - "envoy.transport_sockets.tls", null + "envoy.transport_sockets.tls", null, null )), Any.pack(mf.buildEdsCluster(CDS_RESOURCE, EDS_RESOURCE, "round_robin", null, null, false, - null, "envoy.transport_sockets.tls", null))); + null, "envoy.transport_sockets.tls", null, null))); call.sendResponse(CDS, clusters, VERSION_1, "0000"); verify(cdsWatcher).onChanged(cdsUpdateCaptor.capture()); CdsUpdate cdsUpdate = cdsUpdateCaptor.getValue(); @@ -2692,9 +2918,9 @@ public void edsResourceDeletedByCds() { clusters = ImmutableList.of( Any.pack(mf.buildEdsCluster(resource, null, "round_robin", null, null, true, null, - "envoy.transport_sockets.tls", null)), // no change + "envoy.transport_sockets.tls", null, null)), // no change Any.pack(mf.buildEdsCluster(CDS_RESOURCE, null, "round_robin", null, null, false, null, - "envoy.transport_sockets.tls", null + "envoy.transport_sockets.tls", null, null ))); call.sendResponse(CDS, clusters, VERSION_2, "0001"); verify(cdsResourceWatcher, times(2)).onChanged(cdsUpdateCaptor.capture()); @@ -3256,7 +3482,7 @@ protected abstract Message buildVirtualHost( protected abstract Message buildEdsCluster(String clusterName, @Nullable String edsServiceName, String lbPolicy, @Nullable Message ringHashLbConfig, @Nullable Message leastRequestLbConfig, boolean enableLrs, @Nullable Message upstreamTlsContext, String transportSocketName, - @Nullable Message circuitBreakers); + @Nullable Message circuitBreakers, @Nullable Message outlierDetection); protected abstract Message buildLogicalDnsCluster(String clusterName, String dnsHostAddr, int dnsHostPort, String lbPolicy, @Nullable Message ringHashLbConfig, diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientV2Test.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientV2Test.java index b302b56dbb8..3e2cafd58fd 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientV2Test.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientV2Test.java @@ -52,6 +52,7 @@ import io.envoyproxy.envoy.api.v2.auth.UpstreamTlsContext; import io.envoyproxy.envoy.api.v2.cluster.CircuitBreakers; import io.envoyproxy.envoy.api.v2.cluster.CircuitBreakers.Thresholds; +import io.envoyproxy.envoy.api.v2.cluster.OutlierDetection; import io.envoyproxy.envoy.api.v2.core.Address; import io.envoyproxy.envoy.api.v2.core.AggregatedConfigSource; import io.envoyproxy.envoy.api.v2.core.ApiConfigSource; @@ -420,10 +421,10 @@ protected Message buildEdsCluster(String clusterName, @Nullable String edsServic String lbPolicy, @Nullable Message ringHashLbConfig, @Nullable Message leastRequestLbConfig, boolean enableLrs, @Nullable Message upstreamTlsContext, String transportSocketName, - @Nullable Message circuitBreakers) { + @Nullable Message circuitBreakers, @Nullable Message outlierDetection) { Cluster.Builder builder = initClusterBuilder( clusterName, lbPolicy, ringHashLbConfig, leastRequestLbConfig, - enableLrs, upstreamTlsContext, circuitBreakers); + enableLrs, upstreamTlsContext, circuitBreakers, outlierDetection); builder.setType(DiscoveryType.EDS); EdsClusterConfig.Builder edsClusterConfigBuilder = EdsClusterConfig.newBuilder(); edsClusterConfigBuilder.setEdsConfig( @@ -442,7 +443,7 @@ protected Message buildLogicalDnsCluster(String clusterName, String dnsHostAddr, @Nullable Message upstreamTlsContext, @Nullable Message circuitBreakers) { Cluster.Builder builder = initClusterBuilder( clusterName, lbPolicy, ringHashLbConfig, leastRequestLbConfig, - enableLrs, upstreamTlsContext, circuitBreakers); + enableLrs, upstreamTlsContext, circuitBreakers, null); builder.setType(DiscoveryType.LOGICAL_DNS); builder.setLoadAssignment( ClusterLoadAssignment.newBuilder().addEndpoints( @@ -483,7 +484,7 @@ protected Message buildAggregateCluster(String clusterName, String lbPolicy, private Cluster.Builder initClusterBuilder(String clusterName, String lbPolicy, @Nullable Message ringHashLbConfig, @Nullable Message leastRequestLbConfig, boolean enableLrs, @Nullable Message upstreamTlsContext, - @Nullable Message circuitBreakers) { + @Nullable Message circuitBreakers, @Nullable Message outlierDetection) { Cluster.Builder builder = Cluster.newBuilder(); builder.setName(clusterName); if (lbPolicy.equals("round_robin")) { @@ -511,6 +512,9 @@ private Cluster.Builder initClusterBuilder(String clusterName, String lbPolicy, if (circuitBreakers != null) { builder.setCircuitBreakers((CircuitBreakers) circuitBreakers); } + if (outlierDetection != null) { + builder.setOutlierDetection((OutlierDetection) outlierDetection); + } return builder; } diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientV3Test.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientV3Test.java index 4f86b178d2d..b051643c0bf 100644 --- a/xds/src/test/java/io/grpc/xds/ClientXdsClientV3Test.java +++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientV3Test.java @@ -41,6 +41,7 @@ import io.envoyproxy.envoy.config.cluster.v3.Cluster.LeastRequestLbConfig; import io.envoyproxy.envoy.config.cluster.v3.Cluster.RingHashLbConfig; import io.envoyproxy.envoy.config.cluster.v3.Cluster.RingHashLbConfig.HashFunction; +import io.envoyproxy.envoy.config.cluster.v3.OutlierDetection; import io.envoyproxy.envoy.config.core.v3.Address; import io.envoyproxy.envoy.config.core.v3.AggregatedConfigSource; import io.envoyproxy.envoy.config.core.v3.ConfigSource; @@ -476,10 +477,10 @@ protected Message buildEdsCluster(String clusterName, @Nullable String edsServic String lbPolicy, @Nullable Message ringHashLbConfig, @Nullable Message leastRequestLbConfig, boolean enableLrs, @Nullable Message upstreamTlsContext, String transportSocketName, - @Nullable Message circuitBreakers) { + @Nullable Message circuitBreakers, @Nullable Message outlierDetection) { Cluster.Builder builder = initClusterBuilder( clusterName, lbPolicy, ringHashLbConfig, leastRequestLbConfig, - enableLrs, upstreamTlsContext, transportSocketName, circuitBreakers); + enableLrs, upstreamTlsContext, transportSocketName, circuitBreakers, outlierDetection); builder.setType(DiscoveryType.EDS); EdsClusterConfig.Builder edsClusterConfigBuilder = EdsClusterConfig.newBuilder(); edsClusterConfigBuilder.setEdsConfig( @@ -498,7 +499,7 @@ protected Message buildLogicalDnsCluster(String clusterName, String dnsHostAddr, @Nullable Message upstreamTlsContext, @Nullable Message circuitBreakers) { Cluster.Builder builder = initClusterBuilder( clusterName, lbPolicy, ringHashLbConfig, leastRequestLbConfig, - enableLrs, upstreamTlsContext, "envoy.transport_sockets.tls", circuitBreakers); + enableLrs, upstreamTlsContext, "envoy.transport_sockets.tls", circuitBreakers, null); builder.setType(DiscoveryType.LOGICAL_DNS); builder.setLoadAssignment( ClusterLoadAssignment.newBuilder().addEndpoints( @@ -539,7 +540,7 @@ protected Message buildAggregateCluster(String clusterName, String lbPolicy, private Cluster.Builder initClusterBuilder(String clusterName, String lbPolicy, @Nullable Message ringHashLbConfig, @Nullable Message leastRequestLbConfig, boolean enableLrs, @Nullable Message upstreamTlsContext, String transportSocketName, - @Nullable Message circuitBreakers) { + @Nullable Message circuitBreakers, @Nullable Message outlierDetection) { Cluster.Builder builder = Cluster.newBuilder(); builder.setName(clusterName); if (lbPolicy.equals("round_robin")) { @@ -567,6 +568,9 @@ private Cluster.Builder initClusterBuilder(String clusterName, String lbPolicy, if (circuitBreakers != null) { builder.setCircuitBreakers((CircuitBreakers) circuitBreakers); } + if (outlierDetection != null) { + builder.setOutlierDetection((OutlierDetection) outlierDetection); + } return builder; } diff --git a/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java index 22c54bbb94a..f18643260e2 100644 --- a/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java @@ -58,6 +58,8 @@ import io.grpc.internal.GrpcUtil; import io.grpc.internal.ObjectPool; import io.grpc.internal.ServiceConfigUtil.PolicySelection; +import io.grpc.util.OutlierDetectionLoadBalancer.OutlierDetectionLoadBalancerConfig; +import io.grpc.util.OutlierDetectionLoadBalancerProvider; import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.ClusterImplLoadBalancerProvider.ClusterImplConfig; import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig; @@ -65,6 +67,9 @@ import io.grpc.xds.Endpoints.DropOverload; import io.grpc.xds.Endpoints.LbEndpoint; import io.grpc.xds.Endpoints.LocalityLbEndpoints; +import io.grpc.xds.EnvoyServerProtoData.FailurePercentageEjection; +import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; +import io.grpc.xds.EnvoyServerProtoData.SuccessRateEjection; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.LeastRequestLoadBalancer.LeastRequestConfig; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig; @@ -116,10 +121,18 @@ public class ClusterResolverLoadBalancerTest { Locality.create("test-region-3", "test-zone-3", "test-subzone-3"); private final UpstreamTlsContext tlsContext = CommonTlsContextTestsUtil.buildUpstreamTlsContext("google_cloud_private_spiffe", true); + private final OutlierDetection outlierDetection = OutlierDetection.create( + 100L, 100L, 100L, 100, SuccessRateEjection.create(100, 100, 100, 100), + FailurePercentageEjection.create(100, 100, 100, 100)); private final DiscoveryMechanism edsDiscoveryMechanism1 = - DiscoveryMechanism.forEds(CLUSTER1, EDS_SERVICE_NAME1, LRS_SERVER_INFO, 100L, tlsContext); + DiscoveryMechanism.forEds(CLUSTER1, EDS_SERVICE_NAME1, LRS_SERVER_INFO, 100L, tlsContext, + null); private final DiscoveryMechanism edsDiscoveryMechanism2 = - DiscoveryMechanism.forEds(CLUSTER2, EDS_SERVICE_NAME2, LRS_SERVER_INFO, 200L, tlsContext); + DiscoveryMechanism.forEds(CLUSTER2, EDS_SERVICE_NAME2, LRS_SERVER_INFO, 200L, tlsContext, + null); + private final DiscoveryMechanism edsDiscoveryMechanismWithOutlierDetection = + DiscoveryMechanism.forEds(CLUSTER1, EDS_SERVICE_NAME1, LRS_SERVER_INFO, 100L, tlsContext, + outlierDetection); private final DiscoveryMechanism logicalDnsDiscoveryMechanism = DiscoveryMechanism.forLogicalDns(CLUSTER_DNS, DNS_HOST_NAME, LRS_SERVER_INFO, 300L, null); @@ -182,6 +195,7 @@ public void setUp() throws URISyntaxException { lbRegistry.register(new FakeLoadBalancerProvider(WEIGHTED_TARGET_POLICY_NAME)); lbRegistry.register( new FakeLoadBalancerProvider("pick_first")); // needed by logical_dns + lbRegistry.register(new OutlierDetectionLoadBalancerProvider()); NameResolver.Args args = NameResolver.Args.newBuilder() .setDefaultPort(8080) .setProxyDetector(GrpcUtil.NOOP_PROXY_DETECTOR) @@ -317,6 +331,90 @@ public void edsClustersWithLeastRequestEndpointLbPolicy() { locality1, 100); } + @Test + public void edsClustersWithOutlierDetection() { + ClusterResolverConfig config = new ClusterResolverConfig( + Collections.singletonList(edsDiscoveryMechanismWithOutlierDetection), leastRequest); + deliverLbConfig(config); + assertThat(xdsClient.watchers.keySet()).containsExactly(EDS_SERVICE_NAME1); + assertThat(childBalancers).isEmpty(); + + // Simple case with one priority and one locality + EquivalentAddressGroup endpoint = makeAddress("endpoint-addr-1"); + LocalityLbEndpoints localityLbEndpoints = + LocalityLbEndpoints.create( + Arrays.asList( + LbEndpoint.create(endpoint, 0 /* loadBalancingWeight */, true)), + 100 /* localityWeight */, 1 /* priority */); + xdsClient.deliverClusterLoadAssignment( + EDS_SERVICE_NAME1, + ImmutableMap.of(locality1, localityLbEndpoints)); + assertThat(childBalancers).hasSize(1); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + assertThat(childBalancer.addresses).hasSize(1); + EquivalentAddressGroup addr = childBalancer.addresses.get(0); + assertThat(addr.getAddresses()).isEqualTo(endpoint.getAddresses()); + assertThat(childBalancer.name).isEqualTo(PRIORITY_POLICY_NAME); + PriorityLbConfig priorityLbConfig = (PriorityLbConfig) childBalancer.config; + assertThat(priorityLbConfig.priorities).containsExactly(CLUSTER1 + "[child1]"); + PriorityChildConfig priorityChildConfig = + Iterables.getOnlyElement(priorityLbConfig.childConfigs.values()); + + // The child config for priority should be outlier detection. + assertThat(priorityChildConfig.policySelection.getProvider().getPolicyName()) + .isEqualTo("outlier_detection_experimental"); + OutlierDetectionLoadBalancerConfig outlierDetectionConfig = + (OutlierDetectionLoadBalancerConfig) priorityChildConfig.policySelection.getConfig(); + + // The outlier detection config should faithfully represent what came down from xDS. + assertThat(outlierDetectionConfig.intervalNanos).isEqualTo(outlierDetection.intervalNanos()); + assertThat(outlierDetectionConfig.baseEjectionTimeNanos).isEqualTo( + outlierDetection.baseEjectionTimeNanos()); + assertThat(outlierDetectionConfig.baseEjectionTimeNanos).isEqualTo( + outlierDetection.baseEjectionTimeNanos()); + assertThat(outlierDetectionConfig.maxEjectionTimeNanos).isEqualTo( + outlierDetection.maxEjectionTimeNanos()); + assertThat(outlierDetectionConfig.maxEjectionPercent).isEqualTo( + outlierDetection.maxEjectionPercent()); + + OutlierDetectionLoadBalancerConfig.SuccessRateEjection successRateEjection + = outlierDetectionConfig.successRateEjection; + assertThat(successRateEjection.stdevFactor).isEqualTo( + outlierDetectionConfig.successRateEjection.stdevFactor); + assertThat(successRateEjection.enforcementPercentage).isEqualTo( + outlierDetectionConfig.successRateEjection.enforcementPercentage); + assertThat(successRateEjection.minimumHosts).isEqualTo( + outlierDetectionConfig.successRateEjection.minimumHosts); + assertThat(successRateEjection.requestVolume).isEqualTo( + outlierDetectionConfig.successRateEjection.requestVolume); + + OutlierDetectionLoadBalancerConfig.FailurePercentageEjection failurePercentageEjection + = outlierDetectionConfig.failurePercentageEjection; + assertThat(failurePercentageEjection.threshold).isEqualTo( + outlierDetection.failurePercentageEjection().threshold()); + assertThat(failurePercentageEjection.enforcementPercentage).isEqualTo( + outlierDetection.failurePercentageEjection().enforcementPercentage()); + assertThat(failurePercentageEjection.minimumHosts).isEqualTo( + outlierDetection.failurePercentageEjection().minimumHosts()); + assertThat(failurePercentageEjection.requestVolume).isEqualTo( + outlierDetection.failurePercentageEjection().requestVolume()); + + // The wrapped configuration should not have been tampered with. + ClusterImplConfig clusterImplConfig = + (ClusterImplConfig) outlierDetectionConfig.childPolicy.getConfig(); + assertClusterImplConfig(clusterImplConfig, CLUSTER1, EDS_SERVICE_NAME1, LRS_SERVER_INFO, 100L, + tlsContext, Collections.emptyList(), WRR_LOCALITY_POLICY_NAME); + WrrLocalityConfig wrrLocalityConfig = + (WrrLocalityConfig) clusterImplConfig.childPolicy.getConfig(); + assertThat(wrrLocalityConfig.childPolicy.getProvider().getPolicyName()).isEqualTo( + "least_request_experimental"); + + assertThat( + childBalancer.attributes.get(InternalXdsAttributes.ATTR_LOCALITY_WEIGHTS)).containsEntry( + locality1, 100); + } + + @Test public void onlyEdsClusters_receivedEndpoints() { ClusterResolverConfig config = new ClusterResolverConfig( From e3551c830cff2b3a89512b7eb9dbc9b1fca09b74 Mon Sep 17 00:00:00 2001 From: Terry Wilson Date: Thu, 18 Aug 2022 15:34:41 -0700 Subject: [PATCH 2/2] Fix test assertion. --- .../java/io/grpc/xds/ClusterResolverLoadBalancerTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java index f18643260e2..2a8b95260fc 100644 --- a/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java @@ -380,13 +380,13 @@ public void edsClustersWithOutlierDetection() { OutlierDetectionLoadBalancerConfig.SuccessRateEjection successRateEjection = outlierDetectionConfig.successRateEjection; assertThat(successRateEjection.stdevFactor).isEqualTo( - outlierDetectionConfig.successRateEjection.stdevFactor); + outlierDetection.successRateEjection().stdevFactor()); assertThat(successRateEjection.enforcementPercentage).isEqualTo( - outlierDetectionConfig.successRateEjection.enforcementPercentage); + outlierDetection.successRateEjection().enforcementPercentage()); assertThat(successRateEjection.minimumHosts).isEqualTo( - outlierDetectionConfig.successRateEjection.minimumHosts); + outlierDetection.successRateEjection().minimumHosts()); assertThat(successRateEjection.requestVolume).isEqualTo( - outlierDetectionConfig.successRateEjection.requestVolume); + outlierDetection.successRateEjection().requestVolume()); OutlierDetectionLoadBalancerConfig.FailurePercentageEjection failurePercentageEjection = outlierDetectionConfig.failurePercentageEjection;