diff --git a/core/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java b/core/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java index 3fb32fdb0782..dea997c7494c 100644 --- a/core/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java +++ b/core/src/main/java/io/grpc/util/OutlierDetectionLoadBalancer.java @@ -837,13 +837,13 @@ private static boolean hasSingleAddress(List addressGrou */ public static final class OutlierDetectionLoadBalancerConfig { - final Long intervalNanos; - final Long baseEjectionTimeNanos; - final Long maxEjectionTimeNanos; - final Integer maxEjectionPercent; - final SuccessRateEjection successRateEjection; - final FailurePercentageEjection failurePercentageEjection; - final PolicySelection childPolicy; + public final Long intervalNanos; + public final Long baseEjectionTimeNanos; + public final Long maxEjectionTimeNanos; + public final Integer maxEjectionPercent; + public final SuccessRateEjection successRateEjection; + public final FailurePercentageEjection failurePercentageEjection; + public final PolicySelection childPolicy; private OutlierDetectionLoadBalancerConfig(Long intervalNanos, Long baseEjectionTimeNanos, @@ -932,10 +932,10 @@ public OutlierDetectionLoadBalancerConfig build() { /** The configuration for success rate ejection. */ public static class SuccessRateEjection { - final Integer stdevFactor; - final Integer enforcementPercentage; - final Integer minimumHosts; - final Integer requestVolume; + public final Integer stdevFactor; + public final Integer enforcementPercentage; + public final Integer minimumHosts; + public final Integer requestVolume; SuccessRateEjection(Integer stdevFactor, Integer enforcementPercentage, Integer minimumHosts, Integer requestVolume) { @@ -996,10 +996,10 @@ public SuccessRateEjection build() { /** The configuration for failure percentage ejection. */ public static class FailurePercentageEjection { - final Integer threshold; - final Integer enforcementPercentage; - final Integer minimumHosts; - final Integer requestVolume; + public final Integer threshold; + public final Integer enforcementPercentage; + public final Integer minimumHosts; + public final Integer requestVolume; FailurePercentageEjection(Integer threshold, Integer enforcementPercentage, Integer minimumHosts, Integer requestVolume) { diff --git a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java index 4a599ef232c0..4fa701c3c423 100644 --- a/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java +++ b/xds/src/main/java/io/grpc/xds/CdsLoadBalancer2.java @@ -159,7 +159,7 @@ private void handleClusterDiscovered() { instance = DiscoveryMechanism.forEds( clusterState.name, clusterState.result.edsServiceName(), clusterState.result.lrsServerInfo(), clusterState.result.maxConcurrentRequests(), - clusterState.result.upstreamTlsContext()); + clusterState.result.upstreamTlsContext(), clusterState.result.outlierDetection()); } else { // logical DNS instance = DiscoveryMechanism.forLogicalDns( clusterState.name, clusterState.result.dnsHostName(), diff --git a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java index 9a4c366f84f1..a9a5220f994d 100644 --- a/xds/src/main/java/io/grpc/xds/ClientXdsClient.java +++ b/xds/src/main/java/io/grpc/xds/ClientXdsClient.java @@ -90,6 +90,7 @@ import io.grpc.xds.EnvoyServerProtoData.ConnectionSourceType; import io.grpc.xds.EnvoyServerProtoData.FilterChain; import io.grpc.xds.EnvoyServerProtoData.FilterChainMatch; +import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.Filter.ClientInterceptorBuilder; import io.grpc.xds.Filter.FilterConfig; @@ -166,6 +167,10 @@ final class ClientXdsClient extends XdsClient implements XdsResponseHandler, Res static boolean enableCustomLbConfig = Strings.isNullOrEmpty(System.getenv("GRPC_EXPERIMENTAL_XDS_CUSTOM_LB_CONFIG")) || Boolean.parseBoolean(System.getenv("GRPC_EXPERIMENTAL_XDS_CUSTOM_LB_CONFIG")); + @VisibleForTesting + static boolean enableOutlierDetection = + Strings.isNullOrEmpty(System.getenv("GRPC_EXPERIMENTAL_XDS_OUTLIER_DETECTION")) + || Boolean.parseBoolean(System.getenv("GRPC_EXPERIMENTAL_XDS_OUTLIER_DETECTION")); private static final String TYPE_URL_HTTP_CONNECTION_MANAGER_V2 = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2" + ".HttpConnectionManager"; @@ -1704,6 +1709,7 @@ private static StructOrError parseNonAggregateCluster( ServerInfo lrsServerInfo = null; Long maxConcurrentRequests = null; UpstreamTlsContext upstreamTlsContext = null; + OutlierDetection outlierDetection = null; if (cluster.hasLrsServer()) { if (!cluster.getLrsServer().hasSelf()) { return StructOrError.fromError( @@ -1743,6 +1749,9 @@ private static StructOrError parseNonAggregateCluster( "Cluster " + clusterName + ": malformed UpstreamTlsContext: " + e); } } + if (cluster.hasOutlierDetection() && enableOutlierDetection) { + outlierDetection = OutlierDetection.fromEnvoyOutlierDetection(cluster.getOutlierDetection()); + } DiscoveryType type = cluster.getType(); if (type == DiscoveryType.EDS) { @@ -1763,7 +1772,8 @@ private static StructOrError parseNonAggregateCluster( edsResources.add(clusterName); } return StructOrError.fromStruct(CdsUpdate.forEds( - clusterName, edsServiceName, lrsServerInfo, maxConcurrentRequests, upstreamTlsContext)); + clusterName, edsServiceName, lrsServerInfo, maxConcurrentRequests, upstreamTlsContext, + outlierDetection)); } else if (type.equals(DiscoveryType.LOGICAL_DNS)) { if (!cluster.hasLoadAssignment()) { return StructOrError.fromError( diff --git a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java index 91da43eb24ad..96e2cf441a09 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancer.java @@ -38,6 +38,7 @@ import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.util.ForwardingLoadBalancerHelper; import io.grpc.util.GracefulSwitchLoadBalancer; +import io.grpc.util.OutlierDetectionLoadBalancer.OutlierDetectionLoadBalancerConfig; import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.ClusterImplLoadBalancerProvider.ClusterImplConfig; import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig; @@ -45,6 +46,9 @@ import io.grpc.xds.Endpoints.DropOverload; import io.grpc.xds.Endpoints.LbEndpoint; import io.grpc.xds.Endpoints.LocalityLbEndpoints; +import io.grpc.xds.EnvoyServerProtoData.FailurePercentageEjection; +import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; +import io.grpc.xds.EnvoyServerProtoData.SuccessRateEjection; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig.PriorityChildConfig; @@ -176,7 +180,8 @@ public void handleResolvedAddresses(ResolvedAddresses resolvedAddresses) { ClusterState state; if (instance.type == DiscoveryMechanism.Type.EDS) { state = new EdsClusterState(instance.cluster, instance.edsServiceName, - instance.lrsServerInfo, instance.maxConcurrentRequests, instance.tlsContext); + instance.lrsServerInfo, instance.maxConcurrentRequests, instance.tlsContext, + instance.outlierDetection); } else { // logical DNS state = new LogicalDnsClusterState(instance.cluster, instance.dnsHostName, instance.lrsServerInfo, instance.maxConcurrentRequests, instance.tlsContext); @@ -316,6 +321,8 @@ private abstract class ClusterState { protected final Long maxConcurrentRequests; @Nullable protected final UpstreamTlsContext tlsContext; + @Nullable + protected final OutlierDetection outlierDetection; // Resolution status, may contain most recent error encountered. protected Status status = Status.OK; // True if has received resolution result. @@ -327,11 +334,13 @@ private abstract class ClusterState { protected boolean shutdown; private ClusterState(String name, @Nullable ServerInfo lrsServerInfo, - @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext) { + @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext, + @Nullable OutlierDetection outlierDetection) { this.name = name; this.lrsServerInfo = lrsServerInfo; this.maxConcurrentRequests = maxConcurrentRequests; this.tlsContext = tlsContext; + this.outlierDetection = outlierDetection; } abstract void start(); @@ -349,8 +358,8 @@ private final class EdsClusterState extends ClusterState implements EdsResourceW private EdsClusterState(String name, @Nullable String edsServiceName, @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, - @Nullable UpstreamTlsContext tlsContext) { - super(name, lrsServerInfo, maxConcurrentRequests, tlsContext); + @Nullable UpstreamTlsContext tlsContext, @Nullable OutlierDetection outlierDetection) { + super(name, lrsServerInfo, maxConcurrentRequests, tlsContext, outlierDetection); this.edsServiceName = edsServiceName; } @@ -434,7 +443,8 @@ public void run() { Map priorityChildConfigs = generateEdsBasedPriorityChildConfigs( name, edsServiceName, lrsServerInfo, maxConcurrentRequests, tlsContext, - endpointLbPolicy, lbRegistry, prioritizedLocalityWeights, dropOverloads); + outlierDetection, endpointLbPolicy, lbRegistry, prioritizedLocalityWeights, + dropOverloads); status = Status.OK; resolved = true; result = new ClusterResolutionResult(addresses, priorityChildConfigs, @@ -530,7 +540,7 @@ private final class LogicalDnsClusterState extends ClusterState { private LogicalDnsClusterState(String name, String dnsHostName, @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext) { - super(name, lrsServerInfo, maxConcurrentRequests, tlsContext); + super(name, lrsServerInfo, maxConcurrentRequests, tlsContext, null); this.dnsHostName = checkNotNull(dnsHostName, "dnsHostName"); nameResolverFactory = checkNotNull(helper.getNameResolverRegistry().asFactory(), "nameResolverFactory"); @@ -730,9 +740,9 @@ private static PriorityChildConfig generateDnsBasedPriorityChildConfig( private static Map generateEdsBasedPriorityChildConfigs( String cluster, @Nullable String edsServiceName, @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext, - PolicySelection endpointLbPolicy, LoadBalancerRegistry lbRegistry, - Map> prioritizedLocalityWeights, - List dropOverloads) { + @Nullable OutlierDetection outlierDetection, PolicySelection endpointLbPolicy, + LoadBalancerRegistry lbRegistry, Map> prioritizedLocalityWeights, List dropOverloads) { Map configs = new HashMap<>(); for (String priority : prioritizedLocalityWeights.keySet()) { ClusterImplConfig clusterImplConfig = @@ -740,15 +750,98 @@ private static Map generateEdsBasedPriorityChildCon dropOverloads, endpointLbPolicy, tlsContext); LoadBalancerProvider clusterImplLbProvider = lbRegistry.getProvider(XdsLbPolicies.CLUSTER_IMPL_POLICY_NAME); - PolicySelection clusterImplPolicy = + PolicySelection priorityChildPolicy = new PolicySelection(clusterImplLbProvider, clusterImplConfig); + + // If outlier detection has been configured we wrap the child policy in the outlier detection + // load balancer. + if (outlierDetection != null) { + LoadBalancerProvider outlierDetectionProvider = lbRegistry.getProvider( + "outlier_detection_experimental"); + priorityChildPolicy = new PolicySelection(outlierDetectionProvider, + buildOutlierDetectionLbConfig(outlierDetection, priorityChildPolicy)); + } + PriorityChildConfig priorityChildConfig = - new PriorityChildConfig(clusterImplPolicy, true /* ignoreReresolution */); + new PriorityChildConfig(priorityChildPolicy, true /* ignoreReresolution */); configs.put(priority, priorityChildConfig); } return configs; } + /** + * Converts {@link OutlierDetection} that represents the xDS configuration to {@link + * OutlierDetectionLoadBalancerConfig} that the {@link io.grpc.util.OutlierDetectionLoadBalancer} + * understands. + */ + private static OutlierDetectionLoadBalancerConfig buildOutlierDetectionLbConfig( + OutlierDetection outlierDetection, PolicySelection childPolicy) { + OutlierDetectionLoadBalancerConfig.Builder configBuilder + = new OutlierDetectionLoadBalancerConfig.Builder(); + + configBuilder.setChildPolicy(childPolicy); + + if (outlierDetection.intervalNanos() != null) { + configBuilder.setIntervalNanos(outlierDetection.intervalNanos()); + } + if (outlierDetection.baseEjectionTimeNanos() != null) { + configBuilder.setBaseEjectionTimeNanos(outlierDetection.baseEjectionTimeNanos()); + } + if (outlierDetection.maxEjectionTimeNanos() != null) { + configBuilder.setMaxEjectionTimeNanos(outlierDetection.maxEjectionTimeNanos()); + } + if (outlierDetection.maxEjectionPercent() != null) { + configBuilder.setMaxEjectionPercent(outlierDetection.maxEjectionPercent()); + } + + SuccessRateEjection successRate = outlierDetection.successRateEjection(); + if (successRate != null) { + OutlierDetectionLoadBalancerConfig.SuccessRateEjection.Builder + successRateConfigBuilder = new OutlierDetectionLoadBalancerConfig + .SuccessRateEjection.Builder(); + + if (successRate.stdevFactor() != null) { + successRateConfigBuilder.setStdevFactor(successRate.stdevFactor()); + } + if (successRate.enforcementPercentage() != null) { + successRateConfigBuilder.setEnforcementPercentage(successRate.enforcementPercentage()); + } + if (successRate.minimumHosts() != null) { + successRateConfigBuilder.setMinimumHosts(successRate.minimumHosts()); + } + if (successRate.requestVolume() != null) { + successRateConfigBuilder.setRequestVolume(successRate.requestVolume()); + } + + configBuilder.setSuccessRateEjection(successRateConfigBuilder.build()); + } + + FailurePercentageEjection failurePercentage = outlierDetection.failurePercentageEjection(); + if (failurePercentage != null) { + OutlierDetectionLoadBalancerConfig.FailurePercentageEjection.Builder + failurePercentageConfigBuilder = new OutlierDetectionLoadBalancerConfig + .FailurePercentageEjection.Builder(); + + if (failurePercentage.threshold() != null) { + failurePercentageConfigBuilder.setThreshold(failurePercentage.threshold()); + } + if (failurePercentage.enforcementPercentage() != null) { + failurePercentageConfigBuilder.setEnforcementPercentage( + failurePercentage.enforcementPercentage()); + } + if (failurePercentage.minimumHosts() != null) { + failurePercentageConfigBuilder.setMinimumHosts(failurePercentage.minimumHosts()); + } + if (failurePercentage.requestVolume() != null) { + failurePercentageConfigBuilder.setRequestVolume(failurePercentage.requestVolume()); + } + + configBuilder.setFailurePercentageEjection(failurePercentageConfigBuilder.build()); + } + + return configBuilder.build(); + } + /** * Generates a string that represents the priority in the LB policy config. The string is unique * across priorities in all clusters and priorityName(c, p1) < priorityName(c, p2) iff p1 < p2. diff --git a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancerProvider.java b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancerProvider.java index 6f6f887e9252..38da1f465c11 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancerProvider.java +++ b/xds/src/main/java/io/grpc/xds/ClusterResolverLoadBalancerProvider.java @@ -26,6 +26,7 @@ import io.grpc.NameResolver.ConfigOrError; import io.grpc.internal.ServiceConfigUtil.PolicySelection; import io.grpc.xds.Bootstrapper.ServerInfo; +import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import java.util.List; import java.util.Map; @@ -124,6 +125,8 @@ static final class DiscoveryMechanism { // Hostname for resolving endpoints via DNS. Only valid for LOGICAL_DNS clusters. @Nullable final String dnsHostName; + @Nullable + final OutlierDetection outlierDetection; enum Type { EDS, @@ -132,7 +135,8 @@ enum Type { private DiscoveryMechanism(String cluster, Type type, @Nullable String edsServiceName, @Nullable String dnsHostName, @Nullable ServerInfo lrsServerInfo, - @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext) { + @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext, + @Nullable OutlierDetection outlierDetection) { this.cluster = checkNotNull(cluster, "cluster"); this.type = checkNotNull(type, "type"); this.edsServiceName = edsServiceName; @@ -140,20 +144,22 @@ private DiscoveryMechanism(String cluster, Type type, @Nullable String edsServic this.lrsServerInfo = lrsServerInfo; this.maxConcurrentRequests = maxConcurrentRequests; this.tlsContext = tlsContext; + this.outlierDetection = outlierDetection; } static DiscoveryMechanism forEds(String cluster, @Nullable String edsServiceName, @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, - @Nullable UpstreamTlsContext tlsContext) { + @Nullable UpstreamTlsContext tlsContext, + OutlierDetection outlierDetection) { return new DiscoveryMechanism(cluster, Type.EDS, edsServiceName, null, lrsServerInfo, - maxConcurrentRequests, tlsContext); + maxConcurrentRequests, tlsContext, outlierDetection); } static DiscoveryMechanism forLogicalDns(String cluster, String dnsHostName, @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, @Nullable UpstreamTlsContext tlsContext) { return new DiscoveryMechanism(cluster, Type.LOGICAL_DNS, null, dnsHostName, - lrsServerInfo, maxConcurrentRequests, tlsContext); + lrsServerInfo, maxConcurrentRequests, tlsContext, null); } @Override diff --git a/xds/src/main/java/io/grpc/xds/EnvoyServerProtoData.java b/xds/src/main/java/io/grpc/xds/EnvoyServerProtoData.java index e53439755be5..ce088efca6cb 100644 --- a/xds/src/main/java/io/grpc/xds/EnvoyServerProtoData.java +++ b/xds/src/main/java/io/grpc/xds/EnvoyServerProtoData.java @@ -19,6 +19,7 @@ import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; +import com.google.protobuf.util.Durations; import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; import io.grpc.Internal; import io.grpc.xds.internal.sds.SslContextProviderSupplier; @@ -254,4 +255,148 @@ static Listener create( defaultFilterChain); } } + + /** + * Corresponds to Envoy proto message {@link + * io.envoyproxy.envoy.config.cluster.v3.OutlierDetection}.Only the fields supported by gRPC are + * included. + * + *

Protobuf Duration fields are represented in their string format (e.g. "10s"). + */ + @AutoValue + abstract static class OutlierDetection { + + @Nullable + abstract Long intervalNanos(); + + @Nullable + abstract Long baseEjectionTimeNanos(); + + @Nullable + abstract Long maxEjectionTimeNanos(); + + @Nullable + abstract Integer maxEjectionPercent(); + + @Nullable + abstract SuccessRateEjection successRateEjection(); + + @Nullable + abstract FailurePercentageEjection failurePercentageEjection(); + + static OutlierDetection create( + @Nullable Long intervalNanos, + @Nullable Long baseEjectionTimeNanos, + @Nullable Long maxEjectionTimeNanos, + @Nullable Integer maxEjectionPercentage, + @Nullable SuccessRateEjection successRateEjection, + @Nullable FailurePercentageEjection failurePercentageEjection) { + return new AutoValue_EnvoyServerProtoData_OutlierDetection(intervalNanos, + baseEjectionTimeNanos, maxEjectionTimeNanos, maxEjectionPercentage, successRateEjection, + failurePercentageEjection); + } + + static OutlierDetection fromEnvoyOutlierDetection( + io.envoyproxy.envoy.config.cluster.v3.OutlierDetection envoyOutlierDetection) { + + Long intervalNanos = envoyOutlierDetection.hasInterval() + ? Durations.toNanos(envoyOutlierDetection.getInterval()) : null; + Long baseEjectionTimeNanos = envoyOutlierDetection.hasBaseEjectionTime() + ? Durations.toNanos(envoyOutlierDetection.getBaseEjectionTime()) : null; + Long maxEjectionTimeNanos = envoyOutlierDetection.hasMaxEjectionTime() + ? Durations.toNanos(envoyOutlierDetection.getMaxEjectionTime()) : null; + Integer maxEjectionPercentage = envoyOutlierDetection.hasMaxEjectionPercent() + ? envoyOutlierDetection.getMaxEjectionPercent().getValue() : null; + + SuccessRateEjection successRateEjection; + // If success rate enforcement has been turned completely off, don't configure this ejection. + if (envoyOutlierDetection.hasEnforcingSuccessRate() + && envoyOutlierDetection.getEnforcingSuccessRate().getValue() == 0) { + successRateEjection = null; + } else { + Integer stdevFactor = envoyOutlierDetection.hasSuccessRateStdevFactor() + ? envoyOutlierDetection.getSuccessRateStdevFactor().getValue() : null; + Integer enforcementPercentage = envoyOutlierDetection.hasEnforcingSuccessRate() + ? envoyOutlierDetection.getEnforcingSuccessRate().getValue() : null; + Integer minimumHosts = envoyOutlierDetection.hasSuccessRateMinimumHosts() + ? envoyOutlierDetection.getSuccessRateMinimumHosts().getValue() : null; + Integer requestVolume = envoyOutlierDetection.hasSuccessRateRequestVolume() + ? envoyOutlierDetection.getSuccessRateMinimumHosts().getValue() : null; + + successRateEjection = SuccessRateEjection.create(stdevFactor, enforcementPercentage, + minimumHosts, requestVolume); + } + + FailurePercentageEjection failurePercentageEjection; + if (envoyOutlierDetection.hasEnforcingFailurePercentage() + && envoyOutlierDetection.getEnforcingFailurePercentage().getValue() == 0) { + failurePercentageEjection = null; + } else { + Integer threshold = envoyOutlierDetection.hasFailurePercentageThreshold() + ? envoyOutlierDetection.getFailurePercentageThreshold().getValue() : null; + Integer enforcementPercentage = envoyOutlierDetection.hasEnforcingFailurePercentage() + ? envoyOutlierDetection.getEnforcingFailurePercentage().getValue() : null; + Integer minimumHosts = envoyOutlierDetection.hasFailurePercentageMinimumHosts() + ? envoyOutlierDetection.getFailurePercentageMinimumHosts().getValue() : null; + Integer requestVolume = envoyOutlierDetection.hasFailurePercentageRequestVolume() + ? envoyOutlierDetection.getFailurePercentageRequestVolume().getValue() : null; + + failurePercentageEjection = FailurePercentageEjection.create(threshold, + enforcementPercentage, minimumHosts, requestVolume); + } + + return create(intervalNanos, baseEjectionTimeNanos, maxEjectionTimeNanos, + maxEjectionPercentage, successRateEjection, failurePercentageEjection); + } + } + + @AutoValue + abstract static class SuccessRateEjection { + + @Nullable + abstract Integer stdevFactor(); + + @Nullable + abstract Integer enforcementPercentage(); + + @Nullable + abstract Integer minimumHosts(); + + @Nullable + abstract Integer requestVolume(); + + static SuccessRateEjection create( + @Nullable Integer stdevFactor, + @Nullable Integer enforcementPercentage, + @Nullable Integer minimumHosts, + @Nullable Integer requestVolume) { + return new AutoValue_EnvoyServerProtoData_SuccessRateEjection(stdevFactor, + enforcementPercentage, minimumHosts, requestVolume); + } + } + + @AutoValue + abstract static class FailurePercentageEjection { + + @Nullable + abstract Integer threshold(); + + @Nullable + abstract Integer enforcementPercentage(); + + @Nullable + abstract Integer minimumHosts(); + + @Nullable + abstract Integer requestVolume(); + + static FailurePercentageEjection create( + @Nullable Integer threshold, + @Nullable Integer enforcementPercentage, + @Nullable Integer minimumHosts, + @Nullable Integer requestVolume) { + return new AutoValue_EnvoyServerProtoData_FailurePercentageEjection(threshold, + enforcementPercentage, minimumHosts, requestVolume); + } + } } diff --git a/xds/src/main/java/io/grpc/xds/XdsClient.java b/xds/src/main/java/io/grpc/xds/XdsClient.java index bdffc3611930..028ae155ba6c 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClient.java +++ b/xds/src/main/java/io/grpc/xds/XdsClient.java @@ -34,6 +34,7 @@ import io.grpc.xds.Endpoints.DropOverload; import io.grpc.xds.Endpoints.LocalityLbEndpoints; import io.grpc.xds.EnvoyServerProtoData.Listener; +import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.LoadStatsManager2.ClusterDropStats; import io.grpc.xds.LoadStatsManager2.ClusterLocalityStats; @@ -221,6 +222,10 @@ abstract static class CdsUpdate implements ResourceUpdate { @Nullable abstract ImmutableList prioritizedClusterNames(); + // Outlier detection configuration. + @Nullable + abstract OutlierDetection outlierDetection(); + static Builder forAggregate(String clusterName, List prioritizedClusterNames) { checkNotNull(prioritizedClusterNames, "prioritizedClusterNames"); return new AutoValue_XdsClient_CdsUpdate.Builder() @@ -234,7 +239,8 @@ static Builder forAggregate(String clusterName, List prioritizedClusterN static Builder forEds(String clusterName, @Nullable String edsServiceName, @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, - @Nullable UpstreamTlsContext upstreamTlsContext) { + @Nullable UpstreamTlsContext upstreamTlsContext, + @Nullable OutlierDetection outlierDetection) { return new AutoValue_XdsClient_CdsUpdate.Builder() .clusterName(clusterName) .clusterType(ClusterType.EDS) @@ -244,7 +250,8 @@ static Builder forEds(String clusterName, @Nullable String edsServiceName, .edsServiceName(edsServiceName) .lrsServerInfo(lrsServerInfo) .maxConcurrentRequests(maxConcurrentRequests) - .upstreamTlsContext(upstreamTlsContext); + .upstreamTlsContext(upstreamTlsContext) + .outlierDetection(outlierDetection); } static Builder forLogicalDns(String clusterName, String dnsHostName, @@ -284,8 +291,9 @@ public final String toString() { .add("dnsHostName", dnsHostName()) .add("lrsServerInfo", lrsServerInfo()) .add("maxConcurrentRequests", maxConcurrentRequests()) - // Exclude upstreamTlsContext as its string representation is cumbersome. .add("prioritizedClusterNames", prioritizedClusterNames()) + // Exclude upstreamTlsContext and outlierDetection as their string representations are + // cumbersome. .toString(); } @@ -341,6 +349,8 @@ Builder leastRequestLbPolicy(Integer choiceCount) { // Private, use CdsUpdate.forAggregate() instead. protected abstract Builder prioritizedClusterNames(List prioritizedClusterNames); + protected abstract Builder outlierDetection(OutlierDetection outlierDetection); + abstract CdsUpdate build(); } } diff --git a/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java b/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java index a90df303e048..128d7bd395ea 100644 --- a/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java +++ b/xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java @@ -50,6 +50,8 @@ import io.grpc.xds.CdsLoadBalancerProvider.CdsConfig; import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig; import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig.DiscoveryMechanism; +import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; +import io.grpc.xds.EnvoyServerProtoData.SuccessRateEjection; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.LeastRequestLoadBalancer.LeastRequestConfig; import io.grpc.xds.RingHashLoadBalancer.RingHashConfig; @@ -85,6 +87,9 @@ public class CdsLoadBalancer2Test { ServerInfo.create("lrs.googleapis.com", InsecureChannelCredentials.create(), true); private final UpstreamTlsContext upstreamTlsContext = CommonTlsContextTestsUtil.buildUpstreamTlsContext("google_cloud_private_spiffe", true); + private final OutlierDetection outlierDetection = OutlierDetection.create( + null, null, null, null, SuccessRateEjection.create(null, null, null, null), null); + private final SynchronizationContext syncContext = new SynchronizationContext( new Thread.UncaughtExceptionHandler() { @@ -154,7 +159,8 @@ public void tearDown() { @Test public void discoverTopLevelEdsCluster() { CdsUpdate update = - CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext) + CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext, + outlierDetection) .roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(CLUSTER, update); assertThat(childBalancers).hasSize(1); @@ -164,7 +170,7 @@ public void discoverTopLevelEdsCluster() { assertThat(childLbConfig.discoveryMechanisms).hasSize(1); DiscoveryMechanism instance = Iterables.getOnlyElement(childLbConfig.discoveryMechanisms); assertDiscoveryMechanism(instance, CLUSTER, DiscoveryMechanism.Type.EDS, EDS_SERVICE_NAME, - null, LRS_SERVER_INFO, 100L, upstreamTlsContext); + null, LRS_SERVER_INFO, 100L, upstreamTlsContext, outlierDetection); assertThat(childLbConfig.lbPolicy.getProvider().getPolicyName()).isEqualTo("round_robin"); } @@ -181,7 +187,7 @@ public void discoverTopLevelLogicalDnsCluster() { assertThat(childLbConfig.discoveryMechanisms).hasSize(1); DiscoveryMechanism instance = Iterables.getOnlyElement(childLbConfig.discoveryMechanisms); assertDiscoveryMechanism(instance, CLUSTER, DiscoveryMechanism.Type.LOGICAL_DNS, null, - DNS_HOST_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext); + DNS_HOST_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext, null); assertThat(childLbConfig.lbPolicy.getProvider().getPolicyName()) .isEqualTo("least_request_experimental"); assertThat(((LeastRequestConfig) childLbConfig.lbPolicy.getConfig()).choiceCount).isEqualTo(3); @@ -201,7 +207,7 @@ public void nonAggregateCluster_resourceNotExist_returnErrorPicker() { @Test public void nonAggregateCluster_resourceUpdate() { CdsUpdate update = - CdsUpdate.forEds(CLUSTER, null, null, 100L, upstreamTlsContext) + CdsUpdate.forEds(CLUSTER, null, null, 100L, upstreamTlsContext, outlierDetection) .roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(CLUSTER, update); assertThat(childBalancers).hasSize(1); @@ -209,15 +215,15 @@ public void nonAggregateCluster_resourceUpdate() { ClusterResolverConfig childLbConfig = (ClusterResolverConfig) childBalancer.config; DiscoveryMechanism instance = Iterables.getOnlyElement(childLbConfig.discoveryMechanisms); assertDiscoveryMechanism(instance, CLUSTER, DiscoveryMechanism.Type.EDS, null, null, null, - 100L, upstreamTlsContext); + 100L, upstreamTlsContext, outlierDetection); - update = CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, null) - .roundRobinLbPolicy().build(); + update = CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, null, + outlierDetection).roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(CLUSTER, update); childLbConfig = (ClusterResolverConfig) childBalancer.config; instance = Iterables.getOnlyElement(childLbConfig.discoveryMechanisms); assertDiscoveryMechanism(instance, CLUSTER, DiscoveryMechanism.Type.EDS, EDS_SERVICE_NAME, - null, LRS_SERVER_INFO, 200L, null); + null, LRS_SERVER_INFO, 200L, null, outlierDetection); } @Test @@ -231,7 +237,7 @@ public void nonAggregateCluster_resourceRevoked() { ClusterResolverConfig childLbConfig = (ClusterResolverConfig) childBalancer.config; DiscoveryMechanism instance = Iterables.getOnlyElement(childLbConfig.discoveryMechanisms); assertDiscoveryMechanism(instance, CLUSTER, DiscoveryMechanism.Type.LOGICAL_DNS, null, - DNS_HOST_NAME, null, 100L, upstreamTlsContext); + DNS_HOST_NAME, null, 100L, upstreamTlsContext, null); xdsClient.deliverResourceNotExist(CLUSTER); assertThat(childBalancer.shutdown).isTrue(); @@ -265,9 +271,8 @@ public void discoverAggregateCluster() { assertThat(xdsClient.watchers.keySet()).containsExactly( CLUSTER, cluster1, cluster2, cluster3, cluster4); assertThat(childBalancers).isEmpty(); - CdsUpdate update3 = - CdsUpdate.forEds(cluster3, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, upstreamTlsContext) - .roundRobinLbPolicy().build(); + CdsUpdate update3 = CdsUpdate.forEds(cluster3, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, + upstreamTlsContext, outlierDetection).roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(cluster3, update3); assertThat(childBalancers).isEmpty(); CdsUpdate update2 = @@ -276,7 +281,7 @@ public void discoverAggregateCluster() { xdsClient.deliverCdsUpdate(cluster2, update2); assertThat(childBalancers).isEmpty(); CdsUpdate update4 = - CdsUpdate.forEds(cluster4, null, LRS_SERVER_INFO, 300L, null) + CdsUpdate.forEds(cluster4, null, LRS_SERVER_INFO, 300L, null, outlierDetection) .roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(cluster4, update4); assertThat(childBalancers).hasSize(1); // all non-aggregate clusters discovered @@ -286,12 +291,12 @@ public void discoverAggregateCluster() { assertThat(childLbConfig.discoveryMechanisms).hasSize(3); // Clusters on higher level has higher priority: [cluster2, cluster3, cluster4] assertDiscoveryMechanism(childLbConfig.discoveryMechanisms.get(0), cluster2, - DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, null, 100L, null); + DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, null, 100L, null, null); assertDiscoveryMechanism(childLbConfig.discoveryMechanisms.get(1), cluster3, DiscoveryMechanism.Type.EDS, EDS_SERVICE_NAME, null, LRS_SERVER_INFO, 200L, - upstreamTlsContext); + upstreamTlsContext, outlierDetection); assertDiscoveryMechanism(childLbConfig.discoveryMechanisms.get(2), cluster4, - DiscoveryMechanism.Type.EDS, null, null, LRS_SERVER_INFO, 300L, null); + DiscoveryMechanism.Type.EDS, null, null, LRS_SERVER_INFO, 300L, null, outlierDetection); assertThat(childLbConfig.lbPolicy.getProvider().getPolicyName()) .isEqualTo("ring_hash_experimental"); // dominated by top-level cluster's config assertThat(((RingHashConfig) childLbConfig.lbPolicy.getConfig()).minRingSize).isEqualTo(100L); @@ -326,9 +331,8 @@ public void aggregateCluster_descendantClustersRevoked() { .roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(CLUSTER, update); assertThat(xdsClient.watchers.keySet()).containsExactly(CLUSTER, cluster1, cluster2); - CdsUpdate update1 = - CdsUpdate.forEds(cluster1, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, upstreamTlsContext) - .roundRobinLbPolicy().build(); + CdsUpdate update1 = CdsUpdate.forEds(cluster1, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, + upstreamTlsContext, outlierDetection).roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(cluster1, update1); CdsUpdate update2 = CdsUpdate.forLogicalDns(cluster2, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null) @@ -339,9 +343,10 @@ public void aggregateCluster_descendantClustersRevoked() { assertThat(childLbConfig.discoveryMechanisms).hasSize(2); assertDiscoveryMechanism(childLbConfig.discoveryMechanisms.get(0), cluster1, DiscoveryMechanism.Type.EDS, EDS_SERVICE_NAME, null, LRS_SERVER_INFO, 200L, - upstreamTlsContext); + upstreamTlsContext, outlierDetection); assertDiscoveryMechanism(childLbConfig.discoveryMechanisms.get(1), cluster2, - DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null); + DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null, + null); // Revoke cluster1, should still be able to proceed with cluster2. xdsClient.deliverResourceNotExist(cluster1); @@ -349,7 +354,8 @@ public void aggregateCluster_descendantClustersRevoked() { childLbConfig = (ClusterResolverConfig) childBalancer.config; assertThat(childLbConfig.discoveryMechanisms).hasSize(1); assertDiscoveryMechanism(Iterables.getOnlyElement(childLbConfig.discoveryMechanisms), cluster2, - DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null); + DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null, + null); verify(helper, never()).updateBalancingState( eq(ConnectivityState.TRANSIENT_FAILURE), any(SubchannelPicker.class)); @@ -374,9 +380,8 @@ public void aggregateCluster_rootClusterRevoked() { .roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(CLUSTER, update); assertThat(xdsClient.watchers.keySet()).containsExactly(CLUSTER, cluster1, cluster2); - CdsUpdate update1 = - CdsUpdate.forEds(cluster1, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, upstreamTlsContext) - .roundRobinLbPolicy().build(); + CdsUpdate update1 = CdsUpdate.forEds(cluster1, EDS_SERVICE_NAME, LRS_SERVER_INFO, 200L, + upstreamTlsContext, outlierDetection).roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(cluster1, update1); CdsUpdate update2 = CdsUpdate.forLogicalDns(cluster2, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null) @@ -387,9 +392,10 @@ public void aggregateCluster_rootClusterRevoked() { assertThat(childLbConfig.discoveryMechanisms).hasSize(2); assertDiscoveryMechanism(childLbConfig.discoveryMechanisms.get(0), cluster1, DiscoveryMechanism.Type.EDS, EDS_SERVICE_NAME, null, LRS_SERVER_INFO, 200L, - upstreamTlsContext); + upstreamTlsContext, outlierDetection); assertDiscoveryMechanism(childLbConfig.discoveryMechanisms.get(1), cluster2, - DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null); + DiscoveryMechanism.Type.LOGICAL_DNS, null, DNS_HOST_NAME, LRS_SERVER_INFO, 100L, null, + null); xdsClient.deliverResourceNotExist(CLUSTER); assertThat(xdsClient.watchers.keySet()) @@ -428,16 +434,15 @@ public void aggregateCluster_intermediateClusterChanges() { .roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(cluster2, update2); assertThat(xdsClient.watchers.keySet()).containsExactly(CLUSTER, cluster2, cluster3); - CdsUpdate update3 = - CdsUpdate.forEds(cluster3, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext) - .roundRobinLbPolicy().build(); + CdsUpdate update3 = CdsUpdate.forEds(cluster3, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, + upstreamTlsContext, outlierDetection).roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(cluster3, update3); FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); ClusterResolverConfig childLbConfig = (ClusterResolverConfig) childBalancer.config; assertThat(childLbConfig.discoveryMechanisms).hasSize(1); DiscoveryMechanism instance = Iterables.getOnlyElement(childLbConfig.discoveryMechanisms); assertDiscoveryMechanism(instance, cluster3, DiscoveryMechanism.Type.EDS, EDS_SERVICE_NAME, - null, LRS_SERVER_INFO, 100L, upstreamTlsContext); + null, LRS_SERVER_INFO, 100L, upstreamTlsContext, outlierDetection); // cluster2 revoked xdsClient.deliverResourceNotExist(cluster2); @@ -506,9 +511,8 @@ public void handleNameResolutionErrorFromUpstream_beforeChildLbCreated_returnErr @Test public void handleNameResolutionErrorFromUpstream_afterChildLbCreated_fallThrough() { - CdsUpdate update = - CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext) - .roundRobinLbPolicy().build(); + CdsUpdate update = CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, + upstreamTlsContext, outlierDetection).roundRobinLbPolicy().build(); xdsClient.deliverCdsUpdate(CLUSTER, update); FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); assertThat(childBalancer.shutdown).isFalse(); @@ -523,7 +527,8 @@ public void handleNameResolutionErrorFromUpstream_afterChildLbCreated_fallThroug public void unknownLbProvider() { try { xdsClient.deliverCdsUpdate(CLUSTER, - CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext) + CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext, + outlierDetection) .lbPolicyConfig(ImmutableMap.of("unknown", ImmutableMap.of("foo", "bar"))).build()); } catch (Exception e) { assertThat(e).hasCauseThat().hasMessageThat().contains("No provider available"); @@ -536,8 +541,8 @@ public void unknownLbProvider() { public void invalidLbConfig() { try { xdsClient.deliverCdsUpdate(CLUSTER, - CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext) - .lbPolicyConfig( + CdsUpdate.forEds(CLUSTER, EDS_SERVICE_NAME, LRS_SERVER_INFO, 100L, upstreamTlsContext, + outlierDetection).lbPolicyConfig( ImmutableMap.of("ring_hash_experimental", ImmutableMap.of("minRingSize", "-1"))) .build()); } catch (Exception e) { @@ -561,7 +566,7 @@ private static void assertPicker(SubchannelPicker picker, Status expectedStatus, private static void assertDiscoveryMechanism(DiscoveryMechanism instance, String name, DiscoveryMechanism.Type type, @Nullable String edsServiceName, @Nullable String dnsHostName, @Nullable ServerInfo lrsServerInfo, @Nullable Long maxConcurrentRequests, - @Nullable UpstreamTlsContext tlsContext) { + @Nullable UpstreamTlsContext tlsContext, @Nullable OutlierDetection outlierDetection) { assertThat(instance.cluster).isEqualTo(name); assertThat(instance.type).isEqualTo(type); assertThat(instance.edsServiceName).isEqualTo(edsServiceName); @@ -569,6 +574,7 @@ private static void assertDiscoveryMechanism(DiscoveryMechanism instance, String assertThat(instance.lrsServerInfo).isEqualTo(lrsServerInfo); assertThat(instance.maxConcurrentRequests).isEqualTo(maxConcurrentRequests); assertThat(instance.tlsContext).isEqualTo(tlsContext); + assertThat(instance.outlierDetection).isEqualTo(outlierDetection); } private final class FakeLoadBalancerProvider extends LoadBalancerProvider { diff --git a/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java index 22c54bbb94ae..f18643260e28 100644 --- a/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java @@ -58,6 +58,8 @@ import io.grpc.internal.GrpcUtil; import io.grpc.internal.ObjectPool; import io.grpc.internal.ServiceConfigUtil.PolicySelection; +import io.grpc.util.OutlierDetectionLoadBalancer.OutlierDetectionLoadBalancerConfig; +import io.grpc.util.OutlierDetectionLoadBalancerProvider; import io.grpc.xds.Bootstrapper.ServerInfo; import io.grpc.xds.ClusterImplLoadBalancerProvider.ClusterImplConfig; import io.grpc.xds.ClusterResolverLoadBalancerProvider.ClusterResolverConfig; @@ -65,6 +67,9 @@ import io.grpc.xds.Endpoints.DropOverload; import io.grpc.xds.Endpoints.LbEndpoint; import io.grpc.xds.Endpoints.LocalityLbEndpoints; +import io.grpc.xds.EnvoyServerProtoData.FailurePercentageEjection; +import io.grpc.xds.EnvoyServerProtoData.OutlierDetection; +import io.grpc.xds.EnvoyServerProtoData.SuccessRateEjection; import io.grpc.xds.EnvoyServerProtoData.UpstreamTlsContext; import io.grpc.xds.LeastRequestLoadBalancer.LeastRequestConfig; import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig; @@ -116,10 +121,18 @@ public class ClusterResolverLoadBalancerTest { Locality.create("test-region-3", "test-zone-3", "test-subzone-3"); private final UpstreamTlsContext tlsContext = CommonTlsContextTestsUtil.buildUpstreamTlsContext("google_cloud_private_spiffe", true); + private final OutlierDetection outlierDetection = OutlierDetection.create( + 100L, 100L, 100L, 100, SuccessRateEjection.create(100, 100, 100, 100), + FailurePercentageEjection.create(100, 100, 100, 100)); private final DiscoveryMechanism edsDiscoveryMechanism1 = - DiscoveryMechanism.forEds(CLUSTER1, EDS_SERVICE_NAME1, LRS_SERVER_INFO, 100L, tlsContext); + DiscoveryMechanism.forEds(CLUSTER1, EDS_SERVICE_NAME1, LRS_SERVER_INFO, 100L, tlsContext, + null); private final DiscoveryMechanism edsDiscoveryMechanism2 = - DiscoveryMechanism.forEds(CLUSTER2, EDS_SERVICE_NAME2, LRS_SERVER_INFO, 200L, tlsContext); + DiscoveryMechanism.forEds(CLUSTER2, EDS_SERVICE_NAME2, LRS_SERVER_INFO, 200L, tlsContext, + null); + private final DiscoveryMechanism edsDiscoveryMechanismWithOutlierDetection = + DiscoveryMechanism.forEds(CLUSTER1, EDS_SERVICE_NAME1, LRS_SERVER_INFO, 100L, tlsContext, + outlierDetection); private final DiscoveryMechanism logicalDnsDiscoveryMechanism = DiscoveryMechanism.forLogicalDns(CLUSTER_DNS, DNS_HOST_NAME, LRS_SERVER_INFO, 300L, null); @@ -182,6 +195,7 @@ public void setUp() throws URISyntaxException { lbRegistry.register(new FakeLoadBalancerProvider(WEIGHTED_TARGET_POLICY_NAME)); lbRegistry.register( new FakeLoadBalancerProvider("pick_first")); // needed by logical_dns + lbRegistry.register(new OutlierDetectionLoadBalancerProvider()); NameResolver.Args args = NameResolver.Args.newBuilder() .setDefaultPort(8080) .setProxyDetector(GrpcUtil.NOOP_PROXY_DETECTOR) @@ -317,6 +331,90 @@ public void edsClustersWithLeastRequestEndpointLbPolicy() { locality1, 100); } + @Test + public void edsClustersWithOutlierDetection() { + ClusterResolverConfig config = new ClusterResolverConfig( + Collections.singletonList(edsDiscoveryMechanismWithOutlierDetection), leastRequest); + deliverLbConfig(config); + assertThat(xdsClient.watchers.keySet()).containsExactly(EDS_SERVICE_NAME1); + assertThat(childBalancers).isEmpty(); + + // Simple case with one priority and one locality + EquivalentAddressGroup endpoint = makeAddress("endpoint-addr-1"); + LocalityLbEndpoints localityLbEndpoints = + LocalityLbEndpoints.create( + Arrays.asList( + LbEndpoint.create(endpoint, 0 /* loadBalancingWeight */, true)), + 100 /* localityWeight */, 1 /* priority */); + xdsClient.deliverClusterLoadAssignment( + EDS_SERVICE_NAME1, + ImmutableMap.of(locality1, localityLbEndpoints)); + assertThat(childBalancers).hasSize(1); + FakeLoadBalancer childBalancer = Iterables.getOnlyElement(childBalancers); + assertThat(childBalancer.addresses).hasSize(1); + EquivalentAddressGroup addr = childBalancer.addresses.get(0); + assertThat(addr.getAddresses()).isEqualTo(endpoint.getAddresses()); + assertThat(childBalancer.name).isEqualTo(PRIORITY_POLICY_NAME); + PriorityLbConfig priorityLbConfig = (PriorityLbConfig) childBalancer.config; + assertThat(priorityLbConfig.priorities).containsExactly(CLUSTER1 + "[child1]"); + PriorityChildConfig priorityChildConfig = + Iterables.getOnlyElement(priorityLbConfig.childConfigs.values()); + + // The child config for priority should be outlier detection. + assertThat(priorityChildConfig.policySelection.getProvider().getPolicyName()) + .isEqualTo("outlier_detection_experimental"); + OutlierDetectionLoadBalancerConfig outlierDetectionConfig = + (OutlierDetectionLoadBalancerConfig) priorityChildConfig.policySelection.getConfig(); + + // The outlier detection config should faithfully represent what came down from xDS. + assertThat(outlierDetectionConfig.intervalNanos).isEqualTo(outlierDetection.intervalNanos()); + assertThat(outlierDetectionConfig.baseEjectionTimeNanos).isEqualTo( + outlierDetection.baseEjectionTimeNanos()); + assertThat(outlierDetectionConfig.baseEjectionTimeNanos).isEqualTo( + outlierDetection.baseEjectionTimeNanos()); + assertThat(outlierDetectionConfig.maxEjectionTimeNanos).isEqualTo( + outlierDetection.maxEjectionTimeNanos()); + assertThat(outlierDetectionConfig.maxEjectionPercent).isEqualTo( + outlierDetection.maxEjectionPercent()); + + OutlierDetectionLoadBalancerConfig.SuccessRateEjection successRateEjection + = outlierDetectionConfig.successRateEjection; + assertThat(successRateEjection.stdevFactor).isEqualTo( + outlierDetectionConfig.successRateEjection.stdevFactor); + assertThat(successRateEjection.enforcementPercentage).isEqualTo( + outlierDetectionConfig.successRateEjection.enforcementPercentage); + assertThat(successRateEjection.minimumHosts).isEqualTo( + outlierDetectionConfig.successRateEjection.minimumHosts); + assertThat(successRateEjection.requestVolume).isEqualTo( + outlierDetectionConfig.successRateEjection.requestVolume); + + OutlierDetectionLoadBalancerConfig.FailurePercentageEjection failurePercentageEjection + = outlierDetectionConfig.failurePercentageEjection; + assertThat(failurePercentageEjection.threshold).isEqualTo( + outlierDetection.failurePercentageEjection().threshold()); + assertThat(failurePercentageEjection.enforcementPercentage).isEqualTo( + outlierDetection.failurePercentageEjection().enforcementPercentage()); + assertThat(failurePercentageEjection.minimumHosts).isEqualTo( + outlierDetection.failurePercentageEjection().minimumHosts()); + assertThat(failurePercentageEjection.requestVolume).isEqualTo( + outlierDetection.failurePercentageEjection().requestVolume()); + + // The wrapped configuration should not have been tampered with. + ClusterImplConfig clusterImplConfig = + (ClusterImplConfig) outlierDetectionConfig.childPolicy.getConfig(); + assertClusterImplConfig(clusterImplConfig, CLUSTER1, EDS_SERVICE_NAME1, LRS_SERVER_INFO, 100L, + tlsContext, Collections.emptyList(), WRR_LOCALITY_POLICY_NAME); + WrrLocalityConfig wrrLocalityConfig = + (WrrLocalityConfig) clusterImplConfig.childPolicy.getConfig(); + assertThat(wrrLocalityConfig.childPolicy.getProvider().getPolicyName()).isEqualTo( + "least_request_experimental"); + + assertThat( + childBalancer.attributes.get(InternalXdsAttributes.ATTR_LOCALITY_WEIGHTS)).containsEntry( + locality1, 100); + } + + @Test public void onlyEdsClusters_receivedEndpoints() { ClusterResolverConfig config = new ClusterResolverConfig(