xds: Implement ignore_resource_deletion server feature

xds/src/main/java/io/grpc/xds/Bootstrapper.java

@@ -62,10 +62,14 @@ abstract static class ServerInfo {
 
     abstract boolean useProtocolV3();
 
+    abstract boolean ignoreResourceDeletion();
+
     @VisibleForTesting
     static ServerInfo create(
-        String target, ChannelCredentials channelCredentials, boolean useProtocolV3) {
-      return new AutoValue_Bootstrapper_ServerInfo(target, channelCredentials, useProtocolV3);
+        String target, ChannelCredentials channelCredentials, boolean useProtocolV3,
+        boolean ignoreResourceDeletion) {
+      return new AutoValue_Bootstrapper_ServerInfo(
+          target, channelCredentials, useProtocolV3, ignoreResourceDeletion);
     }
   }
 

xds/src/main/java/io/grpc/xds/BootstrapperImpl.java

@@ -55,6 +55,7 @@ class BootstrapperImpl extends Bootstrapper {
   @VisibleForTesting
   static String bootstrapConfigFromSysProp = System.getProperty(BOOTSTRAP_CONFIG_SYS_PROPERTY);
   private static final String XDS_V3_SERVER_FEATURE = "xds_v3";
+  private static final String IGNORE_RESOURCE_DELETION_FEATURE = "ignore_resource_deletion";
   @VisibleForTesting
   static final String CLIENT_FEATURE_DISABLE_OVERPROVISIONING =
       "envoy.lb.does_not_support_overprovisioning";
@@ -275,12 +276,15 @@ private static List<ServerInfo> parseServerInfos(List<?> rawServerConfigs, XdsLo
       }
 
       boolean useProtocolV3 = false;
+      boolean ignoreResourceDeletion = false;
       List<String> serverFeatures = JsonUtil.getListOfStrings(serverConfig, "server_features");
       if (serverFeatures != null) {
         logger.log(XdsLogLevel.INFO, "Server features: {0}", serverFeatures);
         useProtocolV3 = serverFeatures.contains(XDS_V3_SERVER_FEATURE);
+        ignoreResourceDeletion = serverFeatures.contains(IGNORE_RESOURCE_DELETION_FEATURE);
       }
-      servers.add(ServerInfo.create(serverUri, channelCredentials, useProtocolV3));
+      servers.add(
+          ServerInfo.create(serverUri, channelCredentials, useProtocolV3, ignoreResourceDeletion));
     }
     return servers.build();
   }

xds/src/main/java/io/grpc/xds/ClientXdsClient.java

@@ -2438,6 +2438,7 @@ private final class ResourceSubscriber {
     private final Set<ResourceWatcher> watchers = new HashSet<>();
     private ResourceUpdate data;
     private boolean absent;
+    private boolean resourceDeletionIgnored;
     private ScheduledHandle respTimer;
     private ResourceMetadata metadata;
     @Nullable private String errorDescription;
@@ -2552,12 +2553,26 @@ void onData(ParsedResource parsedResource, String version, long updateTime) {
           notifyWatcher(watcher, data);
         }
       }
+      if (resourceDeletionIgnored) {
+        logger.log(
+            XdsLogLevel.INFO, "Recovered deleted resource type {0} name {1}", type, resource);
+        resourceDeletionIgnored = false;
+      }
     }
 
     void onAbsent() {
       if (respTimer != null && respTimer.isPending()) {  // too early to conclude absence
         return;
       }
+      if (serverInfo.ignoreResourceDeletion()) {
+        // Only log warning if not previously logged
+        if (!resourceDeletionIgnored) {
+          logger.log(
+              XdsLogLevel.WARNING, "Ignoring deletion type {0} name {1}", type, resource);
+          resourceDeletionIgnored = true;
+        }
+        return;
+      }
       logger.log(XdsLogLevel.INFO, "Conclude {0} resource {1} not exist", type, resource);
       if (!absent) {
         data = null;

xds/src/test/java/io/grpc/xds/CdsLoadBalancer2Test.java

@@ -82,7 +82,7 @@ public class CdsLoadBalancer2Test {
   private static final String EDS_SERVICE_NAME = "backend-service-1.googleapis.com";
   private static final String DNS_HOST_NAME = "backend-service-dns.googleapis.com:443";
   private static final ServerInfo LRS_SERVER_INFO =
-      ServerInfo.create("lrs.googleapis.com", InsecureChannelCredentials.create(), true);
+      ServerInfo.create("lrs.googleapis.com", InsecureChannelCredentials.create(), true, false);
   private final UpstreamTlsContext upstreamTlsContext =
       CommonTlsContextTestsUtil.buildUpstreamTlsContext("google_cloud_private_spiffe", true);
 

xds/src/test/java/io/grpc/xds/ClientXdsClientDataTest.java

@@ -155,7 +155,7 @@
 public class ClientXdsClientDataTest {
 
   private static final ServerInfo LRS_SERVER_INFO =
-      ServerInfo.create("lrs.googleapis.com", InsecureChannelCredentials.create(), true);
+      ServerInfo.create("lrs.googleapis.com", InsecureChannelCredentials.create(), true, false);
 
   @SuppressWarnings("deprecation") // https://github.com/grpc/grpc-java/issues/7467
   @Rule

xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java

@@ -134,7 +134,7 @@ public abstract class ClientXdsClientTestBase {
   private static final Any FAILING_ANY = MessageFactory.FAILING_ANY;
   private static final ChannelCredentials CHANNEL_CREDENTIALS = InsecureChannelCredentials.create();
   private final ServerInfo lrsServerInfo =
-      ServerInfo.create(SERVER_URI, CHANNEL_CREDENTIALS, useProtocolV3());
+      ServerInfo.create(SERVER_URI, CHANNEL_CREDENTIALS, useProtocolV3(), false);
 
   private static final FakeClock.TaskFilter RPC_RETRY_TASK_FILTER =
       new FakeClock.TaskFilter() {
@@ -319,19 +319,21 @@ ManagedChannel create(ServerInfo serverInfo) {
     Bootstrapper.BootstrapInfo bootstrapInfo =
         Bootstrapper.BootstrapInfo.builder()
             .servers(Arrays.asList(
-                Bootstrapper.ServerInfo.create(SERVER_URI, CHANNEL_CREDENTIALS, useProtocolV3())))
+                Bootstrapper.ServerInfo.create(
+                    SERVER_URI, CHANNEL_CREDENTIALS, useProtocolV3(), false)))
             .node(NODE)
             .authorities(ImmutableMap.of(
                 "authority.xds.com",
                 AuthorityInfo.create(
                     "xdstp://authority.xds.com/envoy.config.listener.v3.Listener/%s",
                     ImmutableList.of(Bootstrapper.ServerInfo.create(
-                        SERVER_URI_CUSTOME_AUTHORITY, CHANNEL_CREDENTIALS, useProtocolV3()))),
+                        SERVER_URI_CUSTOME_AUTHORITY, CHANNEL_CREDENTIALS, useProtocolV3(),
+                        false))),
                 "",
                 AuthorityInfo.create(
                     "xdstp:///envoy.config.listener.v3.Listener/%s",
                     ImmutableList.of(Bootstrapper.ServerInfo.create(
-                        SERVER_URI_EMPTY_AUTHORITY, CHANNEL_CREDENTIALS, useProtocolV3())))))
+                        SERVER_URI_EMPTY_AUTHORITY, CHANNEL_CREDENTIALS, useProtocolV3(), false)))))
             .certProviders(ImmutableMap.of("cert-instance-name",
                 CertificateProviderInfo.create("file-watcher", ImmutableMap.<String, Object>of())))
             .build();

xds/src/test/java/io/grpc/xds/ClusterImplLoadBalancerTest.java

@@ -88,7 +88,7 @@ public class ClusterImplLoadBalancerTest {
   private static final String CLUSTER = "cluster-foo.googleapis.com";
   private static final String EDS_SERVICE_NAME = "service.googleapis.com";
   private static final ServerInfo LRS_SERVER_INFO =
-      ServerInfo.create("api.google.com", InsecureChannelCredentials.create(), true);
+      ServerInfo.create("api.google.com", InsecureChannelCredentials.create(), true, false);
   private final SynchronizationContext syncContext = new SynchronizationContext(
       new Thread.UncaughtExceptionHandler() {
         @Override

xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java

@@ -107,7 +107,7 @@ public class ClusterResolverLoadBalancerTest {
   private static final String EDS_SERVICE_NAME2 = "backend-service-bar.googleapis.com";
   private static final String DNS_HOST_NAME = "dns-service.googleapis.com";
   private static final ServerInfo LRS_SERVER_INFO =
-      ServerInfo.create("lrs.googleapis.com", InsecureChannelCredentials.create(), true);
+      ServerInfo.create("lrs.googleapis.com", InsecureChannelCredentials.create(), true, false);
   private final Locality locality1 =
       Locality.create("test-region-1", "test-zone-1", "test-subzone-1");
   private final Locality locality2 =

xds/src/test/java/io/grpc/xds/CsdsServiceTest.java

@@ -77,7 +77,7 @@ public class CsdsServiceTest {
       EnvoyProtoData.Node.newBuilder().setId(NODE_ID).build();
   private static final BootstrapInfo BOOTSTRAP_INFO = BootstrapInfo.builder()
       .servers(ImmutableList.of(
-          ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create(), true)))
+          ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create(), true, false)))
       .node(BOOTSTRAP_NODE)
       .build();
   private static final XdsClient XDS_CLIENT_NO_RESOURCES = new FakeXdsClient();

xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java

@@ -67,7 +67,8 @@ public void noServer() throws XdsInitializationException {
 
   @Test
   public void sharedXdsClientObjectPool() throws XdsInitializationException {
-    ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create(), false);
+    ServerInfo server = ServerInfo.create(
+        SERVER_URI, InsecureChannelCredentials.create(), false, false);
     BootstrapInfo bootstrapInfo =
         BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build();
     when(bootstrapper.bootstrap()).thenReturn(bootstrapInfo);
@@ -84,7 +85,8 @@ public void sharedXdsClientObjectPool() throws XdsInitializationException {
 
   @Test
   public void refCountedXdsClientObjectPool_delayedCreation() {
-    ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create(), false);
+    ServerInfo server = ServerInfo.create(
+        SERVER_URI, InsecureChannelCredentials.create(), false, false);
     BootstrapInfo bootstrapInfo =
         BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build();
     RefCountedXdsClientObjectPool xdsClientPool = new RefCountedXdsClientObjectPool(bootstrapInfo);
@@ -96,7 +98,8 @@ public void refCountedXdsClientObjectPool_delayedCreation() {
 
   @Test
   public void refCountedXdsClientObjectPool_refCounted() {
-    ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create(), false);
+    ServerInfo server = ServerInfo.create(
+        SERVER_URI, InsecureChannelCredentials.create(), false, false);
     BootstrapInfo bootstrapInfo =
         BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build();
     RefCountedXdsClientObjectPool xdsClientPool = new RefCountedXdsClientObjectPool(bootstrapInfo);
@@ -115,7 +118,8 @@ public void refCountedXdsClientObjectPool_refCounted() {
 
   @Test
   public void refCountedXdsClientObjectPool_getObjectCreatesNewInstanceIfAlreadyShutdown() {
-    ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create(), false);
+    ServerInfo server = ServerInfo.create(
+        SERVER_URI, InsecureChannelCredentials.create(), false, false);
     BootstrapInfo bootstrapInfo =
         BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build();
     RefCountedXdsClientObjectPool xdsClientPool = new RefCountedXdsClientObjectPool(bootstrapInfo);

xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java

@@ -143,7 +143,7 @@ public ConfigOrError parseServiceConfig(Map<String, ?> rawServiceConfig) {
   private final TestChannel channel = new TestChannel();
   private BootstrapInfo bootstrapInfo = BootstrapInfo.builder()
       .servers(ImmutableList.of(ServerInfo.create(
-          "td.googleapis.com", InsecureChannelCredentials.create(), true)))
+          "td.googleapis.com", InsecureChannelCredentials.create(), true, false)))
       .node(Node.newBuilder().build())
       .build();
   private String expectedLdsResourceName = AUTHORITY;
@@ -227,7 +227,7 @@ public void resolving_withTargetAuthorityNotFound() {
   public void resolving_noTargetAuthority_templateWithoutXdstp() {
     bootstrapInfo = BootstrapInfo.builder()
         .servers(ImmutableList.of(ServerInfo.create(
-            "td.googleapis.com", InsecureChannelCredentials.create(), true)))
+            "td.googleapis.com", InsecureChannelCredentials.create(), true, false)))
         .node(Node.newBuilder().build())
         .clientDefaultListenerResourceNameTemplate("%s/id=1")
         .build();
@@ -244,7 +244,7 @@ public void resolving_noTargetAuthority_templateWithoutXdstp() {
   public void resolving_noTargetAuthority_templateWithXdstp() {
     bootstrapInfo = BootstrapInfo.builder()
         .servers(ImmutableList.of(ServerInfo.create(
-            "td.googleapis.com", InsecureChannelCredentials.create(), true)))
+            "td.googleapis.com", InsecureChannelCredentials.create(), true, false)))
         .node(Node.newBuilder().build())
         .clientDefaultListenerResourceNameTemplate(
             "xdstp://xds.authority.com/envoy.config.listener.v3.Listener/%s?id=1")
@@ -266,13 +266,13 @@ public void resolving_targetAuthorityInAuthoritiesMap() {
     String serviceAuthority = "[::FFFF:129.144.52.38]:80";
     bootstrapInfo = BootstrapInfo.builder()
         .servers(ImmutableList.of(ServerInfo.create(
-            "td.googleapis.com", InsecureChannelCredentials.create(), true)))
+            "td.googleapis.com", InsecureChannelCredentials.create(), true, false)))
         .node(Node.newBuilder().build())
         .authorities(
             ImmutableMap.of(targetAuthority, AuthorityInfo.create(
                 "xdstp://" + targetAuthority + "/envoy.config.listener.v3.Listener/%s?foo=1&bar=2",
                 ImmutableList.<ServerInfo>of(ServerInfo.create(
-                    "td.googleapis.com", InsecureChannelCredentials.create(), true)))))
+                    "td.googleapis.com", InsecureChannelCredentials.create(), true, false)))))
         .build();
     expectedLdsResourceName = "xdstp://xds.authority.com/envoy.config.listener.v3.Listener/"
         + "%5B::FFFF:129.144.52.38%5D:80?bar=2&foo=1"; // query param canonified

xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java

@@ -54,7 +54,7 @@ public class XdsServerTestHelper {
       Bootstrapper.BootstrapInfo.builder()
           .servers(Arrays.asList(
               Bootstrapper.ServerInfo.create(
-                  SERVER_URI, InsecureChannelCredentials.create(), true)))
+                  SERVER_URI, InsecureChannelCredentials.create(), true, false)))
           .node(BOOTSTRAP_NODE)
           .serverListenerResourceNameTemplate("grpc/server?udpa.resource.listening_address=%s")
           .build();

xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java

@@ -130,7 +130,8 @@ public void testBootstrap_notV3() throws Exception {
     Bootstrapper.BootstrapInfo b =
         Bootstrapper.BootstrapInfo.builder()
             .servers(Arrays.asList(
-                Bootstrapper.ServerInfo.create("uri", InsecureChannelCredentials.create(), false)))
+                Bootstrapper.ServerInfo.create(
+                    "uri", InsecureChannelCredentials.create(), false, false)))
             .node(EnvoyProtoData.Node.newBuilder().setId("id").build())
             .serverListenerResourceNameTemplate("grpc/server?udpa.resource.listening_address=%s")
             .build();
@@ -142,7 +143,8 @@ public void testBootstrap_noTemplate() throws Exception {
     Bootstrapper.BootstrapInfo b =
         Bootstrapper.BootstrapInfo.builder()
             .servers(Arrays.asList(
-                Bootstrapper.ServerInfo.create("uri", InsecureChannelCredentials.create(), true)))
+                Bootstrapper.ServerInfo.create(
+                    "uri", InsecureChannelCredentials.create(), true, false)))
             .node(EnvoyProtoData.Node.newBuilder().setId("id").build())
             .build();
     verifyBootstrapFail(b);
@@ -181,7 +183,7 @@ public void testBootstrap_templateWithXdstp() throws Exception {
     Bootstrapper.BootstrapInfo b = Bootstrapper.BootstrapInfo.builder()
         .servers(Arrays.asList(
             Bootstrapper.ServerInfo.create(
-                "uri", InsecureChannelCredentials.create(), true)))
+                "uri", InsecureChannelCredentials.create(), true, false)))
         .node(EnvoyProtoData.Node.newBuilder().setId("id").build())
         .serverListenerResourceNameTemplate(
             "xdstp://xds.authority.com/envoy.config.listener.v3.Listener/grpc/server/%s")