HDDS-10820. Freon tool DN-Echo to support GRPC and Ratis read/write mode

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java

@@ -424,7 +424,6 @@ public static boolean isReadOnly(
     case ListContainer:
     case ListChunk:
     case GetCommittedBlockLength:
-    case Echo:
       return true;
     case CloseContainer:
     case WriteChunk:
@@ -438,6 +437,9 @@ public static boolean isReadOnly(
     case PutSmallFile:
     case StreamInit:
     case StreamWrite:
+      return false;
+    case Echo:
+      return proto.getEcho().hasReadOnly() && proto.getEcho().getReadOnly();
     default:
       return false;
     }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java

@@ -692,13 +692,15 @@ public static GetSmallFileResponseProto readSmallFile(XceiverClientSpi client,
    * @return EchoResponseProto
    */
   public static EchoResponseProto echo(XceiverClientSpi client, String encodedContainerID,
-      long containerID, ByteString payloadReqBytes, int payloadRespSizeKB, int sleepTimeMs) throws IOException {
+      long containerID, ByteString payloadReqBytes, int payloadRespSizeKB, int sleepTimeMs, boolean readOnly)
+      throws IOException {
     ContainerProtos.EchoRequestProto getEcho =
         EchoRequestProto
             .newBuilder()
             .setPayload(payloadReqBytes)
             .setPayloadSizeResp(payloadRespSizeKB)
             .setSleepTimeMs(sleepTimeMs)
+            .setReadOnly(readOnly)
             .build();
     String id = client.getPipeline().getClosestNode().getUuidString();
 

hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto

@@ -380,6 +380,7 @@ message  EchoRequestProto {
   optional bytes payload = 1;
   optional int32 payloadSizeResp = 2;
   optional int32 sleepTimeMs = 3;
+  optional bool readOnly = 4;
 }
 
 message  EchoResponseProto {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestContainerSmallFile.java

@@ -203,7 +203,7 @@ public void testEcho() throws Exception {
         container.getContainerInfo().getContainerID(), null);
     ByteString byteString = UnsafeByteOperations.unsafeWrap(new byte[0]);
     ContainerProtos.EchoResponseProto response =
-        ContainerProtocolCalls.echo(client, "", container.getContainerInfo().getContainerID(), byteString, 1, 0);
+        ContainerProtocolCalls.echo(client, "", container.getContainerInfo().getContainerID(), byteString, 1, 0, true);
     assertEquals(1, response.getPayload().size());
     xceiverClientManager.releaseClient(client, false);
   }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDNRPCLoadGenerator.java

@@ -32,10 +32,16 @@
 import org.apache.hadoop.ozone.container.common.SCMTestUtils;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 import picocli.CommandLine;
 
 import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Stream;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
@@ -97,15 +103,34 @@ public static void shutdown() {
     shutdownCluster();
   }
 
-  @Test
-  public void test() {
+  private static Stream<Arguments> provideParameters() {
+    return Stream.of(
+        Arguments.of(true, true),
+        Arguments.of(true, false),
+        Arguments.of(false, true),
+        Arguments.of(false, false)
+    );
+  }
+
+  @ParameterizedTest
+  @MethodSource("provideParameters")
+  public void test(boolean readOnly, boolean ratis) {
     DNRPCLoadGenerator randomKeyGenerator =
         new DNRPCLoadGenerator(cluster.getConf());
     CommandLine cmd = new CommandLine(randomKeyGenerator);
-    int exitCode = cmd.execute(
+    List<String> cmdArgs = new ArrayList<>(Arrays.asList(
         "--container-id", Long.toString(container.getContainerInfo().getContainerID()),
         "--clients", "5",
-        "-t", "10");
+        "-t", "10"));
+
+    if (readOnly) {
+      cmdArgs.add("--read-only");
+    }
+    if (ratis) {
+      cmdArgs.add("--ratis");
+    }
+
+    int exitCode = cmd.execute(cmdArgs.toArray(new String[0]));
     assertEquals(0, exitCode);
   }
 }

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DNRPCLoadGenerator.java

@@ -20,6 +20,7 @@
 import com.codahale.metrics.Timer;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
 import org.apache.hadoop.hdds.scm.client.ClientTrustManager;
 import org.apache.hadoop.hdds.security.x509.certificate.client.CACertificateProvider;
 import org.apache.hadoop.hdds.utils.HAUtils;
@@ -36,6 +37,8 @@
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
 import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import picocli.CommandLine;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;
@@ -44,6 +47,8 @@
 import java.util.List;
 import java.util.concurrent.Callable;
 
+import static org.apache.hadoop.hdds.client.ReplicationConfig.getLegacyFactor;
+
 /**
  * Utility to generate RPC request to DN.
  */
@@ -56,7 +61,8 @@
         showDefaultValues = true)
 public class DNRPCLoadGenerator extends BaseFreonGenerator
         implements Callable<Void> {
-
+  private static final Logger LOG =
+      LoggerFactory.getLogger(DNRPCLoadGenerator.class);
   private static final int RPC_PAYLOAD_MULTIPLICATION_FACTOR = 1024;
   private static final int MAX_SIZE_KB = 2097151;
   private Timer timer;
@@ -91,6 +97,16 @@ public class DNRPCLoadGenerator extends BaseFreonGenerator
       defaultValue = "1")
   private int numClients = 1;
 
+  @Option(names = {"--read-only"},
+      description = "if Ratis, read only or not",
+      defaultValue = "false")
+  private boolean readOnly = false;
+
+  @Option(names = {"--ratis"},
+      description = "if Ratis or grpc",
+      defaultValue = "false")
+  private boolean ratis = false;
+
   @CommandLine.ParentCommand
   private Freon freon;
 
@@ -121,6 +137,17 @@ public Void call() throws Exception {
         .filter(p -> p.getId().equals(containerInfo.getPipelineID()))
         .findFirst()
         .orElse(null);
+    // If GRPC, use STANDALONE pipeline
+    if (!ratis) {
+      if (!readOnly) {
+        LOG.warn("Read only is not set to true for GRPC, setting it to true");
+        readOnly = true;
+      }
+      pipeline = Pipeline.newBuilder(pipeline)
+          .setReplicationConfig(StandaloneReplicationConfig.getInstance(
+              getLegacyFactor(pipeline.getReplicationConfig())))
+          .build();
+    }
     encodedContainerToken = scmClient.getEncodedContainerToken(containerID);
     XceiverClientFactory xceiverClientManager;
     if (OzoneSecurityUtil.isSecurityEnabled(configuration)) {
@@ -171,7 +198,7 @@ private void sendRPCReq(long l) throws Exception {
       int clientIndex = (numClients == 1) ? 0 : (int)l % numClients;
       ContainerProtos.EchoResponseProto response =
           ContainerProtocolCalls.echo(clients.get(clientIndex), encodedContainerToken,
-              containerID, payloadReqBytes, payloadRespSize, sleepTimeMs);
+              containerID, payloadReqBytes, payloadRespSize, sleepTimeMs, readOnly);
       return null;
     });
   }