From 8648171abc27497d87a8b47e509e8adabd622f5b Mon Sep 17 00:00:00 2001
From: Norman Maurer <norman_maurer@apple.com>
Date: Sat, 21 Sep 2019 14:58:36 +0200
Subject: [PATCH] Fix *SslEngineTest to not throw ClassCastException and pass
 in all cases (#9588)

Motivation:

Due some bug we did endup with ClassCastExceptions in some cases. Beside this we also did not correctly handle the case when ReferenceCountedOpenSslEngineTest did produce tasks to run in on test.

Modifications:

- Correctly unwrap the engine before to fix ClassCastExceptions
- Run delegated tasks when needed.

Result:

All tests pass with different OpenSSL implementations (OpenSSL, BoringSSL etc)
---
 .../netty/handler/ssl/OpenSslEngineTest.java  | 33 ++++++++++++++++---
 .../ReferenceCountedOpenSslEngineTest.java    |  4 ++-
 2 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java b/handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java
index a1ac8103441..bcdc5d06150 100644
--- a/handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java
+++ b/handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java
@@ -36,6 +36,7 @@
 import javax.crypto.spec.SecretKeySpec;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLEngineResult;
+import javax.net.ssl.SSLEngineResult.HandshakeStatus;
 import javax.net.ssl.SSLException;
 import javax.net.ssl.SSLParameters;
 import java.nio.ByteBuffer;
@@ -61,6 +62,7 @@
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
@@ -1131,6 +1133,19 @@ public boolean permits(
         }
     }
 
+    private static void runTasksIfNeeded(SSLEngine engine) {
+        if (engine.getHandshakeStatus() == HandshakeStatus.NEED_TASK) {
+            for (;;) {
+                Runnable task = engine.getDelegatedTask();
+                if (task == null) {
+                    assertNotEquals(HandshakeStatus.NEED_TASK, engine.getHandshakeStatus());
+                    break;
+                }
+                task.run();
+            }
+        }
+    }
+
     @Test
     public void testExtractMasterkeyWorksCorrectly() throws Exception {
         SelfSignedCertificate cert = new SelfSignedCertificate();
@@ -1199,17 +1214,23 @@ public void testExtractMasterkeyWorksCorrectly() throws Exception {
                 cTOs.flip();
                 sTOc.flip();
 
+                runTasksIfNeeded(clientEngine);
+                runTasksIfNeeded(serverEngine);
+
                 clientEngine.unwrap(sTOc, clientIn);
                 serverEngine.unwrap(cTOs, serverIn);
 
+                runTasksIfNeeded(clientEngine);
+                runTasksIfNeeded(serverEngine);
+
                 // check when the application data has fully been consumed and sent
                 // for both the client and server
                 if ((clientOut.limit() == serverIn.position()) &&
                         (serverOut.limit() == clientIn.position())) {
-                    byte[] serverRandom = SSL.getServerRandom(((OpenSslEngine) serverEngine).sslPointer());
-                    byte[] clientRandom = SSL.getClientRandom(((OpenSslEngine) clientEngine).sslPointer());
-                    byte[] serverMasterKey = SSL.getMasterKey(((OpenSslEngine) serverEngine).sslPointer());
-                    byte[] clientMasterKey = SSL.getMasterKey(((OpenSslEngine) clientEngine).sslPointer());
+                    byte[] serverRandom = SSL.getServerRandom(unwrapEngine(serverEngine).sslPointer());
+                    byte[] clientRandom = SSL.getClientRandom(unwrapEngine(clientEngine).sslPointer());
+                    byte[] serverMasterKey = SSL.getMasterKey(unwrapEngine(serverEngine).sslPointer());
+                    byte[] clientMasterKey = SSL.getMasterKey(unwrapEngine(clientEngine).sslPointer());
 
                     asserted = true;
                     assertArrayEquals(serverMasterKey, clientMasterKey);
@@ -1318,7 +1339,9 @@ ReferenceCountedOpenSslEngine unwrapEngine(SSLEngine engine) {
 
     @Override
     protected SslContext wrapContext(SslContext context) {
-        ((OpenSslContext) context).setUseTasks(useTasks);
+        if (context instanceof OpenSslContext) {
+            ((OpenSslContext) context).setUseTasks(useTasks);
+        }
         return context;
     }
 }
diff --git a/handler/src/test/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngineTest.java b/handler/src/test/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngineTest.java
index a3effd88f5a..9137dc3d984 100644
--- a/handler/src/test/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngineTest.java
+++ b/handler/src/test/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngineTest.java
@@ -72,7 +72,9 @@ public void testNotLeakOnException() throws Exception {
 
     @Override
     protected SslContext wrapContext(SslContext context) {
-        ((ReferenceCountedOpenSslContext) context).setUseTasks(useTasks);
+        if (context instanceof ReferenceCountedOpenSslContext) {
+            ((ReferenceCountedOpenSslContext) context).setUseTasks(useTasks);
+        }
         return context;
     }
 }