WebSocket frame decoding: replace byte array[4] frame mask with int. (#…

…12989) (#13010)

Motivation:

Make implementation consistent with recent changes on WebSocket08FrameEncoder #12969.

Modifications:

WebSocket08FrameDecoder: replace frame mask byte array[4] with int.

Result:

Simpler implementation consistent with WebSocket08FrameEncoder.

Co-authored-by: Maksym Ostroverkhov <m.ostroverkhov@gmail.com>

codec-http/src/main/java/io/netty5/handler/codec/http/websocketx/WebSocket13FrameDecoder.java

@@ -82,7 +82,7 @@ public class WebSocket13FrameDecoder extends ByteToMessageDecoder implements Web
     private int frameRsv;
     private int frameOpcode;
     private long framePayloadLength;
-    private byte[] maskingKey;
+    private int mask;
     private int framePayloadLen1;
     private boolean receivedClosingHandshake;
     private State state = State.READING_FIRST;
@@ -289,10 +289,7 @@ protected void decode(ChannelHandlerContext ctx, Buffer in) throws Exception {
                 if (in.readableBytes() < 4) {
                     return;
                 }
-                if (maskingKey == null) {
-                    maskingKey = new byte[4];
-                }
-                in.readBytes(maskingKey, 0, maskingKey.length);
+                mask = in.readInt();
             }
             state = State.PAYLOAD;
         }
@@ -392,20 +389,15 @@ private void unmask(Buffer frame) {
         int len = frame.readableBytes();
         int index = 0;
 
-        // Remark: & 0xFF is necessary because Java will do signed expansion from
-        // byte to int which we don't want.
-        int intMask = (maskingKey[0] & 0xFF) << 24
-                      | (maskingKey[1] & 0xFF) << 16
-                      | (maskingKey[2] & 0xFF) << 8
-                      | maskingKey[3] & 0xFF;
-
+        int intMask = mask;
         for (; index + 3 < len; index += Integer.BYTES) {
             int off = base + index;
             frame.setInt(off, frame.getInt(off) ^ intMask);
         }
+        int maskOffset = 0;
         for (; index < len; index++) {
             int off = base + index;
-            frame.setByte(off, (byte) (frame.getByte(off) ^ maskingKey[index % 4]));
+            frame.setByte(off, (byte) (frame.getByte(off) ^ WebSocketUtil.byteAtIndex(intMask, maskOffset++ & 3)));
         }
     }
 

codec-http/src/main/java/io/netty5/handler/codec/http/websocketx/WebSocket13FrameEncoder.java

@@ -174,14 +174,13 @@ protected void encode(ChannelHandlerContext ctx, WebSocketFrame msg, List<Object
                 int mask = ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE);
                 buf.writeInt(mask);
 
-                int counter = 0;
                 int i = data.readerOffset();
                 int end = data.writerOffset();
 
                 int maskOffset = 0;
                 for (; i < end; i++) {
                     byte byteData = data.getByte(i);
-                    buf.writeByte((byte) (byteData ^ byteAtIndex(mask, maskOffset++ & 3)));
+                    buf.writeByte((byte) (byteData ^ WebSocketUtil.byteAtIndex(mask, maskOffset++ & 3)));
                 }
                 out.add(buf);
             } else {
@@ -201,8 +200,4 @@ protected void encode(ChannelHandlerContext ctx, WebSocketFrame msg, List<Object
             throw t;
         }
     }
-
-    private static int byteAtIndex(int mask, int index) {
-        return (mask >> 8 * (3 - index)) & 0xFF;
-    }
 }

codec-http/src/main/java/io/netty5/handler/codec/http/websocketx/WebSocketUtil.java

@@ -86,6 +86,10 @@ static String calculateV13Accept(String nonce) {
         return base64(sha1);
     }
 
+    static int byteAtIndex(int mask, int index) {
+        return (mask >> 8 * (3 - index)) & 0xFF;
+    }
+
     private WebSocketUtil() {
     }
 }