Ensure CLOSE_FRAME can be produced when forceClose() is called (#360)

Motivation:

Due a regression introduced in d7b6e36 we did not send the CLOSE_FRAME anymore in some cases

Modifications:

- Only set the connection to null after we are done with sending the CLOSE_FRAME
- Combine reentrance state into one field and use bitwise ops to reduce memory overhead

Result:

Always produce the CLOSE_FRAME

Author: normanmaurer

codec-classes-quic/src/main/java/io/netty/incubator/codec/quic/QuicheQuicChannel.java

@@ -131,9 +131,12 @@ public void operationComplete(ChannelFuture future) {
 
     private boolean recvStreamPending;
     private boolean streamReadable;
-    private boolean inRecv;
-    private boolean inConnectionSend;
-    private boolean inHandleWritableStreams;
+
+    private int reantranceGuard = 0;
+    private static final int IN_RECV = 1 << 1;
+    private static final int IN_CONNECTION_SEND = 1 << 2;
+    private static final int IN_HANDLE_WRITABLE_STREAMS = 1 << 3;
+    private static final int IN_FORCE_CLOSE = 1 << 3;
 
     private static final int CLOSED = 0;
     private static final int OPEN = 1;
@@ -346,13 +349,13 @@ boolean markInFireChannelReadCompleteQueue() {
     }
 
     void forceClose() {
-        if (isConnDestroyed()) {
+        if (isConnDestroyed() || (reantranceGuard & IN_FORCE_CLOSE) != 0) {
             // Just return if we already destroyed the underlying connection.
             return;
         }
-        // We need to set the connection to null now to guard against reentrancy.
+        reantranceGuard |= IN_FORCE_CLOSE;
+
         QuicheQuicConnection conn = connection;
-        connection = null;
 
         unsafe().close(voidPromise());
         // making sure that connection statistics is avaliable
@@ -378,6 +381,7 @@ void forceClose() {
             timeoutHandler.cancel();
         } finally {
             flushParent();
+            connection = null;
             conn.free();
         }
     }
@@ -769,7 +773,7 @@ int streamSend(long streamId, ByteBuf buffer, boolean fin) throws ClosedChannelE
     }
 
     void connectionSendAndFlush() {
-        if (inFireChannelReadCompleteQueue || inHandleWritableStreams) {
+        if (inFireChannelReadCompleteQueue || (reantranceGuard & IN_HANDLE_WRITABLE_STREAMS) != 0) {
             return;
         }
         if (connectionSend()) {
@@ -832,7 +836,7 @@ private boolean handleWritableStreams() {
         if (isConnDestroyed()) {
             return false;
         }
-        inHandleWritableStreams = true;
+        reantranceGuard |= IN_HANDLE_WRITABLE_STREAMS;
         try {
             long connAddr = connection.address();
             boolean mayNeedWrite = false;
@@ -880,7 +884,7 @@ private boolean handleWritableStreams() {
             }
             return mayNeedWrite;
         } finally {
-            inHandleWritableStreams = false;
+            reantranceGuard &= ~IN_HANDLE_WRITABLE_STREAMS;
         }
     }
 
@@ -1072,11 +1076,11 @@ private boolean connectionSendSimple() {
      * {@link Channel#flush()} at some point.
      */
     private boolean connectionSend() {
-        if (isConnDestroyed() || inConnectionSend) {
+        if (isConnDestroyed() || (reantranceGuard & IN_CONNECTION_SEND) != 0) {
             return false;
         }
 
-        inConnectionSend = true;
+        reantranceGuard |= IN_CONNECTION_SEND;
         try {
             boolean packetWasWritten;
             SegmentedDatagramPacketAllocator segmentedDatagramPacketAllocator =
@@ -1091,7 +1095,7 @@ private boolean connectionSend() {
             }
             return packetWasWritten;
         } finally {
-            inConnectionSend = false;
+            reantranceGuard &= ~IN_CONNECTION_SEND;
         }
     }
 
@@ -1189,7 +1193,9 @@ void connectionRecv(InetSocketAddress sender, ByteBuf buffer) {
                 // See also https://github.com/cloudflare/quiche/issues/817
                 return;
             }
-            inRecv = true;
+
+            reantranceGuard |= IN_RECV;
+
             try {
                 ByteBuf tmpBuffer = null;
                 // We need to make a copy if the buffer is read only as recv(...) may modify the input buffer as well.
@@ -1280,12 +1286,12 @@ void connectionRecv(InetSocketAddress sender, ByteBuf buffer) {
                     }
                 }
             } finally {
-                inRecv = false;
+                reantranceGuard &= ~IN_RECV;
             }
         }
 
         void recv() {
-            if (inRecv || isConnDestroyed()) {
+            if ((reantranceGuard & IN_RECV) != 0 || isConnDestroyed()) {
                 return;
             }
 
@@ -1296,13 +1302,13 @@ void recv() {
                 return;
             }
 
-            inRecv = true;
+            reantranceGuard |= IN_RECV;
             try {
                 recvDatagram();
                 recvStream();
             } finally {
                 fireChannelReadCompleteIfNeeded();
-                inRecv = false;
+                reantranceGuard &= ~IN_RECV;
             }
         }