8214418: half-closed SSLEngine status may cause application dead loop
authorxuelei
Mon, 14 Jan 2019 10:00:45 -0800
changeset 53289 5022a4915fe9
parent 53288 f0490430ef7a
child 53290 b685bc048276
8214418: half-closed SSLEngine status may cause application dead loop Reviewed-by: jnimeh, dfuchs, chegar
src/java.base/share/classes/sun/security/ssl/Ciphertext.java
src/java.base/share/classes/sun/security/ssl/SSLEngineImpl.java
src/java.base/share/classes/sun/security/ssl/TransportContext.java
--- a/src/java.base/share/classes/sun/security/ssl/Ciphertext.java	Fri Jan 11 10:03:00 2019 +0100
+++ b/src/java.base/share/classes/sun/security/ssl/Ciphertext.java	Mon Jan 14 10:00:45 2019 -0800
@@ -31,8 +31,6 @@
  * Ciphertext
  */
 final class Ciphertext {
-    static final Ciphertext CIPHERTEXT_NULL = new Ciphertext();
-
     final byte contentType;
     final byte handshakeType;
     final long recordSN;
--- a/src/java.base/share/classes/sun/security/ssl/SSLEngineImpl.java	Fri Jan 11 10:03:00 2019 +0100
+++ b/src/java.base/share/classes/sun/security/ssl/SSLEngineImpl.java	Mon Jan 14 10:00:45 2019 -0800
@@ -247,6 +247,19 @@
             hsStatus = ciphertext.handshakeStatus;
         } else {
             hsStatus = getHandshakeStatus();
+            if (ciphertext == null && !conContext.isNegotiated &&
+                    conContext.isInboundClosed() &&
+                    hsStatus == HandshakeStatus.NEED_WRAP) {
+                // Even the outboud is open, no futher data could be wrapped as:
+                //     1. the outbound is empty
+                //     2. no negotiated connection
+                //     3. the inbound has closed, cannot complete the handshake
+                //
+                // Mark the engine as closed if the handshake status is
+                // NEED_WRAP. Otherwise, it could lead to dead loops in
+                // applications.
+                status = Status.CLOSED;
+            }
         }
 
         int deltaSrcs = srcsRemains;
@@ -279,7 +292,7 @@
         }
 
         if (ciphertext == null) {
-            return Ciphertext.CIPHERTEXT_NULL;
+            return null;
         }
 
         // Is the handshake completed?
--- a/src/java.base/share/classes/sun/security/ssl/TransportContext.java	Fri Jan 11 10:03:00 2019 +0100
+++ b/src/java.base/share/classes/sun/security/ssl/TransportContext.java	Mon Jan 14 10:00:45 2019 -0800
@@ -577,13 +577,7 @@
             } else if (!isOutboundClosed()) {
                 // Special case that the inbound was closed, but outbound open.
                 return HandshakeStatus.NEED_WRAP;
-            }
-        } else if (isOutboundClosed() && !isInboundClosed()) {
-            // Special case that the outbound was closed, but inbound open.
-            return HandshakeStatus.NEED_UNWRAP;
-        } else if (!isOutboundClosed() && isInboundClosed()) {
-            // Special case that the inbound was closed, but outbound open.
-            return HandshakeStatus.NEED_WRAP;
+            }   // Otherwise, both inbound and outbound are closed.
         }
 
         return HandshakeStatus.NOT_HANDSHAKING;