8214418: half-closed SSLEngine status may cause application dead loop
Reviewed-by: jnimeh, dfuchs, chegar
--- a/src/java.base/share/classes/sun/security/ssl/Ciphertext.java Fri Jan 11 10:03:00 2019 +0100
+++ b/src/java.base/share/classes/sun/security/ssl/Ciphertext.java Mon Jan 14 10:00:45 2019 -0800
@@ -31,8 +31,6 @@
* Ciphertext
*/
final class Ciphertext {
- static final Ciphertext CIPHERTEXT_NULL = new Ciphertext();
-
final byte contentType;
final byte handshakeType;
final long recordSN;
--- a/src/java.base/share/classes/sun/security/ssl/SSLEngineImpl.java Fri Jan 11 10:03:00 2019 +0100
+++ b/src/java.base/share/classes/sun/security/ssl/SSLEngineImpl.java Mon Jan 14 10:00:45 2019 -0800
@@ -247,6 +247,19 @@
hsStatus = ciphertext.handshakeStatus;
} else {
hsStatus = getHandshakeStatus();
+ if (ciphertext == null && !conContext.isNegotiated &&
+ conContext.isInboundClosed() &&
+ hsStatus == HandshakeStatus.NEED_WRAP) {
+ // Even the outboud is open, no futher data could be wrapped as:
+ // 1. the outbound is empty
+ // 2. no negotiated connection
+ // 3. the inbound has closed, cannot complete the handshake
+ //
+ // Mark the engine as closed if the handshake status is
+ // NEED_WRAP. Otherwise, it could lead to dead loops in
+ // applications.
+ status = Status.CLOSED;
+ }
}
int deltaSrcs = srcsRemains;
@@ -279,7 +292,7 @@
}
if (ciphertext == null) {
- return Ciphertext.CIPHERTEXT_NULL;
+ return null;
}
// Is the handshake completed?
--- a/src/java.base/share/classes/sun/security/ssl/TransportContext.java Fri Jan 11 10:03:00 2019 +0100
+++ b/src/java.base/share/classes/sun/security/ssl/TransportContext.java Mon Jan 14 10:00:45 2019 -0800
@@ -577,13 +577,7 @@
} else if (!isOutboundClosed()) {
// Special case that the inbound was closed, but outbound open.
return HandshakeStatus.NEED_WRAP;
- }
- } else if (isOutboundClosed() && !isInboundClosed()) {
- // Special case that the outbound was closed, but inbound open.
- return HandshakeStatus.NEED_UNWRAP;
- } else if (!isOutboundClosed() && isInboundClosed()) {
- // Special case that the inbound was closed, but outbound open.
- return HandshakeStatus.NEED_WRAP;
+ } // Otherwise, both inbound and outbound are closed.
}
return HandshakeStatus.NOT_HANDSHAKING;