6750401: SSL stress test with GF leads to 32 bit max process size in less than 5 minutes,with PCKS11 provider
Summary: This is the JSSE portion of the fix. Main part is in PKCS11.
Reviewed-by: valeriep, xuelei
--- a/jdk/src/share/classes/sun/security/ssl/CipherBox.java Wed Dec 17 22:50:37 2008 -0800
+++ b/jdk/src/share/classes/sun/security/ssl/CipherBox.java Fri Dec 19 10:35:56 2008 +0800
@@ -1,5 +1,5 @@
/*
- * Copyright 1996-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1996-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -486,4 +486,21 @@
return newlen;
}
+
+ /*
+ * Dispose of any intermediate state in the underlying cipher.
+ * For PKCS11 ciphers, this will release any attached sessions, and
+ * thus make finalization faster.
+ */
+ void dispose() {
+ try {
+ if (cipher != null) {
+ // ignore return value.
+ cipher.doFinal();
+ }
+ } catch (GeneralSecurityException e) {
+ // swallow for now.
+ }
+ }
+
}
--- a/jdk/src/share/classes/sun/security/ssl/SSLEngineImpl.java Wed Dec 17 22:50:37 2008 -0800
+++ b/jdk/src/share/classes/sun/security/ssl/SSLEngineImpl.java Fri Dec 19 10:35:56 2008 +0800
@@ -1,5 +1,5 @@
/*
- * Copyright 2003-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 2003-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -547,6 +547,8 @@
// ... create decompressor
+ CipherBox oldCipher = readCipher;
+
try {
readCipher = handshaker.newReadCipher();
readMAC = handshaker.newReadMAC();
@@ -555,6 +557,16 @@
throw (SSLException)new SSLException
("Algorithm missing: ").initCause(e);
}
+
+ /*
+ * Dispose of any intermediate state in the underlying cipher.
+ * For PKCS11 ciphers, this will release any attached sessions,
+ * and thus make finalization faster.
+ *
+ * Since MAC's doFinal() is called for every SSL/TLS packet, it's
+ * not necessary to do the same with MAC's.
+ */
+ oldCipher.dispose();
}
/*
@@ -572,6 +584,8 @@
// ... create compressor
+ CipherBox oldCipher = writeCipher;
+
try {
writeCipher = handshaker.newWriteCipher();
writeMAC = handshaker.newWriteMAC();
@@ -580,6 +594,9 @@
throw (SSLException)new SSLException
("Algorithm missing: ").initCause(e);
}
+
+ // See comment above.
+ oldCipher.dispose();
}
/*
@@ -1231,6 +1248,9 @@
break;
}
+ // See comment in changeReadCiphers()
+ writeCipher.dispose();
+
connectionState = cs_CLOSED;
}
@@ -1271,6 +1291,10 @@
closeOutboundInternal();
inboundDone = true;
+
+ // See comment in changeReadCiphers()
+ readCipher.dispose();
+
connectionState = cs_CLOSED;
}
@@ -1457,6 +1481,10 @@
connectionState = cs_CLOSED;
+ // See comment in changeReadCiphers()
+ readCipher.dispose();
+ writeCipher.dispose();
+
if (cause instanceof RuntimeException) {
throw (RuntimeException)cause;
} else {
--- a/jdk/src/share/classes/sun/security/ssl/SSLSocketImpl.java Wed Dec 17 22:50:37 2008 -0800
+++ b/jdk/src/share/classes/sun/security/ssl/SSLSocketImpl.java Fri Dec 19 10:35:56 2008 +0800
@@ -1427,6 +1427,10 @@
waitForClose(false);
}
+ // See comment in changeReadCiphers()
+ readCipher.dispose();
+ writeCipher.dispose();
+
// state will be set to cs_CLOSED in the finally block below
break;
@@ -1633,6 +1637,11 @@
* Clean up our side.
*/
closeSocket();
+
+ // See comment in changeReadCiphers()
+ readCipher.dispose();
+ writeCipher.dispose();
+
connectionState = (oldState == cs_APP_CLOSED) ? cs_APP_CLOSED
: cs_CLOSED;
throw closeReason;
@@ -1763,6 +1772,8 @@
// ... create decompressor
+ CipherBox oldCipher = readCipher;
+
try {
readCipher = handshaker.newReadCipher();
readMAC = handshaker.newReadMAC();
@@ -1771,6 +1782,16 @@
throw (SSLException)new SSLException
("Algorithm missing: ").initCause(e);
}
+
+ /*
+ * Dispose of any intermediate state in the underlying cipher.
+ * For PKCS11 ciphers, this will release any attached sessions,
+ * and thus make finalization faster.
+ *
+ * Since MAC's doFinal() is called for every SSL/TLS packet, it's
+ * not necessary to do the same with MAC's.
+ */
+ oldCipher.dispose();
}
// used by Handshaker
@@ -1783,6 +1804,8 @@
// ... create compressor
+ CipherBox oldCipher = writeCipher;
+
try {
writeCipher = handshaker.newWriteCipher();
writeMAC = handshaker.newWriteMAC();
@@ -1791,6 +1814,9 @@
throw (SSLException)new SSLException
("Algorithm missing: ").initCause(e);
}
+
+ // See comment above.
+ oldCipher.dispose();
}
/*