8130875: Ucrypto library leaks memory when null output buffer is specified
authorvaleriep
Fri, 04 Sep 2015 19:55:40 +0000
changeset 32475 d4e63a5e73c8
parent 32474 5b41b53a4997
child 32476 568d6c299cad
8130875: Ucrypto library leaks memory when null output buffer is specified Summary: Avoid null output buffer to work around Solaris memory leak bug in Ucrypto library Reviewed-by: ascarpino
jdk/src/jdk.crypto.ucrypto/solaris/native/libj2ucrypto/nativeCrypto.c
--- a/jdk/src/jdk.crypto.ucrypto/solaris/native/libj2ucrypto/nativeCrypto.c	Fri Sep 04 12:18:03 2015 +0200
+++ b/jdk/src/jdk.crypto.ucrypto/solaris/native/libj2ucrypto/nativeCrypto.c	Fri Sep 04 19:55:40 2015 +0000
@@ -435,6 +435,11 @@
   int rv = 0;
 
   context = (crypto_ctx_t *) pContext;
+  // Avoid null output buffer to workaround Solaris bug21481818 (fixed in S12)
+  if (bufOut == NULL) {
+    bufOut = (unsigned char *)(&outLen);
+    outLen = 0;
+  }
   rv = CipherFinal(context, encrypt, (unsigned char*)bufOut, outOfs, &outLen);
   free(context);
   if (rv) {
@@ -648,7 +653,8 @@
 
   // out is null when nativeFinal() is called solely for resource clean up
   if (out == NULL) {
-    bufOut = NULL;
+    // Avoid null output buffer to workaround Solaris bug21481818 (fixed in S12)
+    bufOut = (unsigned char *)(&outLen);
     outLen = 0;
   } else {
     outLen = (*env)->GetArrayLength(env, out) - outOfs;
@@ -661,10 +667,12 @@
   rv = CipherFinal(context, encrypt, bufOut, 0, &outLen);
   if (rv) {
     free(context);
-    free(bufOut);
+    if (outLen != 0) {
+      free(bufOut);
+    }
     return -rv;
   } else {
-    if (bufOut != NULL) {
+    if (bufOut != NULL && outLen != 0) {
       (*env)->SetByteArrayRegion(env, out, outOfs, outLen, (jbyte *)bufOut);
       free(bufOut);
     }