8074580: sun/security/pkcs11/rsa/TestKeyPairGenerator.java fails due to PKCS11Exception: CKR_FUNCTION_FAILED
authorasmotrak
Wed, 22 Jun 2016 15:58:08 -0700
changeset 39142 bf48a9f13cf2
parent 39141 e5d52546ba3a
child 39143 ba4958d56eec
8074580: sun/security/pkcs11/rsa/TestKeyPairGenerator.java fails due to PKCS11Exception: CKR_FUNCTION_FAILED Reviewed-by: valeriep
jdk/src/jdk.crypto.pkcs11/share/classes/sun/security/pkcs11/wrapper/PKCS11.java
jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_general.c
jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_keymgmt.c
jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_util.c
jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/pkcs11wrapper.h
jdk/test/ProblemList.txt
jdk/test/sun/security/pkcs11/rsa/TestKeyPairGenerator.java
--- a/jdk/src/jdk.crypto.pkcs11/share/classes/sun/security/pkcs11/wrapper/PKCS11.java	Wed Jun 22 11:31:07 2016 -0700
+++ b/jdk/src/jdk.crypto.pkcs11/share/classes/sun/security/pkcs11/wrapper/PKCS11.java	Wed Jun 22 15:58:08 2016 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
  */
 
 /* Copyright  (c) 2002 Graz University of Technology. All rights reserved.
@@ -54,6 +54,8 @@
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 
+import sun.security.util.Debug;
+
 import static sun.security.pkcs11.wrapper.PKCS11Constants.*;
 
 /**
@@ -85,7 +87,8 @@
                 return null;
             }
         });
-        initializeLibrary();
+        boolean enableDebug = Debug.getInstance("sunpkcs11") != null;
+        initializeLibrary(enableDebug);
     }
 
     public static void loadNative() {
@@ -109,7 +112,7 @@
      * @preconditions
      * @postconditions
      */
-    private static native void initializeLibrary();
+    private static native void initializeLibrary(boolean debug);
 
     // XXX
     /**
--- a/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_general.c	Wed Jun 22 11:31:07 2016 -0700
+++ b/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_general.c	Wed Jun 22 15:58:08 2016 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
  */
 
 /* Copyright  (c) 2002 Graz University of Technology. All rights reserved.
@@ -73,6 +73,8 @@
 
 JavaVM* jvm = NULL;
 
+jboolean debug = 0;
+
 JNIEXPORT jint JNICALL DEF_JNI_OnLoad(JavaVM *vm, void *reserved) {
     jvm = vm;
     return JNI_VERSION_1_4;
@@ -92,7 +94,7 @@
  */
 JNIEXPORT void JNICALL
 Java_sun_security_pkcs11_wrapper_PKCS11_initializeLibrary
-(JNIEnv *env, jclass thisClass)
+(JNIEnv *env, jclass thisClass, jboolean enableDebug)
 {
 #ifndef NO_CALLBACKS
     if (notifyListLock == NULL) {
@@ -101,6 +103,7 @@
 #endif
 
     prefetchFields(env, thisClass);
+    debug = enableDebug;
 }
 
 jclass fetchClass(JNIEnv *env, const char *name) {
--- a/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_keymgmt.c	Wed Jun 22 11:31:07 2016 -0700
+++ b/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_keymgmt.c	Wed Jun 22 15:58:08 2016 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
  */
 
 /* Copyright  (c) 2002 Graz University of Technology. All rights reserved.
@@ -152,6 +152,8 @@
     CK_OBJECT_HANDLE_PTR ckpKeyHandles;     /* pointer to array with Public and Private Key */
     jlongArray jKeyHandles = NULL;
     CK_RV rv;
+    int attempts;
+    const int MAX_ATTEMPTS = 3;
 
     CK_FUNCTION_LIST_PTR ckpFunctions = getFunctionList(env, obj);
     if (ckpFunctions == NULL) { return NULL; }
@@ -190,10 +192,35 @@
         return NULL;
     }
 
-    rv = (*ckpFunctions->C_GenerateKeyPair)(ckSessionHandle, &ckMechanism,
-                     ckpPublicKeyAttributes, ckPublicKeyAttributesLength,
-                     ckpPrivateKeyAttributes, ckPrivateKeyAttributesLength,
-                     ckpPublicKeyHandle, ckpPrivateKeyHandle);
+    /*
+     * Workaround for NSS bug 1012786:
+     *
+     * Key generation may fail with CKR_FUNCTION_FAILED error
+     * if there is insufficient entropy to generate a random key.
+     *
+     * PKCS11 spec says the following about CKR_FUNCTION_FAILED error
+     * (see section 11.1.1):
+     *
+     *      ... In any event, although the function call failed, the situation
+     *      is not necessarily totally hopeless, as it is likely to be
+     *      when CKR_GENERAL_ERROR is returned. Depending on what the root cause of
+     *      the error actually was, it is possible that an attempt
+     *      to make the exact same function call again would succeed.
+     *
+     * Call C_GenerateKeyPair() several times if CKR_FUNCTION_FAILED occurs.
+     */
+    for (attempts = 0; attempts < MAX_ATTEMPTS; attempts++) {
+        rv = (*ckpFunctions->C_GenerateKeyPair)(ckSessionHandle, &ckMechanism,
+                        ckpPublicKeyAttributes, ckPublicKeyAttributesLength,
+                        ckpPrivateKeyAttributes, ckPrivateKeyAttributesLength,
+                        ckpPublicKeyHandle, ckpPrivateKeyHandle);
+        if (rv == CKR_FUNCTION_FAILED) {
+            printDebug("C_1GenerateKeyPair(): C_GenerateKeyPair() failed \
+                    with CKR_FUNCTION_FAILED error, try again\n");
+        } else {
+            break;
+        }
+    }
 
     if (ckAssertReturnValueOK(env, rv) == CK_ASSERT_OK) {
         jKeyHandles = ckULongArrayToJLongArray(env, ckpKeyHandles, 2);
--- a/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_util.c	Wed Jun 22 11:31:07 2016 -0700
+++ b/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_util.c	Wed Jun 22 15:58:08 2016 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
  */
 
 /* Copyright  (c) 2002 Graz University of Technology. All rights reserved.
@@ -1143,3 +1143,15 @@
 
 #endif
 
+// prints a message to stdout if debug output is enabled
+void printDebug(const char *format, ...) {
+    if (debug == JNI_TRUE) {
+        va_list args;
+        fprintf(stdout, "sunpkcs11: ");
+        va_start(args, format);
+        vfprintf(stdout, format, args);
+        va_end(args);
+        fflush(stdout);
+    }
+}
+
--- a/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/pkcs11wrapper.h	Wed Jun 22 11:31:07 2016 -0700
+++ b/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/pkcs11wrapper.h	Wed Jun 22 15:58:08 2016 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
  */
 
 /* Copyright  (c) 2002 Graz University of Technology. All rights reserved.
@@ -159,6 +159,7 @@
 #include "pkcs-11v2-20a3.h"
 #include <jni.h>
 #include <jni_util.h>
+#include <stdarg.h>
 
 #define MAX_STACK_BUFFER_LEN (4 * 1024)
 #define MAX_HEAP_BUFFER_LEN (64 * 1024)
@@ -217,6 +218,10 @@
 #define TRACE_UNINTEND
 #endif
 
+/* debug output */
+extern jboolean debug;
+void printDebug(const char *format, ...);
+
 #define CK_ASSERT_OK 0L
 
 #define CLASS_INFO "sun/security/pkcs11/wrapper/CK_INFO"
--- a/jdk/test/ProblemList.txt	Wed Jun 22 11:31:07 2016 -0700
+++ b/jdk/test/ProblemList.txt	Wed Jun 22 15:58:08 2016 -0700
@@ -272,8 +272,6 @@
 sun/security/pkcs11/tls/TestPRF.java                            8077138,8023434 windows-all
 sun/security/pkcs11/tls/TestPremaster.java                      8077138,8023434 windows-all
 
-sun/security/pkcs11/rsa/TestKeyPairGenerator.java               8074580 generic-all
-
 sun/security/krb5/auto/HttpNegotiateServer.java                 8038079 generic-all
 
 sun/security/tools/keytool/autotest.sh                          8130302 generic-all
--- a/jdk/test/sun/security/pkcs11/rsa/TestKeyPairGenerator.java	Wed Jun 22 11:31:07 2016 -0700
+++ b/jdk/test/sun/security/pkcs11/rsa/TestKeyPairGenerator.java	Wed Jun 22 15:58:08 2016 -0700
@@ -29,8 +29,9 @@
  * @library ..
  * @library /lib/testlibrary
  * @build jdk.testlibrary.*
- * @run main/othervm TestKeyPairGenerator
- * @run main/othervm TestKeyPairGenerator sm TestKeyPairGenerator.policy
+ * @run main/othervm -Djava.security.debug=sunpkcs11 TestKeyPairGenerator
+ * @run main/othervm -Djava.security.debug=sunpkcs11 TestKeyPairGenerator
+ *                                                   sm TestKeyPairGenerator.policy
  * @key intermittent randomness
  */