# HG changeset patch
# User asmotrak
# Date 1466636288 25200
# Node ID bf48a9f13cf21d519f75827c1b1bf13b78930ad8
# Parent e5d52546ba3ace8f4027a064d6751f1b286d72bf
8074580: sun/security/pkcs11/rsa/TestKeyPairGenerator.java fails due to PKCS11Exception: CKR_FUNCTION_FAILED
Reviewed-by: valeriep
diff -r e5d52546ba3a -r bf48a9f13cf2 jdk/src/jdk.crypto.pkcs11/share/classes/sun/security/pkcs11/wrapper/PKCS11.java
--- a/jdk/src/jdk.crypto.pkcs11/share/classes/sun/security/pkcs11/wrapper/PKCS11.java Wed Jun 22 11:31:07 2016 -0700
+++ b/jdk/src/jdk.crypto.pkcs11/share/classes/sun/security/pkcs11/wrapper/PKCS11.java Wed Jun 22 15:58:08 2016 -0700
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
*/
/* Copyright (c) 2002 Graz University of Technology. All rights reserved.
@@ -54,6 +54,8 @@
import java.security.AccessController;
import java.security.PrivilegedAction;
+import sun.security.util.Debug;
+
import static sun.security.pkcs11.wrapper.PKCS11Constants.*;
/**
@@ -85,7 +87,8 @@
return null;
}
});
- initializeLibrary();
+ boolean enableDebug = Debug.getInstance("sunpkcs11") != null;
+ initializeLibrary(enableDebug);
}
public static void loadNative() {
@@ -109,7 +112,7 @@
* @preconditions
* @postconditions
*/
- private static native void initializeLibrary();
+ private static native void initializeLibrary(boolean debug);
// XXX
/**
diff -r e5d52546ba3a -r bf48a9f13cf2 jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_general.c
--- a/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_general.c Wed Jun 22 11:31:07 2016 -0700
+++ b/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_general.c Wed Jun 22 15:58:08 2016 -0700
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
*/
/* Copyright (c) 2002 Graz University of Technology. All rights reserved.
@@ -73,6 +73,8 @@
JavaVM* jvm = NULL;
+jboolean debug = 0;
+
JNIEXPORT jint JNICALL DEF_JNI_OnLoad(JavaVM *vm, void *reserved) {
jvm = vm;
return JNI_VERSION_1_4;
@@ -92,7 +94,7 @@
*/
JNIEXPORT void JNICALL
Java_sun_security_pkcs11_wrapper_PKCS11_initializeLibrary
-(JNIEnv *env, jclass thisClass)
+(JNIEnv *env, jclass thisClass, jboolean enableDebug)
{
#ifndef NO_CALLBACKS
if (notifyListLock == NULL) {
@@ -101,6 +103,7 @@
#endif
prefetchFields(env, thisClass);
+ debug = enableDebug;
}
jclass fetchClass(JNIEnv *env, const char *name) {
diff -r e5d52546ba3a -r bf48a9f13cf2 jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_keymgmt.c
--- a/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_keymgmt.c Wed Jun 22 11:31:07 2016 -0700
+++ b/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_keymgmt.c Wed Jun 22 15:58:08 2016 -0700
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
*/
/* Copyright (c) 2002 Graz University of Technology. All rights reserved.
@@ -152,6 +152,8 @@
CK_OBJECT_HANDLE_PTR ckpKeyHandles; /* pointer to array with Public and Private Key */
jlongArray jKeyHandles = NULL;
CK_RV rv;
+ int attempts;
+ const int MAX_ATTEMPTS = 3;
CK_FUNCTION_LIST_PTR ckpFunctions = getFunctionList(env, obj);
if (ckpFunctions == NULL) { return NULL; }
@@ -190,10 +192,35 @@
return NULL;
}
- rv = (*ckpFunctions->C_GenerateKeyPair)(ckSessionHandle, &ckMechanism,
- ckpPublicKeyAttributes, ckPublicKeyAttributesLength,
- ckpPrivateKeyAttributes, ckPrivateKeyAttributesLength,
- ckpPublicKeyHandle, ckpPrivateKeyHandle);
+ /*
+ * Workaround for NSS bug 1012786:
+ *
+ * Key generation may fail with CKR_FUNCTION_FAILED error
+ * if there is insufficient entropy to generate a random key.
+ *
+ * PKCS11 spec says the following about CKR_FUNCTION_FAILED error
+ * (see section 11.1.1):
+ *
+ * ... In any event, although the function call failed, the situation
+ * is not necessarily totally hopeless, as it is likely to be
+ * when CKR_GENERAL_ERROR is returned. Depending on what the root cause of
+ * the error actually was, it is possible that an attempt
+ * to make the exact same function call again would succeed.
+ *
+ * Call C_GenerateKeyPair() several times if CKR_FUNCTION_FAILED occurs.
+ */
+ for (attempts = 0; attempts < MAX_ATTEMPTS; attempts++) {
+ rv = (*ckpFunctions->C_GenerateKeyPair)(ckSessionHandle, &ckMechanism,
+ ckpPublicKeyAttributes, ckPublicKeyAttributesLength,
+ ckpPrivateKeyAttributes, ckPrivateKeyAttributesLength,
+ ckpPublicKeyHandle, ckpPrivateKeyHandle);
+ if (rv == CKR_FUNCTION_FAILED) {
+ printDebug("C_1GenerateKeyPair(): C_GenerateKeyPair() failed \
+ with CKR_FUNCTION_FAILED error, try again\n");
+ } else {
+ break;
+ }
+ }
if (ckAssertReturnValueOK(env, rv) == CK_ASSERT_OK) {
jKeyHandles = ckULongArrayToJLongArray(env, ckpKeyHandles, 2);
diff -r e5d52546ba3a -r bf48a9f13cf2 jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_util.c
--- a/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_util.c Wed Jun 22 11:31:07 2016 -0700
+++ b/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/p11_util.c Wed Jun 22 15:58:08 2016 -0700
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
*/
/* Copyright (c) 2002 Graz University of Technology. All rights reserved.
@@ -1143,3 +1143,15 @@
#endif
+// prints a message to stdout if debug output is enabled
+void printDebug(const char *format, ...) {
+ if (debug == JNI_TRUE) {
+ va_list args;
+ fprintf(stdout, "sunpkcs11: ");
+ va_start(args, format);
+ vfprintf(stdout, format, args);
+ va_end(args);
+ fflush(stdout);
+ }
+}
+
diff -r e5d52546ba3a -r bf48a9f13cf2 jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/pkcs11wrapper.h
--- a/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/pkcs11wrapper.h Wed Jun 22 11:31:07 2016 -0700
+++ b/jdk/src/jdk.crypto.pkcs11/share/native/libj2pkcs11/pkcs11wrapper.h Wed Jun 22 15:58:08 2016 -0700
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
*/
/* Copyright (c) 2002 Graz University of Technology. All rights reserved.
@@ -159,6 +159,7 @@
#include "pkcs-11v2-20a3.h"
#include <jni.h>
#include <jni_util.h>
+#include <stdarg.h>
#define MAX_STACK_BUFFER_LEN (4 * 1024)
#define MAX_HEAP_BUFFER_LEN (64 * 1024)
@@ -217,6 +218,10 @@
#define TRACE_UNINTEND
#endif
+/* debug output */
+extern jboolean debug;
+void printDebug(const char *format, ...);
+
#define CK_ASSERT_OK 0L
#define CLASS_INFO "sun/security/pkcs11/wrapper/CK_INFO"
diff -r e5d52546ba3a -r bf48a9f13cf2 jdk/test/ProblemList.txt
--- a/jdk/test/ProblemList.txt Wed Jun 22 11:31:07 2016 -0700
+++ b/jdk/test/ProblemList.txt Wed Jun 22 15:58:08 2016 -0700
@@ -272,8 +272,6 @@
sun/security/pkcs11/tls/TestPRF.java 8077138,8023434 windows-all
sun/security/pkcs11/tls/TestPremaster.java 8077138,8023434 windows-all
-sun/security/pkcs11/rsa/TestKeyPairGenerator.java 8074580 generic-all
-
sun/security/krb5/auto/HttpNegotiateServer.java 8038079 generic-all
sun/security/tools/keytool/autotest.sh 8130302 generic-all
diff -r e5d52546ba3a -r bf48a9f13cf2 jdk/test/sun/security/pkcs11/rsa/TestKeyPairGenerator.java
--- a/jdk/test/sun/security/pkcs11/rsa/TestKeyPairGenerator.java Wed Jun 22 11:31:07 2016 -0700
+++ b/jdk/test/sun/security/pkcs11/rsa/TestKeyPairGenerator.java Wed Jun 22 15:58:08 2016 -0700
@@ -29,8 +29,9 @@
* @library ..
* @library /lib/testlibrary
* @build jdk.testlibrary.*
- * @run main/othervm TestKeyPairGenerator
- * @run main/othervm TestKeyPairGenerator sm TestKeyPairGenerator.policy
+ * @run main/othervm -Djava.security.debug=sunpkcs11 TestKeyPairGenerator
+ * @run main/othervm -Djava.security.debug=sunpkcs11 TestKeyPairGenerator
+ * sm TestKeyPairGenerator.policy
* @key intermittent randomness
*/