--- a/src/hotspot/share/jvmci/jvmciRuntime.cpp	Fri Oct 05 18:25:15 2018 +0100
+++ b/src/hotspot/share/jvmci/jvmciRuntime.cpp	Fri Oct 05 20:03:14 2018 +0200
@@ -109,22 +109,72 @@
   }
 }
 
-JRT_BLOCK_ENTRY(void, JVMCIRuntime::new_instance(JavaThread* thread, Klass* klass))
+// Manages a scope for a JVMCI runtime call that attempts a heap allocation.
+// If there is a pending exception upon closing the scope and the runtime
+// call is of the variety where allocation failure returns NULL without an
+// exception, the following action is taken:
+//   1. The pending exception is cleared
+//   2. NULL is written to JavaThread::_vm_result
+//   3. Checks that an OutOfMemoryError is Universe::out_of_memory_error_retry().
+class RetryableAllocationMark: public StackObj {
+ private:
+  JavaThread* _thread;
+ public:
+  RetryableAllocationMark(JavaThread* thread, bool activate) {
+    if (activate) {
+      assert(!thread->in_retryable_allocation(), "retryable allocation scope is non-reentrant");
+      _thread = thread;
+      _thread->set_in_retryable_allocation(true);
+    } else {
+      _thread = NULL;
+    }
+  }
+  ~RetryableAllocationMark() {
+    if (_thread != NULL) {
+      _thread->set_in_retryable_allocation(false);
+      JavaThread* THREAD = _thread;
+      if (HAS_PENDING_EXCEPTION) {
+        oop ex = PENDING_EXCEPTION;
+        CLEAR_PENDING_EXCEPTION;
+        oop retry_oome = Universe::out_of_memory_error_retry();
+        if (ex->is_a(retry_oome->klass()) && retry_oome != ex) {
+          ResourceMark rm;
+          fatal("Unexpected exception in scope of retryable allocation: " INTPTR_FORMAT " of type %s", p2i(ex), ex->klass()->external_name());
+        }
+        _thread->set_vm_result(NULL);
+      }
+    }
+  }
+};
+
+JRT_BLOCK_ENTRY(void, JVMCIRuntime::new_instance_common(JavaThread* thread, Klass* klass, bool null_on_fail))
   JRT_BLOCK;
   assert(klass->is_klass(), "not a class");
   Handle holder(THREAD, klass->klass_holder()); // keep the klass alive
   InstanceKlass* ik = InstanceKlass::cast(klass);
-  ik->check_valid_for_instantiation(true, CHECK);
-  // make sure klass is initialized
-  ik->initialize(CHECK);
-  // allocate instance and return via TLS
-  oop obj = ik->allocate_instance(CHECK);
-  thread->set_vm_result(obj);
+  {
+    RetryableAllocationMark ram(thread, null_on_fail);
+    ik->check_valid_for_instantiation(true, CHECK);
+    oop obj;
+    if (null_on_fail) {
+      if (!ik->is_initialized()) {
+        // Cannot re-execute class initialization without side effects
+        // so return without attempting the initialization
+        return;
+      }
+    } else {
+      // make sure klass is initialized
+      ik->initialize(CHECK);
+    }
+    // allocate instance and return via TLS
+    obj = ik->allocate_instance(CHECK);
+    thread->set_vm_result(obj);
+  }
   JRT_BLOCK_END;
   SharedRuntime::on_slowpath_allocation_exit(thread);
 JRT_END
 
-JRT_BLOCK_ENTRY(void, JVMCIRuntime::new_array(JavaThread* thread, Klass* array_klass, jint length))
+JRT_BLOCK_ENTRY(void, JVMCIRuntime::new_array_common(JavaThread* thread, Klass* array_klass, jint length, bool null_on_fail))
   JRT_BLOCK;
   // Note: no handle for klass needed since they are not used
   //       anymore after new_objArray() and no GC can happen before.
@@ -133,10 +183,12 @@
   oop obj;
   if (array_klass->is_typeArray_klass()) {
     BasicType elt_type = TypeArrayKlass::cast(array_klass)->element_type();
+    RetryableAllocationMark ram(thread, null_on_fail);
     obj = oopFactory::new_typeArray(elt_type, length, CHECK);
   } else {
     Handle holder(THREAD, array_klass->klass_holder()); // keep the klass alive
     Klass* elem_klass = ObjArrayKlass::cast(array_klass)->element_klass();
+    RetryableAllocationMark ram(thread, null_on_fail);
     obj = oopFactory::new_objArray(elem_klass, length, CHECK);
   }
   thread->set_vm_result(obj);
@@ -146,8 +198,12 @@
     static int deopts = 0;
     // Alternate between deoptimizing and raising an error (which will also cause a deopt)
     if (deopts++ % 2 == 0) {
-      ResourceMark rm(THREAD);
-      THROW(vmSymbols::java_lang_OutOfMemoryError());
+      if (null_on_fail) {
+        return;
+      } else {
+        ResourceMark rm(THREAD);
+        THROW(vmSymbols::java_lang_OutOfMemoryError());
+      }
     } else {
       deopt_caller();
     }
@@ -156,32 +212,43 @@
   SharedRuntime::on_slowpath_allocation_exit(thread);
 JRT_END
 
-JRT_ENTRY(void, JVMCIRuntime::new_multi_array(JavaThread* thread, Klass* klass, int rank, jint* dims))
+JRT_ENTRY(void, JVMCIRuntime::new_multi_array_common(JavaThread* thread, Klass* klass, int rank, jint* dims, bool null_on_fail))
   assert(klass->is_klass(), "not a class");
   assert(rank >= 1, "rank must be nonzero");
   Handle holder(THREAD, klass->klass_holder()); // keep the klass alive
+  RetryableAllocationMark ram(thread, null_on_fail);
   oop obj = ArrayKlass::cast(klass)->multi_allocate(rank, dims, CHECK);
   thread->set_vm_result(obj);
 JRT_END
 
-JRT_ENTRY(void, JVMCIRuntime::dynamic_new_array(JavaThread* thread, oopDesc* element_mirror, jint length))
+JRT_ENTRY(void, JVMCIRuntime::dynamic_new_array_common(JavaThread* thread, oopDesc* element_mirror, jint length, bool null_on_fail))
+  RetryableAllocationMark ram(thread, null_on_fail);
   oop obj = Reflection::reflect_new_array(element_mirror, length, CHECK);
   thread->set_vm_result(obj);
 JRT_END
 
-JRT_ENTRY(void, JVMCIRuntime::dynamic_new_instance(JavaThread* thread, oopDesc* type_mirror))
+JRT_ENTRY(void, JVMCIRuntime::dynamic_new_instance_common(JavaThread* thread, oopDesc* type_mirror, bool null_on_fail))
   InstanceKlass* klass = InstanceKlass::cast(java_lang_Class::as_Klass(type_mirror));
 
   if (klass == NULL) {
     ResourceMark rm(THREAD);
     THROW(vmSymbols::java_lang_InstantiationException());
   }
+  RetryableAllocationMark ram(thread, null_on_fail);
 
   // Create new instance (the receiver)
   klass->check_valid_for_instantiation(false, CHECK);
 
-  // Make sure klass gets initialized
-  klass->initialize(CHECK);
+  if (null_on_fail) {
+    if (!klass->is_initialized()) {
+      // Cannot re-execute class initialization without side effects
+      // so return without attempting the initialization
+      return;
+    }
+  } else {
+    // Make sure klass gets initialized
+    klass->initialize(CHECK);
+  }
 
   oop obj = klass->allocate_instance(CHECK);
   thread->set_vm_result(obj);