--- a/hotspot/src/share/vm/memory/allocation.cpp	Tue Aug 28 15:15:29 2012 +0200
+++ b/hotspot/src/share/vm/memory/allocation.cpp	Wed Oct 17 17:36:48 2012 +0200
@@ -92,6 +92,26 @@
   return res;
 }
 
+void* ResourceObj::operator new(size_t size, const std::nothrow_t&  nothrow_constant,
+    allocation_type type, MEMFLAGS flags) {
+  //should only call this with std::nothrow, use other operator new() otherwise
+  address res;
+  switch (type) {
+   case C_HEAP:
+    res = (address)AllocateHeap(size, flags, CALLER_PC, AllocFailStrategy::RETURN_NULL);
+    DEBUG_ONLY(if (res!= NULL) set_allocation_type(res, C_HEAP);)
+    break;
+   case RESOURCE_AREA:
+    // new(size) sets allocation type RESOURCE_AREA.
+    res = (address)operator new(size, std::nothrow);
+    break;
+   default:
+    ShouldNotReachHere();
+  }
+  return res;
+}
+
+
 void ResourceObj::operator delete(void* p) {
   assert(((ResourceObj *)p)->allocated_on_C_heap(),
          "delete only allowed for C_HEAP objects");
@@ -506,7 +526,7 @@
 }
 
 // Grow a new Chunk
-void* Arena::grow( size_t x ) {
+void* Arena::grow(size_t x, AllocFailType alloc_failmode) {
   // Get minimal required size.  Either real big, or even bigger for giant objs
   size_t len = MAX2(x, (size_t) Chunk::size);
 
@@ -514,7 +534,10 @@
   _chunk = new (len) Chunk(len);
 
   if (_chunk == NULL) {
-    signal_out_of_memory(len * Chunk::aligned_overhead_size(), "Arena::grow");
+    if (alloc_failmode == AllocFailStrategy::EXIT_OOM) {
+      signal_out_of_memory(len * Chunk::aligned_overhead_size(), "Arena::grow");
+    }
+    return NULL;
   }
   if (k) k->set_next(_chunk);   // Append new chunk to end of linked list
   else _first = _chunk;
@@ -529,13 +552,16 @@
 
 
 // Reallocate storage in Arena.
-void *Arena::Arealloc(void* old_ptr, size_t old_size, size_t new_size) {
+void *Arena::Arealloc(void* old_ptr, size_t old_size, size_t new_size, AllocFailType alloc_failmode) {
   assert(new_size >= 0, "bad size");
   if (new_size == 0) return NULL;
 #ifdef ASSERT
   if (UseMallocOnly) {
     // always allocate a new object  (otherwise we'll free this one twice)
-    char* copy = (char*)Amalloc(new_size);
+    char* copy = (char*)Amalloc(new_size, alloc_failmode);
+    if (copy == NULL) {
+      return NULL;
+    }
     size_t n = MIN2(old_size, new_size);
     if (n > 0) memcpy(copy, old_ptr, n);
     Afree(old_ptr,old_size);    // Mostly done to keep stats accurate
@@ -561,7 +587,10 @@
   }
 
   // Oops, got to relocate guts
-  void *new_ptr = Amalloc(new_size);
+  void *new_ptr = Amalloc(new_size, alloc_failmode);
+  if (new_ptr == NULL) {
+    return NULL;
+  }
   memcpy( new_ptr, c_old, old_size );
   Afree(c_old,old_size);        // Mostly done to keep stats accurate
   return new_ptr;

--- a/hotspot/src/share/vm/memory/allocation.hpp	Tue Aug 28 15:15:29 2012 +0200
+++ b/hotspot/src/share/vm/memory/allocation.hpp	Wed Oct 17 17:36:48 2012 +0200
@@ -53,6 +53,12 @@
   #endif
 #endif
 
+class AllocFailStrategy {
+public:
+  enum AllocFailEnum { EXIT_OOM, RETURN_NULL };
+};
+typedef AllocFailStrategy::AllocFailEnum AllocFailType;
+
 // All classes in the virtual machine must be subclassed
 // by one of the following allocation classes:
 //
@@ -315,7 +321,8 @@
   Chunk *_first;                // First chunk
   Chunk *_chunk;                // current chunk
   char *_hwm, *_max;            // High water mark and max in current chunk
-  void* grow(size_t x);         // Get a new Chunk of at least size x
+  // Get a new Chunk of at least size x
+  void* grow(size_t x, AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
   size_t _size_in_bytes;        // Size of arena (used for native memory tracking)
 
   NOT_PRODUCT(static julong _bytes_allocated;) // total #bytes allocated since start
@@ -350,14 +357,14 @@
   void  operator delete(void* p);
 
   // Fast allocate in the arena.  Common case is: pointer test + increment.
-  void* Amalloc(size_t x) {
+  void* Amalloc(size_t x, AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM) {
     assert(is_power_of_2(ARENA_AMALLOC_ALIGNMENT) , "should be a power of 2");
     x = ARENA_ALIGN(x);
     debug_only(if (UseMallocOnly) return malloc(x);)
     check_for_overflow(x, "Arena::Amalloc");
     NOT_PRODUCT(inc_bytes_allocated(x);)
     if (_hwm + x > _max) {
-      return grow(x);
+      return grow(x, alloc_failmode);
     } else {
       char *old = _hwm;
       _hwm += x;
@@ -365,13 +372,13 @@
     }
   }
   // Further assume size is padded out to words
-  void *Amalloc_4(size_t x) {
+  void *Amalloc_4(size_t x, AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM) {
     assert( (x&(sizeof(char*)-1)) == 0, "misaligned size" );
     debug_only(if (UseMallocOnly) return malloc(x);)
     check_for_overflow(x, "Arena::Amalloc_4");
     NOT_PRODUCT(inc_bytes_allocated(x);)
     if (_hwm + x > _max) {
-      return grow(x);
+      return grow(x, alloc_failmode);
     } else {
       char *old = _hwm;
       _hwm += x;
@@ -381,7 +388,7 @@
 
   // Allocate with 'double' alignment. It is 8 bytes on sparc.
   // In other cases Amalloc_D() should be the same as Amalloc_4().
-  void* Amalloc_D(size_t x) {
+  void* Amalloc_D(size_t x, AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM) {
     assert( (x&(sizeof(char*)-1)) == 0, "misaligned size" );
     debug_only(if (UseMallocOnly) return malloc(x);)
 #if defined(SPARC) && !defined(_LP64)
@@ -392,7 +399,7 @@
     check_for_overflow(x, "Arena::Amalloc_D");
     NOT_PRODUCT(inc_bytes_allocated(x);)
     if (_hwm + x > _max) {
-      return grow(x); // grow() returns a result aligned >= 8 bytes.
+      return grow(x, alloc_failmode); // grow() returns a result aligned >= 8 bytes.
     } else {
       char *old = _hwm;
       _hwm += x;
@@ -412,7 +419,8 @@
     if (((char*)ptr) + size == _hwm) _hwm = (char*)ptr;
   }
 
-  void *Arealloc( void *old_ptr, size_t old_size, size_t new_size );
+  void *Arealloc( void *old_ptr, size_t old_size, size_t new_size,
+      AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
 
   // Move contents of this arena into an empty arena
   Arena *move_contents(Arena *empty_arena);
@@ -458,9 +466,12 @@
 
 
 //%note allocation_1
-extern char* resource_allocate_bytes(size_t size);
-extern char* resource_allocate_bytes(Thread* thread, size_t size);
-extern char* resource_reallocate_bytes( char *old, size_t old_size, size_t new_size);
+extern char* resource_allocate_bytes(size_t size,
+    AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
+extern char* resource_allocate_bytes(Thread* thread, size_t size,
+    AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
+extern char* resource_reallocate_bytes( char *old, size_t old_size, size_t new_size,
+    AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM);
 extern void resource_free_bytes( char *old, size_t size );
 
 //----------------------------------------------------------------------
@@ -496,6 +507,8 @@
 
  public:
   void* operator new(size_t size, allocation_type type, MEMFLAGS flags);
+  void* operator new(size_t size, const std::nothrow_t&  nothrow_constant,
+      allocation_type type, MEMFLAGS flags);
   void* operator new(size_t size, Arena *arena) {
       address res = (address)arena->Amalloc(size);
       DEBUG_ONLY(set_allocation_type(res, ARENA);)
@@ -506,6 +519,13 @@
       DEBUG_ONLY(set_allocation_type(res, RESOURCE_AREA);)
       return res;
   }
+
+  void* operator new(size_t size, const std::nothrow_t& nothrow_constant) {
+      address res = (address)resource_allocate_bytes(size, AllocFailStrategy::RETURN_NULL);
+      DEBUG_ONLY(if (res != NULL) set_allocation_type(res, RESOURCE_AREA);)
+      return res;
+  }
+
   void  operator delete(void* p);
 };
 

--- a/hotspot/src/share/vm/memory/allocation.inline.hpp	Tue Aug 28 15:15:29 2012 +0200
+++ b/hotspot/src/share/vm/memory/allocation.inline.hpp	Wed Oct 17 17:36:48 2012 +0200
@@ -48,7 +48,8 @@
 #endif
 
 // allocate using malloc; will fail if no memory available
-inline char* AllocateHeap(size_t size, MEMFLAGS flags, address pc = 0) {
+inline char* AllocateHeap(size_t size, MEMFLAGS flags, address pc = 0,
+    AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM) {
   if (pc == 0) {
     pc = CURRENT_PC;
   }
@@ -56,16 +57,17 @@
   #ifdef ASSERT
   if (PrintMallocFree) trace_heap_malloc(size, "AllocateHeap", p);
   #endif
-  if (p == NULL) vm_exit_out_of_memory(size, "AllocateHeap");
+  if (p == NULL && alloc_failmode == AllocFailStrategy::EXIT_OOM) vm_exit_out_of_memory(size, "AllocateHeap");
   return p;
 }
 
-inline char* ReallocateHeap(char *old, size_t size, MEMFLAGS flags) {
+inline char* ReallocateHeap(char *old, size_t size, MEMFLAGS flags,
+    AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM) {
   char* p = (char*) os::realloc(old, size, flags, CURRENT_PC);
   #ifdef ASSERT
   if (PrintMallocFree) trace_heap_malloc(size, "ReallocateHeap", p);
   #endif
-  if (p == NULL) vm_exit_out_of_memory(size, "ReallocateHeap");
+  if (p == NULL && alloc_failmode == AllocFailStrategy::EXIT_OOM) vm_exit_out_of_memory(size, "ReallocateHeap");
   return p;
 }
 
@@ -91,11 +93,13 @@
 template <MEMFLAGS F> void* CHeapObj<F>::operator new (size_t size,
   const std::nothrow_t&  nothrow_constant, address caller_pc) {
 #ifdef ASSERT
-    void* p = os::malloc(size, F, (caller_pc != 0 ? caller_pc : CALLER_PC));
+  void* p = (void*)AllocateHeap(size, F, (caller_pc != 0 ? caller_pc : CALLER_PC),
+      AllocFailStrategy::RETURN_NULL);
     if (PrintMallocFree) trace_heap_malloc(size, "CHeapObj-new", p);
     return p;
 #else
-    return os::malloc(size, F, (caller_pc != 0 ? caller_pc : CALLER_PC));
+  return (void *) AllocateHeap(size, F, (caller_pc != 0 ? caller_pc : CALLER_PC),
+      AllocFailStrategy::RETURN_NULL);
 #endif
 }
 

--- a/hotspot/src/share/vm/memory/resourceArea.cpp	Tue Aug 28 15:15:29 2012 +0200
+++ b/hotspot/src/share/vm/memory/resourceArea.cpp	Wed Oct 17 17:36:48 2012 +0200
@@ -45,15 +45,15 @@
 // The following routines are declared in allocation.hpp and used everywhere:
 
 // Allocation in thread-local resource area
-extern char* resource_allocate_bytes(size_t size) {
-  return Thread::current()->resource_area()->allocate_bytes(size);
+extern char* resource_allocate_bytes(size_t size, AllocFailType alloc_failmode) {
+  return Thread::current()->resource_area()->allocate_bytes(size, alloc_failmode);
 }
-extern char* resource_allocate_bytes(Thread* thread, size_t size) {
-  return thread->resource_area()->allocate_bytes(size);
+extern char* resource_allocate_bytes(Thread* thread, size_t size, AllocFailType alloc_failmode) {
+  return thread->resource_area()->allocate_bytes(size, alloc_failmode);
 }
 
-extern char* resource_reallocate_bytes( char *old, size_t old_size, size_t new_size){
-  return (char*)Thread::current()->resource_area()->Arealloc(old, old_size, new_size);
+extern char* resource_reallocate_bytes( char *old, size_t old_size, size_t new_size, AllocFailType alloc_failmode){
+  return (char*)Thread::current()->resource_area()->Arealloc(old, old_size, new_size, alloc_failmode);
 }
 
 extern void resource_free_bytes( char *old, size_t size ) {

--- a/hotspot/src/share/vm/memory/resourceArea.hpp	Tue Aug 28 15:15:29 2012 +0200
+++ b/hotspot/src/share/vm/memory/resourceArea.hpp	Wed Oct 17 17:36:48 2012 +0200
@@ -68,7 +68,7 @@
     debug_only(_nesting = 0;);
   }
 
-  char* allocate_bytes(size_t size) {
+  char* allocate_bytes(size_t size, AllocFailType alloc_failmode = AllocFailStrategy::EXIT_OOM) {
 #ifdef ASSERT
     if (_nesting < 1 && !_warned++)
       fatal("memory leak: allocating without ResourceMark");
@@ -78,7 +78,7 @@
       return (*save = (char*)os::malloc(size, mtThread));
     }
 #endif
-    return (char*)Amalloc(size);
+    return (char*)Amalloc(size, alloc_failmode);
   }
 
   debug_only(int nesting() const { return _nesting; });

--- a/hotspot/src/share/vm/runtime/thread.cpp	Tue Aug 28 15:15:29 2012 +0200
+++ b/hotspot/src/share/vm/runtime/thread.cpp	Wed Oct 17 17:36:48 2012 +0200
@@ -177,7 +177,8 @@
     const int alignment = markOopDesc::biased_lock_alignment;
     size_t aligned_size = size + (alignment - sizeof(intptr_t));
     void* real_malloc_addr = throw_excpt? AllocateHeap(aligned_size, flags, CURRENT_PC)
-                                          : os::malloc(aligned_size, flags, CURRENT_PC);
+                                          : AllocateHeap(aligned_size, flags, CURRENT_PC,
+                                              AllocFailStrategy::RETURN_NULL);
     void* aligned_addr     = (void*) align_size_up((intptr_t) real_malloc_addr, alignment);
     assert(((uintptr_t) aligned_addr + (uintptr_t) size) <=
            ((uintptr_t) real_malloc_addr + (uintptr_t) aligned_size),
@@ -191,7 +192,7 @@
     return aligned_addr;
   } else {
     return throw_excpt? AllocateHeap(size, flags, CURRENT_PC)
-                       : os::malloc(size, flags, CURRENT_PC);
+                       : AllocateHeap(size, flags, CURRENT_PC, AllocFailStrategy::RETURN_NULL);
   }
 }
 

--- a/hotspot/src/share/vm/runtime/thread.hpp	Tue Aug 28 15:15:29 2012 +0200
+++ b/hotspot/src/share/vm/runtime/thread.hpp	Wed Oct 17 17:36:48 2012 +0200
@@ -110,7 +110,7 @@
   void*       _real_malloc_address;
  public:
   void* operator new(size_t size) { return allocate(size, true); }
-  void* operator new(size_t size, std::nothrow_t& nothrow_constant) { return allocate(size, false); }
+  void* operator new(size_t size, const std::nothrow_t& nothrow_constant) { return allocate(size, false); }
   void  operator delete(void* p);
 
  protected: