diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/compiler/compileBroker.cpp
--- a/hotspot/src/share/vm/compiler/compileBroker.cpp	Fri Jan 29 08:33:24 2010 -0800
+++ b/hotspot/src/share/vm/compiler/compileBroker.cpp	Fri Jan 29 09:27:22 2010 -0800
@@ -69,6 +69,7 @@
 
 bool CompileBroker::_initialized = false;
 volatile bool CompileBroker::_should_block = false;
+volatile jint CompileBroker::_should_compile_new_jobs = run_compilation;
 
 // The installed compiler(s)
 AbstractCompiler* CompileBroker::_compilers[2];
@@ -986,6 +987,13 @@
       return method_code;
     }
     if (method->is_not_compilable(comp_level)) return NULL;
+
+    nmethod* saved = CodeCache::find_and_remove_saved_code(method());
+    if (saved != NULL) {
+      method->set_code(method, saved);
+      return saved;
+    }
+
   } else {
     // osr compilation
 #ifndef TIERED
@@ -1037,6 +1045,14 @@
     method->jmethod_id();
   }
 
+  // If the compiler is shut off due to code cache flushing or otherwise,
+  // fail out now so blocking compiles dont hang the java thread
+  if (!should_compile_new_jobs() || (UseCodeCacheFlushing && CodeCache::needs_flushing())) {
+    method->invocation_counter()->decay();
+    method->backedge_counter()->decay();
+    return NULL;
+  }
+
   // do the compilation
   if (method->is_native()) {
     if (!PreferInterpreterNativeStubs) {
@@ -1325,26 +1341,13 @@
     {
       // We need this HandleMark to avoid leaking VM handles.
       HandleMark hm(thread);
+
       if (CodeCache::unallocated_capacity() < CodeCacheMinimumFreeSpace) {
-        // The CodeCache is full.  Print out warning and disable compilation.
-        UseInterpreter = true;
-        if (UseCompiler || AlwaysCompileLoopMethods ) {
-          if (log != NULL) {
-            log->begin_elem("code_cache_full");
-            log->stamp();
-            log->end_elem();
-          }
-#ifndef PRODUCT
-          warning("CodeCache is full. Compiler has been disabled");
-          if (CompileTheWorld || ExitOnFullCodeCache) {
-            before_exit(thread);
-            exit_globals(); // will delete tty
-            vm_direct_exit(CompileTheWorld ? 0 : 1);
-          }
-#endif
-          UseCompiler               = false;
-          AlwaysCompileLoopMethods  = false;
-        }
+        // the code cache is really full
+        handle_full_code_cache();
+      } else if (UseCodeCacheFlushing && CodeCache::needs_flushing()) {
+        // Attempt to start cleaning the code cache while there is still a little headroom
+        NMethodSweeper::handle_full_code_cache(false);
       }
 
       CompileTask* task = queue->get();
@@ -1369,7 +1372,7 @@
       // Never compile a method if breakpoints are present in it
       if (method()->number_of_breakpoints() == 0) {
         // Compile the method.
-        if (UseCompiler || AlwaysCompileLoopMethods) {
+        if ((UseCompiler || AlwaysCompileLoopMethods) && CompileBroker::should_compile_new_jobs()) {
 #ifdef COMPILER1
           // Allow repeating compilations for the purpose of benchmarking
           // compile speed. This is not useful for customers.
@@ -1614,6 +1617,38 @@
 
 
 // ------------------------------------------------------------------
+// CompileBroker::handle_full_code_cache
+//
+// The CodeCache is full.  Print out warning and disable compilation or
+// try code cache cleaning so compilation can continue later.
+void CompileBroker::handle_full_code_cache() {
+  UseInterpreter = true;
+  if (UseCompiler || AlwaysCompileLoopMethods ) {
+    CompilerThread* thread = CompilerThread::current();
+    CompileLog* log = thread->log();
+    if (log != NULL) {
+      log->begin_elem("code_cache_full");
+      log->stamp();
+      log->end_elem();
+    }
+  #ifndef PRODUCT
+    warning("CodeCache is full. Compiler has been disabled");
+    if (CompileTheWorld || ExitOnFullCodeCache) {
+      before_exit(JavaThread::current());
+      exit_globals(); // will delete tty
+      vm_direct_exit(CompileTheWorld ? 0 : 1);
+    }
+  #endif
+    if (UseCodeCacheFlushing) {
+      NMethodSweeper::handle_full_code_cache(true);
+    } else {
+      UseCompiler               = false;
+      AlwaysCompileLoopMethods  = false;
+    }
+  }
+}
+
+// ------------------------------------------------------------------
 // CompileBroker::set_last_compile
 //
 // Record this compilation for debugging purposes.