# HG changeset patch # User kvn # Date 1264786042 28800 # Node ID 71fd601907dc7be3b3297636c45b1deeb49172a3 # Parent f26b30116e3a2b5f42dc117f157acd5772659693 4360113: Evict nmethods when code cache gets full Summary: Speculatively unload the oldest nmethods when code cache gets full. Reviewed-by: never, kvn Contributed-by: eric.caspole@amd.com diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/ci/ciEnv.cpp --- a/hotspot/src/share/vm/ci/ciEnv.cpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/ci/ciEnv.cpp Fri Jan 29 09:27:22 2010 -0800 @@ -962,18 +962,10 @@ if (nm == NULL) { // The CodeCache is full. Print out warning and disable compilation. record_failure("code cache is full"); - UseInterpreter = true; - if (UseCompiler || AlwaysCompileLoopMethods ) { -#ifndef PRODUCT - warning("CodeCache is full. Compiler has been disabled"); - if (CompileTheWorld || ExitOnFullCodeCache) { - before_exit(JavaThread::current()); - exit_globals(); // will delete tty - vm_direct_exit(CompileTheWorld ? 0 : 1); - } -#endif - UseCompiler = false; - AlwaysCompileLoopMethods = false; + { + MutexUnlocker ml(Compile_lock); + MutexUnlocker locker(MethodCompileQueue_lock); + CompileBroker::handle_full_code_cache(); } } else { NOT_PRODUCT(nm->set_has_debug_info(has_debug_info); ) diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/code/codeCache.cpp --- a/hotspot/src/share/vm/code/codeCache.cpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/code/codeCache.cpp Fri Jan 29 09:27:22 2010 -0800 @@ -96,6 +96,7 @@ int CodeCache::_number_of_nmethods_with_dependencies = 0; bool CodeCache::_needs_cache_clean = false; nmethod* CodeCache::_scavenge_root_nmethods = NULL; +nmethod* CodeCache::_saved_nmethods = NULL; CodeBlob* CodeCache::first() { @@ -395,6 +396,85 @@ } #endif //PRODUCT + +nmethod* CodeCache::find_and_remove_saved_code(methodOop m) { + MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); + nmethod* saved = _saved_nmethods; + nmethod* prev = NULL; + while (saved != NULL) { + if (saved->is_in_use() && saved->method() == m) { + if (prev != NULL) { + prev->set_saved_nmethod_link(saved->saved_nmethod_link()); + } else { + _saved_nmethods = saved->saved_nmethod_link(); + } + assert(saved->is_speculatively_disconnected(), "shouldn't call for other nmethods"); + saved->set_speculatively_disconnected(false); + saved->set_saved_nmethod_link(NULL); + if (PrintMethodFlushing) { + saved->print_on(tty, " ### nmethod is reconnected"); + } + if (LogCompilation && (xtty != NULL)) { + ttyLocker ttyl; + xtty->begin_elem("nmethod_reconnected compile_id='%3d'", saved->compile_id()); + xtty->method(methodOop(m)); + xtty->stamp(); + xtty->end_elem(); + } + return saved; + } + prev = saved; + saved = saved->saved_nmethod_link(); + } + return NULL; +} + +void CodeCache::remove_saved_code(nmethod* nm) { + MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag); + assert(nm->is_speculatively_disconnected(), "shouldn't call for other nmethods"); + nmethod* saved = _saved_nmethods; + nmethod* prev = NULL; + while (saved != NULL) { + if (saved == nm) { + if (prev != NULL) { + prev->set_saved_nmethod_link(saved->saved_nmethod_link()); + } else { + _saved_nmethods = saved->saved_nmethod_link(); + } + if (LogCompilation && (xtty != NULL)) { + ttyLocker ttyl; + xtty->begin_elem("nmethod_removed compile_id='%3d'", nm->compile_id()); + xtty->stamp(); + xtty->end_elem(); + } + return; + } + prev = saved; + saved = saved->saved_nmethod_link(); + } + ShouldNotReachHere(); +} + +void CodeCache::speculatively_disconnect(nmethod* nm) { + assert_locked_or_safepoint(CodeCache_lock); + assert(nm->is_in_use() && !nm->is_speculatively_disconnected(), "should only disconnect live nmethods"); + nm->set_saved_nmethod_link(_saved_nmethods); + _saved_nmethods = nm; + if (PrintMethodFlushing) { + nm->print_on(tty, " ### nmethod is speculatively disconnected"); + } + if (LogCompilation && (xtty != NULL)) { + ttyLocker ttyl; + xtty->begin_elem("nmethod_disconnected compile_id='%3d'", nm->compile_id()); + xtty->method(methodOop(nm->method())); + xtty->stamp(); + xtty->end_elem(); + } + nm->method()->clear_code(); + nm->set_speculatively_disconnected(true); +} + + void CodeCache::gc_prologue() { assert(!nmethod::oops_do_marking_is_active(), "oops_do_marking_epilogue must be called"); } diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/code/codeCache.hpp --- a/hotspot/src/share/vm/code/codeCache.hpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/code/codeCache.hpp Fri Jan 29 09:27:22 2010 -0800 @@ -46,6 +46,7 @@ static int _number_of_nmethods_with_dependencies; static bool _needs_cache_clean; static nmethod* _scavenge_root_nmethods; // linked via nm->scavenge_root_link() + static nmethod* _saved_nmethods; // linked via nm->saved_nmethod_look() static void verify_if_often() PRODUCT_RETURN; @@ -141,11 +142,16 @@ static size_t capacity() { return _heap->capacity(); } static size_t max_capacity() { return _heap->max_capacity(); } static size_t unallocated_capacity() { return _heap->unallocated_capacity(); } + static bool needs_flushing() { return unallocated_capacity() < CodeCacheFlushingMinimumFreeSpace; } static bool needs_cache_clean() { return _needs_cache_clean; } static void set_needs_cache_clean(bool v) { _needs_cache_clean = v; } static void clear_inline_caches(); // clear all inline caches + static nmethod* find_and_remove_saved_code(methodOop m); + static void remove_saved_code(nmethod* nm); + static void speculatively_disconnect(nmethod* nm); + // Deoptimization static int mark_for_deoptimization(DepChange& changes); #ifdef HOTSWAP diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/code/nmethod.cpp --- a/hotspot/src/share/vm/code/nmethod.cpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/code/nmethod.cpp Fri Jan 29 09:27:22 2010 -0800 @@ -587,6 +587,7 @@ _osr_link = NULL; _scavenge_root_link = NULL; _scavenge_root_state = 0; + _saved_nmethod_link = NULL; _compiler = NULL; // We have no exception handler or deopt handler make the // values something that will never match a pc like the nmethod vtable entry @@ -1033,7 +1034,7 @@ if( cb != NULL && cb->is_nmethod() ) { nmethod* nm = (nmethod*)cb; // Clean inline caches pointing to both zombie and not_entrant methods - if (!nm->is_in_use()) ic->set_to_clean(); + if (!nm->is_in_use() || (nm->method()->code() != nm)) ic->set_to_clean(); } break; } @@ -1043,7 +1044,7 @@ if( cb != NULL && cb->is_nmethod() ) { nmethod* nm = (nmethod*)cb; // Clean inline caches pointing to both zombie and not_entrant methods - if (!nm->is_in_use()) csc->set_to_clean(); + if (!nm->is_in_use() || (nm->method()->code() != nm)) csc->set_to_clean(); } break; } @@ -1312,7 +1313,8 @@ // completely deallocate this method EventMark m("flushing nmethod " INTPTR_FORMAT " %s", this, ""); if (PrintMethodFlushing) { - tty->print_cr("*flushing nmethod " INTPTR_FORMAT ". Live blobs: %d", this, CodeCache::nof_blobs()); + tty->print_cr("*flushing nmethod %3d/" INTPTR_FORMAT ". Live blobs:" UINT32_FORMAT "/Free CodeCache:" SIZE_FORMAT "Kb", + _compile_id, this, CodeCache::nof_blobs(), CodeCache::unallocated_capacity()/1024); } // We need to deallocate any ExceptionCache data. @@ -1330,6 +1332,10 @@ CodeCache::drop_scavenge_root_nmethod(this); } + if (is_speculatively_disconnected()) { + CodeCache::remove_saved_code(this); + } + ((CodeBlob*)(this))->flush(); CodeCache::free(this); diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/code/nmethod.hpp --- a/hotspot/src/share/vm/code/nmethod.hpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/code/nmethod.hpp Fri Jan 29 09:27:22 2010 -0800 @@ -95,6 +95,8 @@ unsigned int has_unsafe_access:1; // May fault due to unsafe access. unsigned int has_method_handle_invokes:1; // Has this method MethodHandle invokes? + unsigned int speculatively_disconnected:1; // Marked for potential unload + void clear(); }; @@ -137,6 +139,7 @@ // To support simple linked-list chaining of nmethods: nmethod* _osr_link; // from instanceKlass::osr_nmethods_head nmethod* _scavenge_root_link; // from CodeCache::scavenge_root_nmethods + nmethod* _saved_nmethod_link; // from CodeCache::speculatively_disconnect static nmethod* volatile _oops_do_mark_nmethods; nmethod* volatile _oops_do_mark_link; @@ -413,6 +416,9 @@ bool has_method_handle_invokes() const { return flags.has_method_handle_invokes; } void set_has_method_handle_invokes(bool z) { flags.has_method_handle_invokes = z; } + bool is_speculatively_disconnected() const { return flags.speculatively_disconnected; } + void set_speculatively_disconnected(bool z) { flags.speculatively_disconnected = z; } + int level() const { return flags.level; } void set_level(int newLevel) { check_safepoint(); flags.level = newLevel; } @@ -437,6 +443,9 @@ nmethod* scavenge_root_link() const { return _scavenge_root_link; } void set_scavenge_root_link(nmethod *n) { _scavenge_root_link = n; } + nmethod* saved_nmethod_link() const { return _saved_nmethod_link; } + void set_saved_nmethod_link(nmethod *n) { _saved_nmethod_link = n; } + public: // Sweeper support diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/compiler/compileBroker.cpp --- a/hotspot/src/share/vm/compiler/compileBroker.cpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/compiler/compileBroker.cpp Fri Jan 29 09:27:22 2010 -0800 @@ -69,6 +69,7 @@ bool CompileBroker::_initialized = false; volatile bool CompileBroker::_should_block = false; +volatile jint CompileBroker::_should_compile_new_jobs = run_compilation; // The installed compiler(s) AbstractCompiler* CompileBroker::_compilers[2]; @@ -986,6 +987,13 @@ return method_code; } if (method->is_not_compilable(comp_level)) return NULL; + + nmethod* saved = CodeCache::find_and_remove_saved_code(method()); + if (saved != NULL) { + method->set_code(method, saved); + return saved; + } + } else { // osr compilation #ifndef TIERED @@ -1037,6 +1045,14 @@ method->jmethod_id(); } + // If the compiler is shut off due to code cache flushing or otherwise, + // fail out now so blocking compiles dont hang the java thread + if (!should_compile_new_jobs() || (UseCodeCacheFlushing && CodeCache::needs_flushing())) { + method->invocation_counter()->decay(); + method->backedge_counter()->decay(); + return NULL; + } + // do the compilation if (method->is_native()) { if (!PreferInterpreterNativeStubs) { @@ -1325,26 +1341,13 @@ { // We need this HandleMark to avoid leaking VM handles. HandleMark hm(thread); + if (CodeCache::unallocated_capacity() < CodeCacheMinimumFreeSpace) { - // The CodeCache is full. Print out warning and disable compilation. - UseInterpreter = true; - if (UseCompiler || AlwaysCompileLoopMethods ) { - if (log != NULL) { - log->begin_elem("code_cache_full"); - log->stamp(); - log->end_elem(); - } -#ifndef PRODUCT - warning("CodeCache is full. Compiler has been disabled"); - if (CompileTheWorld || ExitOnFullCodeCache) { - before_exit(thread); - exit_globals(); // will delete tty - vm_direct_exit(CompileTheWorld ? 0 : 1); - } -#endif - UseCompiler = false; - AlwaysCompileLoopMethods = false; - } + // the code cache is really full + handle_full_code_cache(); + } else if (UseCodeCacheFlushing && CodeCache::needs_flushing()) { + // Attempt to start cleaning the code cache while there is still a little headroom + NMethodSweeper::handle_full_code_cache(false); } CompileTask* task = queue->get(); @@ -1369,7 +1372,7 @@ // Never compile a method if breakpoints are present in it if (method()->number_of_breakpoints() == 0) { // Compile the method. - if (UseCompiler || AlwaysCompileLoopMethods) { + if ((UseCompiler || AlwaysCompileLoopMethods) && CompileBroker::should_compile_new_jobs()) { #ifdef COMPILER1 // Allow repeating compilations for the purpose of benchmarking // compile speed. This is not useful for customers. @@ -1614,6 +1617,38 @@ // ------------------------------------------------------------------ +// CompileBroker::handle_full_code_cache +// +// The CodeCache is full. Print out warning and disable compilation or +// try code cache cleaning so compilation can continue later. +void CompileBroker::handle_full_code_cache() { + UseInterpreter = true; + if (UseCompiler || AlwaysCompileLoopMethods ) { + CompilerThread* thread = CompilerThread::current(); + CompileLog* log = thread->log(); + if (log != NULL) { + log->begin_elem("code_cache_full"); + log->stamp(); + log->end_elem(); + } + #ifndef PRODUCT + warning("CodeCache is full. Compiler has been disabled"); + if (CompileTheWorld || ExitOnFullCodeCache) { + before_exit(JavaThread::current()); + exit_globals(); // will delete tty + vm_direct_exit(CompileTheWorld ? 0 : 1); + } + #endif + if (UseCodeCacheFlushing) { + NMethodSweeper::handle_full_code_cache(true); + } else { + UseCompiler = false; + AlwaysCompileLoopMethods = false; + } + } +} + +// ------------------------------------------------------------------ // CompileBroker::set_last_compile // // Record this compilation for debugging purposes. diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/compiler/compileBroker.hpp --- a/hotspot/src/share/vm/compiler/compileBroker.hpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/compiler/compileBroker.hpp Fri Jan 29 09:27:22 2010 -0800 @@ -193,6 +193,9 @@ static bool _initialized; static volatile bool _should_block; + // This flag can be used to stop compilation or turn it back on + static volatile jint _should_compile_new_jobs; + // The installed compiler(s) static AbstractCompiler* _compilers[2]; @@ -319,6 +322,7 @@ static void compiler_thread_loop(); + static uint get_compilation_id() { return _compilation_id; } static bool is_idle(); // Set _should_block. @@ -328,6 +332,20 @@ // Call this from the compiler at convenient points, to poll for _should_block. static void maybe_block(); + enum { + // Flags for toggling compiler activity + stop_compilation = 0, + run_compilation = 1 + }; + + static bool should_compile_new_jobs() { return UseCompiler && (_should_compile_new_jobs == run_compilation); } + static bool set_should_compile_new_jobs(jint new_state) { + // Return success if the current caller set it + jint old = Atomic::cmpxchg(new_state, &_should_compile_new_jobs, 1-new_state); + return (old == (1-new_state)); + } + static void handle_full_code_cache(); + // Return total compilation ticks static jlong total_compilation_ticks() { return _perf_total_compilation != NULL ? _perf_total_compilation->get_value() : 0; diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/includeDB_compiler2 --- a/hotspot/src/share/vm/includeDB_compiler2 Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/includeDB_compiler2 Fri Jan 29 09:27:22 2010 -0800 @@ -775,6 +775,7 @@ output.cpp assembler.inline.hpp output.cpp callnode.hpp output.cpp cfgnode.hpp +output.cpp compileBroker.hpp output.cpp debugInfo.hpp output.cpp debugInfoRec.hpp output.cpp handles.inline.hpp diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/includeDB_core --- a/hotspot/src/share/vm/includeDB_core Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/includeDB_core Fri Jan 29 09:27:22 2010 -0800 @@ -1032,6 +1032,7 @@ codeCache.cpp oop.inline.hpp codeCache.cpp pcDesc.hpp codeCache.cpp resourceArea.hpp +codeCache.cpp xmlstream.hpp codeCache.hpp allocation.hpp codeCache.hpp codeBlob.hpp @@ -1120,6 +1121,7 @@ compileBroker.cpp oop.inline.hpp compileBroker.cpp os.hpp compileBroker.cpp sharedRuntime.hpp +compileBroker.cpp sweeper.hpp compileBroker.cpp systemDictionary.hpp compileBroker.cpp vmSymbols.hpp @@ -3719,6 +3721,7 @@ sharedRuntime.cpp abstractCompiler.hpp sharedRuntime.cpp arguments.hpp sharedRuntime.cpp biasedLocking.hpp +sharedRuntime.cpp compileBroker.hpp sharedRuntime.cpp compiledIC.hpp sharedRuntime.cpp compilerOracle.hpp sharedRuntime.cpp copy.hpp @@ -3973,6 +3976,7 @@ sweeper.cpp atomic.hpp sweeper.cpp codeCache.hpp +sweeper.cpp compileBroker.hpp sweeper.cpp events.hpp sweeper.cpp methodOop.hpp sweeper.cpp mutexLocker.hpp @@ -3980,6 +3984,8 @@ sweeper.cpp os.hpp sweeper.cpp resourceArea.hpp sweeper.cpp sweeper.hpp +sweeper.cpp vm_operations.hpp +sweeper.cpp xmlstream.hpp symbolKlass.cpp gcLocker.hpp symbolKlass.cpp handles.inline.hpp @@ -4633,6 +4639,7 @@ vm_operations.cpp interfaceSupport.hpp vm_operations.cpp isGCActiveMark.hpp vm_operations.cpp resourceArea.hpp +vm_operations.cpp sweeper.hpp vm_operations.cpp threadService.hpp vm_operations.cpp thread_.inline.hpp vm_operations.cpp vmSymbols.hpp diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/oops/methodOop.cpp --- a/hotspot/src/share/vm/oops/methodOop.cpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/oops/methodOop.cpp Fri Jan 29 09:27:22 2010 -0800 @@ -705,6 +705,16 @@ // This function must not hit a safepoint! address methodOopDesc::verified_code_entry() { debug_only(No_Safepoint_Verifier nsv;) + nmethod *code = (nmethod *)OrderAccess::load_ptr_acquire(&_code); + if (code == NULL && UseCodeCacheFlushing) { + nmethod *saved_code = CodeCache::find_and_remove_saved_code(this); + if (saved_code != NULL) { + methodHandle method(this); + assert( ! saved_code->is_osr_method(), "should not get here for osr" ); + set_code( method, saved_code ); + } + } + assert(_from_compiled_entry != NULL, "must be set"); return _from_compiled_entry; } @@ -733,8 +743,8 @@ int comp_level = code->comp_level(); // In theory there could be a race here. In practice it is unlikely // and not worth worrying about. - if (comp_level > highest_tier_compile()) { - set_highest_tier_compile(comp_level); + if (comp_level > mh->highest_tier_compile()) { + mh->set_highest_tier_compile(comp_level); } OrderAccess::storestore(); diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/oops/methodOop.hpp --- a/hotspot/src/share/vm/oops/methodOop.hpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/oops/methodOop.hpp Fri Jan 29 09:27:22 2010 -0800 @@ -303,7 +303,7 @@ bool check_code() const; // Not inline to avoid circular ref nmethod* volatile code() const { assert( check_code(), "" ); return (nmethod *)OrderAccess::load_ptr_acquire(&_code); } void clear_code(); // Clear out any compiled code - void set_code(methodHandle mh, nmethod* code); + static void set_code(methodHandle mh, nmethod* code); void set_adapter_entry(AdapterHandlerEntry* adapter) { _adapter = adapter; } address get_i2c_entry(); address get_c2i_entry(); diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/opto/output.cpp --- a/hotspot/src/share/vm/opto/output.cpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/opto/output.cpp Fri Jan 29 09:27:22 2010 -0800 @@ -1093,7 +1093,7 @@ cb->initialize(total_req, locs_req); // Have we run out of code space? - if (cb->blob() == NULL) { + if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) { turn_off_compiler(this); return; } @@ -1314,7 +1314,7 @@ // Verify that there is sufficient space remaining cb->insts()->maybe_expand_to_ensure_remaining(MAX_inst_size); - if (cb->blob() == NULL) { + if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) { turn_off_compiler(this); return; } @@ -1433,7 +1433,7 @@ } // One last check for failed CodeBuffer::expand: - if (cb->blob() == NULL) { + if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) { turn_off_compiler(this); return; } diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/runtime/compilationPolicy.cpp --- a/hotspot/src/share/vm/runtime/compilationPolicy.cpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/runtime/compilationPolicy.cpp Fri Jan 29 09:27:22 2010 -0800 @@ -66,7 +66,7 @@ if (!canBeCompiled(m)) return false; return !UseInterpreter || // must compile all methods - (UseCompiler && AlwaysCompileLoopMethods && m->has_loops()); // eagerly compile loop methods + (UseCompiler && AlwaysCompileLoopMethods && m->has_loops() && CompileBroker::should_compile_new_jobs()); // eagerly compile loop methods } // Returns true if m is allowed to be compiled @@ -137,7 +137,7 @@ reset_counter_for_invocation_event(m); const char* comment = "count"; - if (!delayCompilationDuringStartup() && canBeCompiled(m) && UseCompiler) { + if (!delayCompilationDuringStartup() && canBeCompiled(m) && UseCompiler && CompileBroker::should_compile_new_jobs()) { nmethod* nm = m->code(); if (nm == NULL ) { const char* comment = "count"; @@ -162,7 +162,7 @@ int hot_count = m->backedge_count(); const char* comment = "backedge_count"; - if (!m->is_not_osr_compilable() && !delayCompilationDuringStartup() && canBeCompiled(m)) { + if (!m->is_not_osr_compilable() && !delayCompilationDuringStartup() && canBeCompiled(m) && CompileBroker::should_compile_new_jobs()) { CompileBroker::compile_method(m, loop_top_bci, m, hot_count, comment, CHECK); NOT_PRODUCT(trace_osr_completion(m->lookup_osr_nmethod_for(loop_top_bci));) @@ -204,7 +204,7 @@ reset_counter_for_invocation_event(m); const char* comment = "count"; - if (m->code() == NULL && !delayCompilationDuringStartup() && canBeCompiled(m) && UseCompiler) { + if (m->code() == NULL && !delayCompilationDuringStartup() && canBeCompiled(m) && UseCompiler && CompileBroker::should_compile_new_jobs()) { ResourceMark rm(THREAD); JavaThread *thread = (JavaThread*)THREAD; frame fr = thread->last_frame(); @@ -248,7 +248,7 @@ int hot_count = m->backedge_count(); const char* comment = "backedge_count"; - if (!m->is_not_osr_compilable() && !delayCompilationDuringStartup() && canBeCompiled(m)) { + if (!m->is_not_osr_compilable() && !delayCompilationDuringStartup() && canBeCompiled(m) && CompileBroker::should_compile_new_jobs()) { CompileBroker::compile_method(m, loop_top_bci, m, hot_count, comment, CHECK); NOT_PRODUCT(trace_osr_completion(m->lookup_osr_nmethod_for(loop_top_bci));) diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/runtime/globals.hpp --- a/hotspot/src/share/vm/runtime/globals.hpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/runtime/globals.hpp Fri Jan 29 09:27:22 2010 -0800 @@ -3117,6 +3117,15 @@ notproduct(bool, ExitOnFullCodeCache, false, \ "Exit the VM if we fill the code cache.") \ \ + product(bool, UseCodeCacheFlushing, false, \ + "Attempt to clean the code cache before shutting off compiler") \ + \ + product(intx, MinCodeCacheFlushingInterval, 30, \ + "Min number of seconds between code cache cleaning sessions") \ + \ + product(uintx, CodeCacheFlushingMinimumFreeSpace, 1500*K, \ + "When less than X space left, start code cache cleaning") \ + \ /* interpreter debugging */ \ develop(intx, BinarySwitchThreshold, 5, \ "Minimal number of lookupswitch entries for rewriting to binary " \ diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/runtime/sharedRuntime.cpp --- a/hotspot/src/share/vm/runtime/sharedRuntime.cpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/runtime/sharedRuntime.cpp Fri Jan 29 09:27:22 2010 -0800 @@ -2146,19 +2146,8 @@ // CodeCache is full, disable compilation // Ought to log this but compile log is only per compile thread // and we're some non descript Java thread. - UseInterpreter = true; - if (UseCompiler || AlwaysCompileLoopMethods ) { -#ifndef PRODUCT - warning("CodeCache is full. Compiler has been disabled"); - if (CompileTheWorld || ExitOnFullCodeCache) { - before_exit(JavaThread::current()); - exit_globals(); // will delete tty - vm_direct_exit(CompileTheWorld ? 0 : 1); - } -#endif - UseCompiler = false; - AlwaysCompileLoopMethods = false; - } + MutexUnlocker mu(AdapterHandlerLibrary_lock); + CompileBroker::handle_full_code_cache(); return NULL; // Out of CodeCache space } entry->relocate(B->instructions_begin()); @@ -2282,19 +2271,8 @@ // CodeCache is full, disable compilation // Ought to log this but compile log is only per compile thread // and we're some non descript Java thread. - UseInterpreter = true; - if (UseCompiler || AlwaysCompileLoopMethods ) { -#ifndef PRODUCT - warning("CodeCache is full. Compiler has been disabled"); - if (CompileTheWorld || ExitOnFullCodeCache) { - before_exit(JavaThread::current()); - exit_globals(); // will delete tty - vm_direct_exit(CompileTheWorld ? 0 : 1); - } -#endif - UseCompiler = false; - AlwaysCompileLoopMethods = false; - } + MutexUnlocker mu(AdapterHandlerLibrary_lock); + CompileBroker::handle_full_code_cache(); } return nm; } diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/runtime/sweeper.cpp --- a/hotspot/src/share/vm/runtime/sweeper.cpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/runtime/sweeper.cpp Fri Jan 29 09:27:22 2010 -0800 @@ -33,6 +33,11 @@ jint NMethodSweeper::_locked_seen = 0; jint NMethodSweeper::_not_entrant_seen_on_stack = 0; bool NMethodSweeper::_rescan = false; +bool NMethodSweeper::_was_full = false; +jint NMethodSweeper::_advise_to_sweep = 0; +jlong NMethodSweeper::_last_was_full = 0; +uint NMethodSweeper::_highest_marked = 0; +long NMethodSweeper::_was_full_traversal = 0; class MarkActivationClosure: public CodeBlobClosure { public: @@ -114,6 +119,40 @@ tty->print_cr("### Couldn't make progress on some nmethods so stopping sweep"); } } + + if (UseCodeCacheFlushing) { + if (!CodeCache::needs_flushing()) { + // In a safepoint, no race with setters + _advise_to_sweep = 0; + } + + if (was_full()) { + // There was some progress so attempt to restart the compiler + jlong now = os::javaTimeMillis(); + jlong max_interval = (jlong)MinCodeCacheFlushingInterval * (jlong)1000; + jlong curr_interval = now - _last_was_full; + if ((!CodeCache::needs_flushing()) && (curr_interval > max_interval)) { + CompileBroker::set_should_compile_new_jobs(CompileBroker::run_compilation); + set_was_full(false); + + // Update the _last_was_full time so we can tell how fast the + // code cache is filling up + _last_was_full = os::javaTimeMillis(); + + if (PrintMethodFlushing) { + tty->print_cr("### sweeper: Live blobs:" UINT32_FORMAT "/Free code cache:" SIZE_FORMAT " bytes, restarting compiler", + CodeCache::nof_blobs(), CodeCache::unallocated_capacity()); + } + if (LogCompilation && (xtty != NULL)) { + ttyLocker ttyl; + xtty->begin_elem("restart_compiler live_blobs='" UINT32_FORMAT "' free_code_cache='" SIZE_FORMAT "'", + CodeCache::nof_blobs(), CodeCache::unallocated_capacity()); + xtty->stamp(); + xtty->end_elem(); + } + } + } + } } @@ -137,12 +176,12 @@ if (nm->is_marked_for_reclamation()) { assert(!nm->is_locked_by_vm(), "must not flush locked nmethods"); if (PrintMethodFlushing && Verbose) { - tty->print_cr("### Nmethod 0x%x (marked for reclamation) being flushed", nm); + tty->print_cr("### Nmethod %3d/" PTR_FORMAT " (marked for reclamation) being flushed", nm->compile_id(), nm); } nm->flush(); } else { if (PrintMethodFlushing && Verbose) { - tty->print_cr("### Nmethod 0x%x (zombie) being marked for reclamation", nm); + tty->print_cr("### Nmethod %3d/" PTR_FORMAT " (zombie) being marked for reclamation", nm->compile_id(), nm); } nm->mark_for_reclamation(); _rescan = true; @@ -152,7 +191,7 @@ // stack we can safely convert it to a zombie method if (nm->can_not_entrant_be_converted()) { if (PrintMethodFlushing && Verbose) { - tty->print_cr("### Nmethod 0x%x (not entrant) being made zombie", nm); + tty->print_cr("### Nmethod %3d/" PTR_FORMAT " (not entrant) being made zombie", nm->compile_id(), nm); } nm->make_zombie(); _rescan = true; @@ -167,7 +206,7 @@ } else if (nm->is_unloaded()) { // Unloaded code, just make it a zombie if (PrintMethodFlushing && Verbose) - tty->print_cr("### Nmethod 0x%x (unloaded) being made zombie", nm); + tty->print_cr("### Nmethod %3d/" PTR_FORMAT " (unloaded) being made zombie", nm->compile_id(), nm); if (nm->is_osr_method()) { // No inline caches will ever point to osr methods, so we can just remove it nm->flush(); @@ -177,7 +216,167 @@ } } else { assert(nm->is_alive(), "should be alive"); + + if (UseCodeCacheFlushing) { + if ((nm->method()->code() != nm) && !(nm->is_locked_by_vm()) && !(nm->is_osr_method()) && + (_traversals > _was_full_traversal+2) && (((uint)nm->compile_id()) < _highest_marked) && + CodeCache::needs_flushing()) { + // This method has not been called since the forced cleanup happened + nm->make_not_entrant(); + } + } + // Clean-up all inline caches that points to zombie/non-reentrant methods nm->cleanup_inline_caches(); } } + +// Code cache unloading: when compilers notice the code cache is getting full, +// they will call a vm op that comes here. This code attempts to speculatively +// unload the oldest half of the nmethods (based on the compile job id) by +// saving the old code in a list in the CodeCache. Then +// execution resumes. If a method so marked is not called by the second +// safepoint from the current one, the nmethod will be marked non-entrant and +// got rid of by normal sweeping. If the method is called, the methodOop's +// _code field is restored and the methodOop/nmethod +// go back to their normal state. +void NMethodSweeper::handle_full_code_cache(bool is_full) { + // Only the first one to notice can advise us to start early cleaning + if (!is_full){ + jint old = Atomic::cmpxchg( 1, &_advise_to_sweep, 0 ); + if (old != 0) { + return; + } + } + + if (is_full) { + // Since code cache is full, immediately stop new compiles + bool did_set = CompileBroker::set_should_compile_new_jobs(CompileBroker::stop_compilation); + if (!did_set) { + // only the first to notice can start the cleaning, + // others will go back and block + return; + } + set_was_full(true); + + // If we run out within MinCodeCacheFlushingInterval of the last unload time, give up + jlong now = os::javaTimeMillis(); + jlong max_interval = (jlong)MinCodeCacheFlushingInterval * (jlong)1000; + jlong curr_interval = now - _last_was_full; + if (curr_interval < max_interval) { + _rescan = true; + if (PrintMethodFlushing) { + tty->print_cr("### handle full too often, turning off compiler"); + } + if (LogCompilation && (xtty != NULL)) { + ttyLocker ttyl; + xtty->begin_elem("disable_compiler flushing_interval='" UINT64_FORMAT "' live_blobs='" UINT32_FORMAT "' free_code_cache='" SIZE_FORMAT "'", + curr_interval/1000, CodeCache::nof_blobs(), CodeCache::unallocated_capacity()); + xtty->stamp(); + xtty->end_elem(); + } + return; + } + } + + VM_HandleFullCodeCache op(is_full); + VMThread::execute(&op); + + // rescan again as soon as possible + _rescan = true; +} + +void NMethodSweeper::speculative_disconnect_nmethods(bool is_full) { + // If there was a race in detecting full code cache, only run + // one vm op for it or keep the compiler shut off + + debug_only(jlong start = os::javaTimeMillis();) + + if ((!was_full()) && (is_full)) { + if (!CodeCache::needs_flushing()) { + if (PrintMethodFlushing) { + tty->print_cr("### sweeper: Live blobs:" UINT32_FORMAT "/Free code cache:" SIZE_FORMAT " bytes, restarting compiler", + CodeCache::nof_blobs(), CodeCache::unallocated_capacity()); + } + if (LogCompilation && (xtty != NULL)) { + ttyLocker ttyl; + xtty->begin_elem("restart_compiler live_blobs='" UINT32_FORMAT "' free_code_cache='" SIZE_FORMAT "'", + CodeCache::nof_blobs(), CodeCache::unallocated_capacity()); + xtty->stamp(); + xtty->end_elem(); + } + CompileBroker::set_should_compile_new_jobs(CompileBroker::run_compilation); + return; + } + } + + // Traverse the code cache trying to dump the oldest nmethods + uint curr_max_comp_id = CompileBroker::get_compilation_id(); + uint flush_target = ((curr_max_comp_id - _highest_marked) >> 1) + _highest_marked; + if (PrintMethodFlushing && Verbose) { + tty->print_cr("### Cleaning code cache: Live blobs:" UINT32_FORMAT "/Free code cache:" SIZE_FORMAT " bytes", + CodeCache::nof_blobs(), CodeCache::unallocated_capacity()); + } + if (LogCompilation && (xtty != NULL)) { + ttyLocker ttyl; + xtty->begin_elem("start_cleaning_code_cache live_blobs='" UINT32_FORMAT "' free_code_cache='" SIZE_FORMAT "'", + CodeCache::nof_blobs(), CodeCache::unallocated_capacity()); + xtty->stamp(); + xtty->end_elem(); + } + + nmethod* nm = CodeCache::alive_nmethod(CodeCache::first()); + jint disconnected = 0; + jint made_not_entrant = 0; + while ((nm != NULL)){ + uint curr_comp_id = nm->compile_id(); + + // OSR methods cannot be flushed like this. Also, don't flush native methods + // since they are part of the JDK in most cases + if (nm->is_in_use() && (!nm->is_osr_method()) && (!nm->is_locked_by_vm()) && + (!nm->is_native_method()) && ((curr_comp_id < flush_target))) { + + if ((nm->method()->code() == nm)) { + // This method has not been previously considered for + // unloading or it was restored already + CodeCache::speculatively_disconnect(nm); + disconnected++; + } else if (nm->is_speculatively_disconnected()) { + // This method was previously considered for preemptive unloading and was not called since then + nm->method()->invocation_counter()->decay(); + nm->method()->backedge_counter()->decay(); + nm->make_not_entrant(); + made_not_entrant++; + } + + if (curr_comp_id > _highest_marked) { + _highest_marked = curr_comp_id; + } + } + nm = CodeCache::alive_nmethod(CodeCache::next(nm)); + } + + if (LogCompilation && (xtty != NULL)) { + ttyLocker ttyl; + xtty->begin_elem("stop_cleaning_code_cache disconnected='" UINT32_FORMAT "' made_not_entrant='" UINT32_FORMAT "' live_blobs='" UINT32_FORMAT "' free_code_cache='" SIZE_FORMAT "'", + disconnected, made_not_entrant, CodeCache::nof_blobs(), CodeCache::unallocated_capacity()); + xtty->stamp(); + xtty->end_elem(); + } + + // Shut off compiler. Sweeper will run exiting from this safepoint + // and turn it back on if it clears enough space + if (was_full()) { + _last_was_full = os::javaTimeMillis(); + CompileBroker::set_should_compile_new_jobs(CompileBroker::stop_compilation); + } + + // After two more traversals the sweeper will get rid of unrestored nmethods + _was_full_traversal = _traversals; +#ifdef ASSERT + jlong end = os::javaTimeMillis(); + if(PrintMethodFlushing && Verbose) { + tty->print_cr("### sweeper: unload time: " INT64_FORMAT, end-start); + } +#endif +} diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/runtime/sweeper.hpp --- a/hotspot/src/share/vm/runtime/sweeper.hpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/runtime/sweeper.hpp Fri Jan 29 09:27:22 2010 -0800 @@ -38,6 +38,11 @@ static int _locked_seen; // Number of locked nmethods encountered during the scan static int _not_entrant_seen_on_stack; // Number of not entrant nmethod were are still on stack + static bool _was_full; // remember if we did emergency unloading + static jint _advise_to_sweep; // flag to indicate code cache getting full + static jlong _last_was_full; // timestamp of last emergency unloading + static uint _highest_marked; // highest compile id dumped at last emergency unloading + static long _was_full_traversal; // trav number at last emergency unloading static void process_nmethod(nmethod *nm); public: @@ -51,4 +56,10 @@ // changes to false at safepoint so we can never overwrite it with false. _rescan = true; } + + static void handle_full_code_cache(bool is_full); // Called by compilers who fail to allocate + static void speculative_disconnect_nmethods(bool was_full); // Called by vm op to deal with alloc failure + + static void set_was_full(bool state) { _was_full = state; } + static bool was_full() { return _was_full; } }; diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/runtime/vm_operations.cpp --- a/hotspot/src/share/vm/runtime/vm_operations.cpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/runtime/vm_operations.cpp Fri Jan 29 09:27:22 2010 -0800 @@ -151,6 +151,10 @@ #endif // !PRODUCT +void VM_HandleFullCodeCache::doit() { + NMethodSweeper::speculative_disconnect_nmethods(_is_full); +} + void VM_Verify::doit() { Universe::verify(); } diff -r f26b30116e3a -r 71fd601907dc hotspot/src/share/vm/runtime/vm_operations.hpp --- a/hotspot/src/share/vm/runtime/vm_operations.hpp Fri Jan 29 08:33:24 2010 -0800 +++ b/hotspot/src/share/vm/runtime/vm_operations.hpp Fri Jan 29 09:27:22 2010 -0800 @@ -41,6 +41,7 @@ template(DeoptimizeFrame) \ template(DeoptimizeAll) \ template(ZombieAll) \ + template(HandleFullCodeCache) \ template(Verify) \ template(PrintJNI) \ template(HeapDumper) \ @@ -241,6 +242,16 @@ bool allow_nested_vm_operations() const { return true; } }; +class VM_HandleFullCodeCache: public VM_Operation { + private: + bool _is_full; + public: + VM_HandleFullCodeCache(bool is_full) { _is_full = is_full; } + VMOp_Type type() const { return VMOp_HandleFullCodeCache; } + void doit(); + bool allow_nested_vm_operations() const { return true; } +}; + #ifndef PRODUCT class VM_DeoptimizeAll: public VM_Operation { private: