--- a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp Wed Nov 27 06:36:41 2019 -0800
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp Wed Nov 27 11:52:57 2019 -0500
@@ -23,9 +23,11 @@
#include "precompiled.hpp"
#include "code/codeCache.hpp"
+#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
+#include "gc/shenandoah/shenandoahCodeRoots.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
-#include "gc/shenandoah/shenandoahCodeRoots.hpp"
+#include "gc/shenandoah/shenandoahNMethod.inline.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
@@ -98,69 +100,45 @@
_finished = true;
}
-class ShenandoahNMethodOopDetector : public OopClosure {
-private:
- ResourceMark rm; // For growable array allocation below.
- GrowableArray<oop*> _oops;
-
-public:
- ShenandoahNMethodOopDetector() : _oops(10) {};
-
- void do_oop(oop* o) {
- _oops.append(o);
- }
- void do_oop(narrowOop* o) {
- fatal("NMethods should not have compressed oops embedded.");
- }
-
- GrowableArray<oop*>* oops() {
- return &_oops;
- }
-
- bool has_oops() {
- return !_oops.is_empty();
- }
-};
-
-GrowableArray<ShenandoahNMethod*>* ShenandoahCodeRoots::_recorded_nms;
-ShenandoahLock ShenandoahCodeRoots::_recorded_nms_lock;
+ShenandoahNMethodTable* ShenandoahCodeRoots::_nmethod_table;
+int ShenandoahCodeRoots::_disarmed_value = 1;
void ShenandoahCodeRoots::initialize() {
- _recorded_nms = new (ResourceObj::C_HEAP, mtGC) GrowableArray<ShenandoahNMethod*>(100, true, mtGC);
+ _nmethod_table = new ShenandoahNMethodTable();
}
-void ShenandoahCodeRoots::add_nmethod(nmethod* nm) {
+void ShenandoahCodeRoots::register_nmethod(nmethod* nm) {
switch (ShenandoahCodeRootsStyle) {
case 0:
case 1:
break;
case 2: {
assert_locked_or_safepoint(CodeCache_lock);
- ShenandoahLocker locker(CodeCache_lock->owned_by_self() ? NULL : &_recorded_nms_lock);
-
- ShenandoahNMethodOopDetector detector;
- nm->oops_do(&detector);
-
- if (detector.has_oops()) {
- ShenandoahNMethod* nmr = new ShenandoahNMethod(nm, detector.oops());
- nmr->assert_alive_and_correct();
- int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
- if (idx != -1) {
- ShenandoahNMethod* old = _recorded_nms->at(idx);
- _recorded_nms->at_put(idx, nmr);
- delete old;
- } else {
- _recorded_nms->append(nmr);
- }
- }
+ _nmethod_table->register_nmethod(nm);
break;
}
default:
ShouldNotReachHere();
}
-};
+}
-void ShenandoahCodeRoots::remove_nmethod(nmethod* nm) {
+void ShenandoahCodeRoots::unregister_nmethod(nmethod* nm) {
+ switch (ShenandoahCodeRootsStyle) {
+ case 0:
+ case 1: {
+ break;
+ }
+ case 2: {
+ assert_locked_or_safepoint(CodeCache_lock);
+ _nmethod_table->unregister_nmethod(nm);
+ break;
+ }
+ default:
+ ShouldNotReachHere();
+ }
+}
+
+void ShenandoahCodeRoots::flush_nmethod(nmethod* nm) {
switch (ShenandoahCodeRootsStyle) {
case 0:
case 1: {
@@ -168,19 +146,7 @@
}
case 2: {
assert_locked_or_safepoint(CodeCache_lock);
- ShenandoahLocker locker(CodeCache_lock->owned_by_self() ? NULL : &_recorded_nms_lock);
-
- ShenandoahNMethodOopDetector detector;
- nm->oops_do(&detector, /* allow_dead = */ true);
-
- if (detector.has_oops()) {
- int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);
- assert(idx != -1, "nmethod " PTR_FORMAT " should be registered", p2i(nm));
- ShenandoahNMethod* old = _recorded_nms->at(idx);
- old->assert_same_oops(detector.oops());
- _recorded_nms->delete_at(idx);
- delete old;
- }
+ _nmethod_table->flush_nmethod(nm);
break;
}
default:
@@ -188,10 +154,186 @@
}
}
+void ShenandoahCodeRoots::prepare_concurrent_unloading() {
+ assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
+ _disarmed_value ++;
+ // 0 is reserved for new nmethod
+ if (_disarmed_value == 0) {
+ _disarmed_value = 1;
+ }
+
+ JavaThreadIteratorWithHandle jtiwh;
+ for (JavaThread *thr = jtiwh.next(); thr != NULL; thr = jtiwh.next()) {
+ ShenandoahThreadLocalData::set_disarmed_value(thr, _disarmed_value);
+ }
+}
+
+class ShenandoahNMethodUnlinkClosure : public NMethodClosure {
+private:
+ bool _unloading_occurred;
+ volatile bool _failed;
+ ShenandoahHeap* _heap;
+
+ void set_failed() {
+ Atomic::store(&_failed, true);
+ }
+
+ void unlink(nmethod* nm) {
+ // Unlinking of the dependencies must happen before the
+ // handshake separating unlink and purge.
+ nm->flush_dependencies(false /* delete_immediately */);
+
+ // unlink_from_method will take the CompiledMethod_lock.
+ // In this case we don't strictly need it when unlinking nmethods from
+ // the Method, because it is only concurrently unlinked by
+ // the entry barrier, which acquires the per nmethod lock.
+ nm->unlink_from_method();
+
+ if (nm->is_osr_method()) {
+ // Invalidate the osr nmethod only once
+ nm->invalidate_osr_method();
+ }
+ }
+public:
+ ShenandoahNMethodUnlinkClosure(bool unloading_occurred) :
+ _unloading_occurred(unloading_occurred),
+ _failed(false),
+ _heap(ShenandoahHeap::heap()) {}
+
+ virtual void do_nmethod(nmethod* nm) {
+ if (failed()) {
+ return;
+ }
+
+ ShenandoahNMethod* nm_data = ShenandoahNMethod::gc_data(nm);
+ assert(!nm_data->is_unregistered(), "Should not see unregistered entry");
+
+ if (!nm->is_alive()) {
+ return;
+ }
+
+ if (nm->is_unloading()) {
+ ShenandoahReentrantLocker locker(nm_data->lock());
+ unlink(nm);
+ return;
+ }
+
+ ShenandoahReentrantLocker locker(nm_data->lock());
+
+ // Heal oops and disarm
+ ShenandoahEvacOOMScope scope;
+ ShenandoahNMethod::heal_nmethod(nm);
+ ShenandoahNMethod::disarm_nmethod(nm);
+
+ // Clear compiled ICs and exception caches
+ if (!nm->unload_nmethod_caches(_unloading_occurred)) {
+ set_failed();
+ }
+ }
+
+ bool failed() const {
+ return Atomic::load(&_failed);
+ }
+};
+
+class ShenandoahUnlinkTask : public AbstractGangTask {
+private:
+ ShenandoahNMethodUnlinkClosure _cl;
+ ICRefillVerifier* _verifier;
+ ShenandoahConcurrentNMethodIterator _iterator;
+
+public:
+ ShenandoahUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier) :
+ AbstractGangTask("ShenandoahNMethodUnlinkTask"),
+ _cl(unloading_occurred),
+ _verifier(verifier),
+ _iterator(ShenandoahCodeRoots::table()) {
+ MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
+ _iterator.nmethods_do_begin();
+ }
+
+ ~ShenandoahUnlinkTask() {
+ MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
+ _iterator.nmethods_do_end();
+ }
+
+ virtual void work(uint worker_id) {
+ ICRefillVerifierMark mark(_verifier);
+ _iterator.nmethods_do(&_cl);
+ }
+
+ bool success() const {
+ return !_cl.failed();
+ }
+};
+
+void ShenandoahCodeRoots::unlink(WorkGang* workers, bool unloading_occurred) {
+ assert(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(),
+ "Only when running concurrent class unloading");
+
+ for (;;) {
+ ICRefillVerifier verifier;
+
+ {
+ ShenandoahUnlinkTask task(unloading_occurred, &verifier);
+ workers->run_task(&task);
+ if (task.success()) {
+ return;
+ }
+ }
+
+ // Cleaning failed because we ran out of transitional IC stubs,
+ // so we have to refill and try again. Refilling requires taking
+ // a safepoint, so we temporarily leave the suspendible thread set.
+ SuspendibleThreadSetLeaver sts;
+ InlineCacheBuffer::refill_ic_stubs();
+ }
+}
+
+class ShenandoahNMethodPurgeClosure : public NMethodClosure {
+public:
+ virtual void do_nmethod(nmethod* nm) {
+ if (nm->is_alive() && nm->is_unloading()) {
+ nm->make_unloaded();
+ }
+ }
+};
+
+class ShenandoahNMethodPurgeTask : public AbstractGangTask {
+private:
+ ShenandoahNMethodPurgeClosure _cl;
+ ShenandoahConcurrentNMethodIterator _iterator;
+
+public:
+ ShenandoahNMethodPurgeTask() :
+ AbstractGangTask("ShenandoahNMethodPurgeTask"),
+ _cl(),
+ _iterator(ShenandoahCodeRoots::table()) {
+ MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
+ _iterator.nmethods_do_begin();
+ }
+
+ ~ShenandoahNMethodPurgeTask() {
+ MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
+ _iterator.nmethods_do_end();
+ }
+
+ virtual void work(uint worker_id) {
+ _iterator.nmethods_do(&_cl);
+ }
+};
+
+void ShenandoahCodeRoots::purge(WorkGang* workers) {
+ assert(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(),
+ "Only when running concurrent class unloading");
+
+ ShenandoahNMethodPurgeTask task;
+ workers->run_task(&task);
+}
+
ShenandoahCodeRootsIterator::ShenandoahCodeRootsIterator() :
- _heap(ShenandoahHeap::heap()),
_par_iterator(CodeCache::heaps()),
- _claimed(0) {
+ _table_snapshot(NULL) {
assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
assert(!Thread::current()->is_Worker_thread(), "Should not be acquired by workers");
switch (ShenandoahCodeRootsStyle) {
@@ -202,6 +344,7 @@
}
case 2: {
CodeCache_lock->lock_without_safepoint_check();
+ _table_snapshot = ShenandoahCodeRoots::table()->snapshot_for_iteration();
break;
}
default:
@@ -217,6 +360,8 @@
break;
}
case 2: {
+ ShenandoahCodeRoots::table()->finish_iteration(_table_snapshot);
+ _table_snapshot = NULL;
CodeCache_lock->unlock();
break;
}
@@ -258,77 +403,7 @@
template <bool CSET_FILTER>
void ShenandoahCodeRootsIterator::fast_parallel_blobs_do(CodeBlobClosure *f) {
assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint");
-
- size_t stride = 256; // educated guess
-
- GrowableArray<ShenandoahNMethod*>* list = ShenandoahCodeRoots::_recorded_nms;
-
- size_t max = (size_t)list->length();
- while (_claimed < max) {
- size_t cur = Atomic::add(&_claimed, stride) - stride;
- size_t start = cur;
- size_t end = MIN2(cur + stride, max);
- if (start >= max) break;
-
- for (size_t idx = start; idx < end; idx++) {
- ShenandoahNMethod* nmr = list->at((int) idx);
- nmr->assert_alive_and_correct();
-
- if (CSET_FILTER && !nmr->has_cset_oops(_heap)) {
- continue;
- }
-
- f->do_code_blob(nmr->nm());
- }
- }
-}
-
-ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>* oops) {
- _nm = nm;
- _oops = NEW_C_HEAP_ARRAY(oop*, oops->length(), mtGC);
- _oops_count = oops->length();
- for (int c = 0; c < _oops_count; c++) {
- _oops[c] = oops->at(c);
- }
+ assert(_table_snapshot != NULL, "Sanity");
+ _table_snapshot->parallel_blobs_do<CSET_FILTER>(f);
}
-ShenandoahNMethod::~ShenandoahNMethod() {
- if (_oops != NULL) {
- FREE_C_HEAP_ARRAY(oop*, _oops);
- }
-}
-
-bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) {
- for (int c = 0; c < _oops_count; c++) {
- oop o = RawAccess<>::oop_load(_oops[c]);
- if (heap->in_collection_set(o)) {
- return true;
- }
- }
- return false;
-}
-
-#ifdef ASSERT
-void ShenandoahNMethod::assert_alive_and_correct() {
- assert(_nm->is_alive(), "only alive nmethods here");
- assert(_oops_count > 0, "should have filtered nmethods without oops before");
- ShenandoahHeap* heap = ShenandoahHeap::heap();
- for (int c = 0; c < _oops_count; c++) {
- oop *loc = _oops[c];
- assert(_nm->code_contains((address) loc) || _nm->oops_contains(loc), "nmethod should contain the oop*");
- oop o = RawAccess<>::oop_load(loc);
- shenandoah_assert_correct_except(loc, o,
- o == NULL ||
- heap->is_full_gc_move_in_progress() ||
- (VMThread::vm_operation() != NULL) && (VMThread::vm_operation()->type() == VM_Operation::VMOp_HeapWalkOperation)
- );
- }
-}
-
-void ShenandoahNMethod::assert_same_oops(GrowableArray<oop*>* oops) {
- assert(_oops_count == oops->length(), "should have the same number of oop*");
- for (int c = 0; c < _oops_count; c++) {
- assert(_oops[c] == oops->at(c), "should be the same oop*");
- }
-}
-#endif