28 #include "gc/shenandoah/shenandoahCodeRoots.hpp"

    30 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"

   101 class ShenandoahNMethodOopDetector : public OopClosure {

   103 ShenandoahNMethodTable* ShenandoahCodeRoots::_nmethod_table;

   102 private:

   104 int ShenandoahCodeRoots::_disarmed_value = 1;

   103   ResourceMark rm; // For growable array allocation below.

   104   GrowableArray<oop*> _oops;

   105 

   106 public:

   107   ShenandoahNMethodOopDetector() : _oops(10) {};

   108 

   109   void do_oop(oop* o) {

   110     _oops.append(o);

   111   }

   112   void do_oop(narrowOop* o) {

   113     fatal("NMethods should not have compressed oops embedded.");

   114   }

   115 

   116   GrowableArray<oop*>* oops() {

   117     return &_oops;

   118   }

   119 

   120   bool has_oops() {

   121     return !_oops.is_empty();

   122   }

   123 };

   124 

   125 GrowableArray<ShenandoahNMethod*>* ShenandoahCodeRoots::_recorded_nms;

   126 ShenandoahLock                     ShenandoahCodeRoots::_recorded_nms_lock;

   129   _recorded_nms = new (ResourceObj::C_HEAP, mtGC) GrowableArray<ShenandoahNMethod*>(100, true, mtGC);

   107   _nmethod_table = new ShenandoahNMethodTable();

   130 }

   108 }

   131 

   109 

   132 void ShenandoahCodeRoots::add_nmethod(nmethod* nm) {

   110 void ShenandoahCodeRoots::register_nmethod(nmethod* nm) {

   139       ShenandoahLocker locker(CodeCache_lock->owned_by_self() ? NULL : &_recorded_nms_lock);

   117       _nmethod_table->register_nmethod(nm);

   140 

   118       break;

   141       ShenandoahNMethodOopDetector detector;

   119     }

   142       nm->oops_do(&detector);

   120     default:

   143 

   121       ShouldNotReachHere();

   144       if (detector.has_oops()) {

   122   }

   145         ShenandoahNMethod* nmr = new ShenandoahNMethod(nm, detector.oops());

   123 }

   146         nmr->assert_alive_and_correct();

   124 

   147         int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);

   125 void ShenandoahCodeRoots::unregister_nmethod(nmethod* nm) {

   148         if (idx != -1) {

   149           ShenandoahNMethod* old = _recorded_nms->at(idx);

   150           _recorded_nms->at_put(idx, nmr);

   151           delete old;

   152         } else {

   153           _recorded_nms->append(nmr);

   154         }

   155       }

   156       break;

   157     }

   158     default:

   159       ShouldNotReachHere();

   160   }

   161 };

   162 

   163 void ShenandoahCodeRoots::remove_nmethod(nmethod* nm) {

   171       ShenandoahLocker locker(CodeCache_lock->owned_by_self() ? NULL : &_recorded_nms_lock);

   133       _nmethod_table->unregister_nmethod(nm);

   172 

   134       break;

   173       ShenandoahNMethodOopDetector detector;

   135     }

   174       nm->oops_do(&detector, /* allow_dead = */ true);

   136     default:

   175 

   137       ShouldNotReachHere();

   176       if (detector.has_oops()) {

   138   }

   177         int idx = _recorded_nms->find(nm, ShenandoahNMethod::find_with_nmethod);

   139 }

   178         assert(idx != -1, "nmethod " PTR_FORMAT " should be registered", p2i(nm));

   140 

   179         ShenandoahNMethod* old = _recorded_nms->at(idx);

   141 void ShenandoahCodeRoots::flush_nmethod(nmethod* nm) {

   180         old->assert_same_oops(detector.oops());

   142   switch (ShenandoahCodeRootsStyle) {

   181         _recorded_nms->delete_at(idx);

   143     case 0:

   182         delete old;

   144     case 1: {

   145       break;

   146     }

   147     case 2: {

   148       assert_locked_or_safepoint(CodeCache_lock);

   149       _nmethod_table->flush_nmethod(nm);

   150       break;

   151     }

   152     default:

   153       ShouldNotReachHere();

   154   }

   155 }

   156 

   157 void ShenandoahCodeRoots::prepare_concurrent_unloading() {

   158   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");

   159   _disarmed_value ++;

   160   // 0 is reserved for new nmethod

   161   if (_disarmed_value == 0) {

   162     _disarmed_value = 1;

   163   }

   164 

   165   JavaThreadIteratorWithHandle jtiwh;

   166   for (JavaThread *thr = jtiwh.next(); thr != NULL; thr = jtiwh.next()) {

   167     ShenandoahThreadLocalData::set_disarmed_value(thr, _disarmed_value);

   168   }

   169 }

   170 

   171 class ShenandoahNMethodUnlinkClosure : public NMethodClosure {

   172 private:

   173   bool            _unloading_occurred;

   174   volatile bool   _failed;

   175   ShenandoahHeap* _heap;

   176 

   177   void set_failed() {

   178     Atomic::store(&_failed, true);

   179   }

   180 

   181    void unlink(nmethod* nm) {

   182      // Unlinking of the dependencies must happen before the

   183      // handshake separating unlink and purge.

   184      nm->flush_dependencies(false /* delete_immediately */);

   185 

   186      // unlink_from_method will take the CompiledMethod_lock.

   187      // In this case we don't strictly need it when unlinking nmethods from

   188      // the Method, because it is only concurrently unlinked by

   189      // the entry barrier, which acquires the per nmethod lock.

   190      nm->unlink_from_method();

   191 

   192      if (nm->is_osr_method()) {

   193        // Invalidate the osr nmethod only once

   194        nm->invalidate_osr_method();

   195      }

   196    }

   197 public:

   198   ShenandoahNMethodUnlinkClosure(bool unloading_occurred) :

   199       _unloading_occurred(unloading_occurred),

   200       _failed(false),

   201       _heap(ShenandoahHeap::heap()) {}

   202 

   203   virtual void do_nmethod(nmethod* nm) {

   204     if (failed()) {

   205       return;

   206     }

   207 

   208     ShenandoahNMethod* nm_data = ShenandoahNMethod::gc_data(nm);

   209     assert(!nm_data->is_unregistered(), "Should not see unregistered entry");

   210 

   211     if (!nm->is_alive()) {

   212       return;

   213     }

   214 

   215     if (nm->is_unloading()) {

   216       ShenandoahReentrantLocker locker(nm_data->lock());

   217       unlink(nm);

   218       return;

   219     }

   220 

   221     ShenandoahReentrantLocker locker(nm_data->lock());

   222 

   223     // Heal oops and disarm

   224     ShenandoahEvacOOMScope scope;

   225     ShenandoahNMethod::heal_nmethod(nm);

   226     ShenandoahNMethod::disarm_nmethod(nm);

   227 

   228     // Clear compiled ICs and exception caches

   229     if (!nm->unload_nmethod_caches(_unloading_occurred)) {

   230       set_failed();

   231     }

   232   }

   233 

   234   bool failed() const {

   235     return Atomic::load(&_failed);

   236   }

   237 };

   238 

   239 class ShenandoahUnlinkTask : public AbstractGangTask {

   240 private:

   241   ShenandoahNMethodUnlinkClosure      _cl;

   242   ICRefillVerifier*                   _verifier;

   243   ShenandoahConcurrentNMethodIterator _iterator;

   244 

   245 public:

   246   ShenandoahUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier) :

   247     AbstractGangTask("ShenandoahNMethodUnlinkTask"),

   248     _cl(unloading_occurred),

   249     _verifier(verifier),

   250     _iterator(ShenandoahCodeRoots::table()) {

   251     MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);

   252     _iterator.nmethods_do_begin();

   253   }

   254 

   255   ~ShenandoahUnlinkTask() {

   256     MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);

   257     _iterator.nmethods_do_end();

   258   }

   259 

   260   virtual void work(uint worker_id) {

   261     ICRefillVerifierMark mark(_verifier);

   262     _iterator.nmethods_do(&_cl);

   263   }

   264 

   265   bool success() const {

   266     return !_cl.failed();

   267   }

   268 };

   269 

   270 void ShenandoahCodeRoots::unlink(WorkGang* workers, bool unloading_occurred) {

   271   assert(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(),

   272          "Only when running concurrent class unloading");

   273 

   274   for (;;) {

   275     ICRefillVerifier verifier;

   276 

   277     {

   278       ShenandoahUnlinkTask task(unloading_occurred, &verifier);

   279       workers->run_task(&task);

   280       if (task.success()) {

   281         return;

   184       break;

   283     }

   185     }

   284 

   186     default:

   285     // Cleaning failed because we ran out of transitional IC stubs,

   187       ShouldNotReachHere();

   286     // so we have to refill and try again. Refilling requires taking

   188   }

   287     // a safepoint, so we temporarily leave the suspendible thread set.

   288     SuspendibleThreadSetLeaver sts;

   289     InlineCacheBuffer::refill_ic_stubs();

   290   }

   291 }

   292 

   293 class ShenandoahNMethodPurgeClosure : public NMethodClosure {

   294 public:

   295   virtual void do_nmethod(nmethod* nm) {

   296     if (nm->is_alive() && nm->is_unloading()) {

   297       nm->make_unloaded();

   298     }

   299   }

   300 };

   301 

   302 class ShenandoahNMethodPurgeTask : public AbstractGangTask {

   303 private:

   304   ShenandoahNMethodPurgeClosure       _cl;

   305   ShenandoahConcurrentNMethodIterator _iterator;

   306 

   307 public:

   308   ShenandoahNMethodPurgeTask() :

   309     AbstractGangTask("ShenandoahNMethodPurgeTask"),

   310     _cl(),

   311     _iterator(ShenandoahCodeRoots::table()) {

   312     MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);

   313     _iterator.nmethods_do_begin();

   314   }

   315 

   316   ~ShenandoahNMethodPurgeTask() {

   317     MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);

   318     _iterator.nmethods_do_end();

   319   }

   320 

   321   virtual void work(uint worker_id) {

   322     _iterator.nmethods_do(&_cl);

   323   }

   324 };

   325 

   326 void ShenandoahCodeRoots::purge(WorkGang* workers) {

   327   assert(ShenandoahConcurrentRoots::should_do_concurrent_class_unloading(),

   328          "Only when running concurrent class unloading");

   329 

   330   ShenandoahNMethodPurgeTask task;

   331   workers->run_task(&task);

   194         _claimed(0) {

   336         _table_snapshot(NULL) {

   261 

   406   assert(_table_snapshot != NULL, "Sanity");

   262   size_t stride = 256; // educated guess

   407   _table_snapshot->parallel_blobs_do<CSET_FILTER>(f);

   263 

   408 }

   264   GrowableArray<ShenandoahNMethod*>* list = ShenandoahCodeRoots::_recorded_nms;

   409 

   265 

   266   size_t max = (size_t)list->length();

   267   while (_claimed < max) {

   268     size_t cur = Atomic::add(&_claimed, stride) - stride;

   269     size_t start = cur;

   270     size_t end = MIN2(cur + stride, max);

   271     if (start >= max) break;

   272 

   273     for (size_t idx = start; idx < end; idx++) {

   274       ShenandoahNMethod* nmr = list->at((int) idx);

   275       nmr->assert_alive_and_correct();

   276 

   277       if (CSET_FILTER && !nmr->has_cset_oops(_heap)) {

   278         continue;

   279       }

   280 

   281       f->do_code_blob(nmr->nm());

   282     }

   283   }

   284 }

   285 

   286 ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>* oops) {

   287   _nm = nm;

   288   _oops = NEW_C_HEAP_ARRAY(oop*, oops->length(), mtGC);

   289   _oops_count = oops->length();

   290   for (int c = 0; c < _oops_count; c++) {

   291     _oops[c] = oops->at(c);

   292   }

   293 }

   294 

   295 ShenandoahNMethod::~ShenandoahNMethod() {

   296   if (_oops != NULL) {

   297     FREE_C_HEAP_ARRAY(oop*, _oops);

   298   }

   299 }

   300 

   301 bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) {

   302   for (int c = 0; c < _oops_count; c++) {

   303     oop o = RawAccess<>::oop_load(_oops[c]);

   304     if (heap->in_collection_set(o)) {

   305       return true;

   306     }

   307   }

   308   return false;

   309 }

   310 

   311 #ifdef ASSERT

   312 void ShenandoahNMethod::assert_alive_and_correct() {

   313   assert(_nm->is_alive(), "only alive nmethods here");

   314   assert(_oops_count > 0, "should have filtered nmethods without oops before");

   315   ShenandoahHeap* heap = ShenandoahHeap::heap();

   316   for (int c = 0; c < _oops_count; c++) {

   317     oop *loc = _oops[c];

   318     assert(_nm->code_contains((address) loc) || _nm->oops_contains(loc), "nmethod should contain the oop*");

   319     oop o = RawAccess<>::oop_load(loc);

   320     shenandoah_assert_correct_except(loc, o,

   321              o == NULL ||

   322              heap->is_full_gc_move_in_progress() ||

   323              (VMThread::vm_operation() != NULL) && (VMThread::vm_operation()->type() == VM_Operation::VMOp_HeapWalkOperation)

   324     );

   325   }

   326 }

   327 

   328 void ShenandoahNMethod::assert_same_oops(GrowableArray<oop*>* oops) {

   329   assert(_oops_count == oops->length(), "should have the same number of oop*");

   330   for (int c = 0; c < _oops_count; c++) {

   331     assert(_oops[c] == oops->at(c), "should be the same oop*");

   332   }

   333 }

   334 #endif