jdk-sandbox: comparison src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

equal deleted inserted replaced

-:3863dfd99d8e
+:55ce65813b62
 // get unmarked objects in the roots.
 if (!cancelled_gc()) {
 concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false);
+// Marking is completed, deactivate SATB barrier
+set_concurrent_mark_in_progress(false);
+mark_complete_marking_context();
+parallel_cleaning(false /* full gc*/);
 if (has_forwarded_objects()) {
 // Degen may be caused by failed evacuation of roots
 if (is_degenerated_gc_in_progress()) {
 concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots);
 } else {
 concurrent_mark()->update_thread_roots(ShenandoahPhaseTimings::update_roots);
 }
+set_has_forwarded_objects(false);
 }
 if (ShenandoahVerify) {
 verifier()->verify_roots_no_forwarded();
 }
+// All allocations past TAMS are implicitly live, adjust the region data.
-stop_concurrent_marking();
+// Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
 {
 ShenandoahGCPhase phase(ShenandoahPhaseTimings::complete_liveness);
-// All allocations past TAMS are implicitly live, adjust the region data.
-// Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
 ShenandoahCompleteLivenessClosure cl;
 parallel_heap_region_iterate(&cl);
 }
+// Force the threads to reacquire their TLABs outside the collection set.
 {
-ShenandoahGCPhase prepare_evac(ShenandoahPhaseTimings::prepare_evac);
+ShenandoahGCPhase phase(ShenandoahPhaseTimings::retire_tlabs);
 make_parsable(true);
+}
+// We are about to select the collection set, make sure it knows about
+// current pinning status. Also, this allows trashing more regions that
+// now have their pinning status dropped.
+{
+ShenandoahGCPhase phase(ShenandoahPhaseTimings::sync_pinned);
+sync_pinned_region_status();
+}
+// Trash the collection set left over from previous cycle, if any.
+{
+ShenandoahGCPhase phase(ShenandoahPhaseTimings::trash_cset);
 trash_cset_regions();
+}
-{
-ShenandoahHeapLocker locker(lock());
+{
-_collection_set->clear();
+ShenandoahGCPhase phase(ShenandoahPhaseTimings::prepare_evac);
-_free_set->clear();
+ShenandoahHeapLocker locker(lock());
-heuristics()->choose_collection_set(_collection_set);
+_collection_set->clear();
+_free_set->clear();
-_free_set->rebuild();
-}
+heuristics()->choose_collection_set(_collection_set);
+_free_set->rebuild();
 }
 // If collection set has candidates, start evacuation.
 // Otherwise, bypass the rest of the cycle.
 if (!collection_set()->is_empty()) {
 Universe::verify();
 }
 }
 } else {
+// If this cycle was updating references, we need to keep the has_forwarded_objects
+// flag on, for subsequent phases to deal with it.
 concurrent_mark()->cancel();
-stop_concurrent_marking();
+set_concurrent_mark_in_progress(false);
 if (process_references()) {
 // Abandon reference processing right away: pre-cleaning must have failed.
 ReferenceProcessor *rp = ref_processor();
 rp->disable_discovery();
 void ShenandoahHeap::op_final_evac() {
 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
 set_evacuation_in_progress(false);
-retire_and_reset_gclabs();
+{
+ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_evac_retire_gclabs);
+retire_and_reset_gclabs();
+}
 if (ShenandoahVerify) {
 verifier()->verify_after_evacuation();
 }
 // the collection set, and then the pin reached the cset region. If we continue
 // the cycle here, we would trash the cset and alive objects in it. To avoid
 // it, we fail degeneration right away and slide into Full GC to recover.
 {
+sync_pinned_region_status();
 collection_set()->clear_current_index();
 ShenandoahHeapRegion* r;
 while ((r = collection_set()->next()) != NULL) {
 if (r->is_pinned()) {
 void ShenandoahHeap::op_degenerated_futile() {
 shenandoah_policy()->record_degenerated_upgrade_to_full();
 op_full(GCCause::_shenandoah_upgrade_to_full_gc);
 }
-void ShenandoahHeap::stop_concurrent_marking() {
-assert(is_concurrent_mark_in_progress(), "How else could we get here?");
-set_concurrent_mark_in_progress(false);
-if (!cancelled_gc()) {
-// If we needed to update refs, and concurrent marking has been cancelled,
-// we need to finish updating references.
-set_has_forwarded_objects(false);
-mark_complete_marking_context();
-}
-}
 void ShenandoahHeap::force_satb_flush_all_threads() {
 if (!is_concurrent_mark_in_progress() && !is_concurrent_traversal_in_progress()) {
 // No need to flush SATBs
 return;
 }
 }
 ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
 }
 void ShenandoahHeap::set_concurrent_traversal_in_progress(bool in_progress) {
-set_gc_state_mask(TRAVERSAL | HAS_FORWARDED | UPDATEREFS, in_progress);
+set_gc_state_mask(TRAVERSAL, in_progress);
 ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
 }
 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
 ShenandoahPhaseTimings::full_gc_purge_par :
 ShenandoahPhaseTimings::purge_par;
 // Cleanup weak roots
 ShenandoahGCPhase phase(timing_phase);
 if (has_forwarded_objects()) {
-ShenandoahForwardedIsAliveClosure is_alive;
+if (is_traversal_mode()) {
-ShenandoahUpdateRefsClosure keep_alive;
+ShenandoahForwardedIsAliveClosure is_alive;
-ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>
+ShenandoahTraversalUpdateRefsClosure keep_alive;
-cleaning_task(&is_alive, &keep_alive, num_workers);
+ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahTraversalUpdateRefsClosure>
-_workers->run_task(&cleaning_task);
+cleaning_task(&is_alive, &keep_alive, num_workers);
+_workers->run_task(&cleaning_task);
+} else {
+ShenandoahForwardedIsAliveClosure is_alive;
+ShenandoahUpdateRefsClosure keep_alive;
+ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahUpdateRefsClosure>
+cleaning_task(&is_alive, &keep_alive, num_workers);
+_workers->run_task(&cleaning_task);
+}
 } else {
 ShenandoahIsAliveClosure is_alive;
 #ifdef ASSERT
 ShenandoahAssertNotForwardedClosure verify_cl;
 ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, ShenandoahAssertNotForwardedClosure>
 stw_process_weak_roots(full_gc);
 stw_unload_classes(full_gc);
 }
 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
-set_gc_state_mask(HAS_FORWARDED, cond);
+if (is_traversal_mode()) {
+set_gc_state_mask(HAS_FORWARDED | UPDATEREFS, cond);
+} else {
+set_gc_state_mask(HAS_FORWARDED, cond);
+}
 }
 void ShenandoahHeap::set_process_references(bool pr) {
 _process_references.set_cond(pr);
 }
 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
 ShenandoahCodeRoots::remove_nmethod(nm);
 }
 oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
+heap_region_containing(o)->record_pin();
+return o;
+}
+void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
+heap_region_containing(o)->record_unpin();
+}
+void ShenandoahHeap::sync_pinned_region_status() {
 ShenandoahHeapLocker locker(lock());
-heap_region_containing(o)->make_pinned();
-return o;
+for (size_t i = 0; i < num_regions(); i++) {
-}
+ShenandoahHeapRegion *r = get_region(i);
+if (r->is_active()) {
-void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
+if (r->is_pinned()) {
-ShenandoahHeapLocker locker(lock());
+if (r->pin_count() == 0) {
-heap_region_containing(o)->make_unpinned();
+r->make_unpinned();
 }
+} else {
+if (r->pin_count() > 0) {
+r->make_pinned();
+}
+}
+}
+}
+assert_pinned_region_status();
+}
+#ifdef ASSERT
+void ShenandoahHeap::assert_pinned_region_status() {
+for (size_t i = 0; i < num_regions(); i++) {
+ShenandoahHeapRegion* r = get_region(i);
+assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
+"Region " SIZE_FORMAT " pinning status is inconsistent", i);
+}
+}
+#endif
 GCTimer* ShenandoahHeap::gc_timer() const {
 return _gc_timer;
 }
 void ShenandoahHeap::op_init_updaterefs() {
 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
 set_evacuation_in_progress(false);
-retire_and_reset_gclabs();
+{
+ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_retire_gclabs);
+retire_and_reset_gclabs();
+}
 if (ShenandoahVerify) {
 if (!is_degenerated_gc_in_progress()) {
 verifier()->verify_roots_no_forwarded_except(ShenandoahRootVerifier::ThreadRoots);
 }
 verifier()->verify_before_updaterefs();
 }
 set_update_refs_in_progress(true);
-make_parsable(true);
-for (uint i = 0; i < num_regions(); i++) {
+{
-ShenandoahHeapRegion* r = get_region(i);
+ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_refs_prepare);
-r->set_concurrent_iteration_safe_limit(r->top());
-}
+make_parsable(true);
+for (uint i = 0; i < num_regions(); i++) {
-// Reset iterator.
+ShenandoahHeapRegion* r = get_region(i);
-_update_refs_iterator.reset();
+r->set_concurrent_iteration_safe_limit(r->top());
+}
+// Reset iterator.
+_update_refs_iterator.reset();
+}
 if (ShenandoahPacing) {
 pacer()->setup_for_updaterefs();
 }
 }
 void ShenandoahHeap::op_final_updaterefs() {
 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
 // Check if there is left-over work, and finish it
 if (_update_refs_iterator.has_next()) {
-ShenandoahGCPhase final_work(ShenandoahPhaseTimings::final_update_refs_finish_work);
+ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_finish_work);
 // Finish updating references where we left off.
 clear_cancelled_gc();
 update_heap_references(false);
 }
 // Has to be done before cset is clear
 if (ShenandoahVerify) {
 verifier()->verify_roots_in_to_space();
 }
-ShenandoahGCPhase final_update_refs(ShenandoahPhaseTimings::final_update_refs_recycle);
+// Drop unnecessary "pinned" state from regions that does not have CP marks
+// anymore, as this would allow trashing them below.
-trash_cset_regions();
+{
+ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_sync_pinned);
+sync_pinned_region_status();
+}
+{
+ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_trash_cset);
+trash_cset_regions();
+}
 set_has_forwarded_objects(false);
 set_update_refs_in_progress(false);
 if (ShenandoahVerify) {
 verifier()->verify_after_updaterefs();

branch	stuefe-new-metaspace-branch
changeset 58838	55ce65813b62
parent 58645	28c7e6711871
parent 58738	ef2b75750838
child 59272	54750b448264