diff -r 044f2ca6ce22 -r 785a12e0f89b src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp Wed May 29 09:25:20 2019 +0200 +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp Wed May 29 12:01:21 2019 +0200 @@ -25,6 +25,7 @@ #include "code/codeCache.hpp" #include "gc/shared/gcTraceTime.inline.hpp" +#include "gc/shared/preservedMarks.inline.hpp" #include "gc/shenandoah/shenandoahForwarding.inline.hpp" #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp" #include "gc/shenandoah/shenandoahCollectionSet.hpp" @@ -46,11 +47,16 @@ #include "memory/universe.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/oop.inline.hpp" +#include "runtime/biasedLocking.hpp" #include "runtime/thread.hpp" #include "utilities/copy.hpp" #include "utilities/growableArray.hpp" #include "gc/shared/workgroup.hpp" +ShenandoahMarkCompact::ShenandoahMarkCompact() : + _gc_timer(NULL), + _preserved_marks(new PreservedMarksSet(true)) {} + void ShenandoahMarkCompact::initialize(GCTimer* gc_timer) { _gc_timer = gc_timer; } @@ -121,6 +127,10 @@ // e. Set back forwarded objects bit back, in case some steps above dropped it. heap->set_has_forwarded_objects(has_forwarded_objects); + + // The rest of prologue: + BiasedLocking::preserve_marks(); + _preserved_marks->init(heap->workers()->active_workers()); } heap->make_parsable(true); @@ -159,6 +169,16 @@ phase4_compact_objects(worker_slices); } + { + // Epilogue + SharedRestorePreservedMarksTaskExecutor exec(heap->workers()); + _preserved_marks->restore(&exec); + BiasedLocking::restore_marks(); + _preserved_marks->reclaim(); + + JvmtiExport::gc_epilogue(); + } + // Resize metaspace MetaspaceGC::compute_new_size(); @@ -168,8 +188,6 @@ } FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices); - JvmtiExport::gc_epilogue(); - heap->set_full_gc_move_in_progress(false); heap->set_full_gc_in_progress(false); @@ -230,6 +248,7 @@ class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure { private: + PreservedMarks* const _preserved_marks; ShenandoahHeap* const _heap; GrowableArray& _empty_regions; int _empty_regions_pos; @@ -238,7 +257,10 @@ HeapWord* _compact_point; public: - ShenandoahPrepareForCompactionObjectClosure(GrowableArray& empty_regions, ShenandoahHeapRegion* to_region) : + ShenandoahPrepareForCompactionObjectClosure(PreservedMarks* preserved_marks, + GrowableArray& empty_regions, + ShenandoahHeapRegion* to_region) : + _preserved_marks(preserved_marks), _heap(ShenandoahHeap::heap()), _empty_regions(empty_regions), _empty_regions_pos(0), @@ -268,7 +290,7 @@ assert(_heap->complete_marking_context()->is_marked(p), "must be marked"); assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked"); - size_t obj_size = p->size() + ShenandoahForwarding::word_size(); + size_t obj_size = p->size(); if (_compact_point + obj_size > _to_region->end()) { finish_region(); @@ -291,13 +313,15 @@ // Object fits into current region, record new location: assert(_compact_point + obj_size <= _to_region->end(), "must fit"); shenandoah_assert_not_forwarded(NULL, p); - ShenandoahForwarding::set_forwardee_raw(p, _compact_point + ShenandoahForwarding::word_size()); + _preserved_marks->push_if_necessary(p, p->mark_raw()); + p->forward_to(oop(_compact_point)); _compact_point += obj_size; } }; class ShenandoahPrepareForCompactionTask : public AbstractGangTask { private: + PreservedMarksSet* const _preserved_marks; ShenandoahHeap* const _heap; ShenandoahHeapRegionSet** const _worker_slices; ShenandoahRegionIterator _heap_regions; @@ -320,8 +344,9 @@ } public: - ShenandoahPrepareForCompactionTask(ShenandoahHeapRegionSet** worker_slices) : + ShenandoahPrepareForCompactionTask(PreservedMarksSet* preserved_marks, ShenandoahHeapRegionSet** worker_slices) : AbstractGangTask("Shenandoah Prepare For Compaction Task"), + _preserved_marks(preserved_marks), _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) { } @@ -337,7 +362,7 @@ // Remember empty regions and reuse them as needed. ResourceMark rm; GrowableArray empty_regions((int)_heap->num_regions()); - ShenandoahPrepareForCompactionObjectClosure cl(empty_regions, from_region); + ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region); while (from_region != NULL) { cl.set_from_region(from_region); if (from_region->has_live()) { @@ -377,8 +402,8 @@ size_t to_begin = heap->num_regions(); size_t to_end = heap->num_regions(); - for (size_t c = heap->num_regions() - 1; c > 0; c--) { - ShenandoahHeapRegion *r = heap->get_region(c); + for (size_t c = heap->num_regions(); c > 0; c--) { + ShenandoahHeapRegion *r = heap->get_region(c - 1); if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) { // To-region candidate: record this, and continue scan to_begin = r->region_number(); @@ -387,15 +412,16 @@ if (r->is_humongous_start() && r->is_move_allowed()) { // From-region candidate: movable humongous region - oop old_obj = oop(r->bottom() + ShenandoahForwarding::word_size()); - size_t words_size = old_obj->size() + ShenandoahForwarding::word_size(); + oop old_obj = oop(r->bottom()); + size_t words_size = old_obj->size(); size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize); size_t start = to_end - num_regions; if (start >= to_begin && start != r->region_number()) { // Fits into current window, and the move is non-trivial. Record the move then, and continue scan. - ShenandoahForwarding::set_forwardee_raw(old_obj, heap->get_region(start)->bottom() + ShenandoahForwarding::word_size()); + _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark_raw()); + old_obj->forward_to(oop(heap->get_region(start)->bottom())); to_end = start; continue; } @@ -443,7 +469,7 @@ void heap_region_do(ShenandoahHeapRegion* r) { if (r->is_humongous_start()) { - oop humongous_obj = oop(r->bottom() + ShenandoahForwarding::word_size()); + oop humongous_obj = oop(r->bottom()); if (!_ctx->is_marked(humongous_obj)) { assert(!r->has_live(), "Region " SIZE_FORMAT " is not marked, should not have live", r->region_number()); @@ -484,7 +510,7 @@ // Compute the new addresses for regular objects { ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular); - ShenandoahPrepareForCompactionTask prepare_task(worker_slices); + ShenandoahPrepareForCompactionTask prepare_task(_preserved_marks, worker_slices); heap->workers()->run_task(&prepare_task); } @@ -506,8 +532,10 @@ if (!CompressedOops::is_null(o)) { oop obj = CompressedOops::decode_not_null(o); assert(_ctx->is_marked(obj), "must be marked"); - oop forw = oop(ShenandoahForwarding::get_forwardee_raw(obj)); - RawAccess::oop_store(p, forw); + if (obj->is_forwarded()) { + oop forw = obj->forwardee(); + RawAccess::oop_store(p, forw); + } } } @@ -531,7 +559,6 @@ } void do_object(oop p) { assert(_heap->complete_marking_context()->is_marked(p), "must be marked"); - HeapWord* forw = ShenandoahForwarding::get_forwardee_raw(p); p->oop_iterate(&_cl); } }; @@ -562,15 +589,17 @@ class ShenandoahAdjustRootPointersTask : public AbstractGangTask { private: ShenandoahRootAdjuster* _rp; - + PreservedMarksSet* _preserved_marks; public: - ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp) : + ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp, PreservedMarksSet* preserved_marks) : AbstractGangTask("Shenandoah Adjust Root Pointers Task"), - _rp(rp) {} + _rp(rp), + _preserved_marks(preserved_marks) {} void work(uint worker_id) { ShenandoahAdjustPointersClosure cl; _rp->roots_do(worker_id, &cl); + _preserved_marks->get(worker_id)->adjust_during_full_gc(); } }; @@ -587,7 +616,7 @@ DerivedPointerTable::clear(); #endif ShenandoahRootAdjuster rp(nworkers, ShenandoahPhaseTimings::full_gc_roots); - ShenandoahAdjustRootPointersTask task(&rp); + ShenandoahAdjustRootPointersTask task(&rp, _preserved_marks); workers->run_task(&task); #if COMPILER2_OR_JVMCI DerivedPointerTable::update_pointers(); @@ -610,13 +639,13 @@ void do_object(oop p) { assert(_heap->complete_marking_context()->is_marked(p), "must be marked"); size_t size = (size_t)p->size(); - HeapWord* compact_to = ShenandoahForwarding::get_forwardee_raw(p); - HeapWord* compact_from = (HeapWord*) p; - if (compact_from != compact_to) { + if (p->is_forwarded()) { + HeapWord* compact_from = (HeapWord*) p; + HeapWord* compact_to = (HeapWord*) p->forwardee(); Copy::aligned_conjoint_words(compact_from, compact_to, size); + oop new_obj = oop(compact_to); + new_obj->init_mark_raw(); } - oop new_obj = oop(compact_to); - ShenandoahForwarding::initialize(new_obj); } }; @@ -707,31 +736,30 @@ ShenandoahHeap* heap = ShenandoahHeap::heap(); - for (size_t c = heap->num_regions() - 1; c > 0; c--) { - ShenandoahHeapRegion* r = heap->get_region(c); + for (size_t c = heap->num_regions(); c > 0; c--) { + ShenandoahHeapRegion* r = heap->get_region(c - 1); if (r->is_humongous_start()) { - oop old_obj = oop(r->bottom() + ShenandoahForwarding::word_size()); - size_t words_size = old_obj->size() + ShenandoahForwarding::word_size(); + oop old_obj = oop(r->bottom()); + if (!old_obj->is_forwarded()) { + // No need to move the object, it stays at the same slot + continue; + } + size_t words_size = old_obj->size(); size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize); size_t old_start = r->region_number(); size_t old_end = old_start + num_regions - 1; - size_t new_start = heap->heap_region_index_containing(ShenandoahForwarding::get_forwardee_raw(old_obj)); + size_t new_start = heap->heap_region_index_containing(old_obj->forwardee()); size_t new_end = new_start + num_regions - 1; - - if (old_start == new_start) { - // No need to move the object, it stays at the same slot - continue; - } - + assert(old_start != new_start, "must be real move"); assert (r->is_move_allowed(), "should be movable"); Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(), heap->get_region(new_start)->bottom(), ShenandoahHeapRegion::region_size_words()*num_regions); - oop new_obj = oop(heap->get_region(new_start)->bottom() + ShenandoahForwarding::word_size()); - ShenandoahForwarding::initialize(new_obj); + oop new_obj = oop(heap->get_region(new_start)->bottom()); + new_obj->init_mark_raw(); { for (size_t c = old_start; c <= old_end; c++) {