jdk-sandbox: comparison src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp

equal deleted inserted replaced

-:044f2ca6ce22
+:785a12e0f89b
 #include "precompiled.hpp"
 #include "code/codeCache.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
+#include "gc/shared/preservedMarks.inline.hpp"
 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
 #include "gc/shenandoah/shenandoahFreeSet.hpp"
 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
 #include "memory/metaspace.hpp"
 #include "memory/universe.hpp"
 #include "oops/compressedOops.inline.hpp"
 #include "oops/oop.inline.hpp"
+#include "runtime/biasedLocking.hpp"
 #include "runtime/thread.hpp"
 #include "utilities/copy.hpp"
 #include "utilities/growableArray.hpp"
 #include "gc/shared/workgroup.hpp"
+ShenandoahMarkCompact::ShenandoahMarkCompact() :
+_gc_timer(NULL),
+_preserved_marks(new PreservedMarksSet(true)) {}
 void ShenandoahMarkCompact::initialize(GCTimer* gc_timer) {
 _gc_timer = gc_timer;
 }
 rp->abandon_partial_discovery();
 rp->verify_no_references_recorded();
 // e. Set back forwarded objects bit back, in case some steps above dropped it.
 heap->set_has_forwarded_objects(has_forwarded_objects);
+// The rest of prologue:
+BiasedLocking::preserve_marks();
+_preserved_marks->init(heap->workers()->active_workers());
 }
 heap->make_parsable(true);
 OrderAccess::fence();
 phase3_update_references();
 phase4_compact_objects(worker_slices);
 }
+{
+// Epilogue
+SharedRestorePreservedMarksTaskExecutor exec(heap->workers());
+_preserved_marks->restore(&exec);
+BiasedLocking::restore_marks();
+_preserved_marks->reclaim();
+JvmtiExport::gc_epilogue();
+}
 // Resize metaspace
 MetaspaceGC::compute_new_size();
 // Free worker slices
 for (uint i = 0; i < heap->max_workers(); i++) {
 delete worker_slices[i];
 }
 FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices);
-JvmtiExport::gc_epilogue();
 heap->set_full_gc_move_in_progress(false);
 heap->set_full_gc_in_progress(false);
 if (ShenandoahVerify) {
 heap->mark_complete_marking_context();
 }
 class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
 private:
+PreservedMarks*          const _preserved_marks;
 ShenandoahHeap*          const _heap;
 GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
 int _empty_regions_pos;
 ShenandoahHeapRegion*          _to_region;
 ShenandoahHeapRegion*          _from_region;
 HeapWord* _compact_point;
 public:
-ShenandoahPrepareForCompactionObjectClosure(GrowableArray<ShenandoahHeapRegion*>& empty_regions, ShenandoahHeapRegion* to_region) :
+ShenandoahPrepareForCompactionObjectClosure(PreservedMarks* preserved_marks,
+GrowableArray<ShenandoahHeapRegion*>& empty_regions,
+ShenandoahHeapRegion* to_region) :
+_preserved_marks(preserved_marks),
 _heap(ShenandoahHeap::heap()),
 _empty_regions(empty_regions),
 _empty_regions_pos(0),
 _to_region(to_region),
 _from_region(NULL),
 void do_object(oop p) {
 assert(_from_region != NULL, "must set before work");
 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
 assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
-size_t obj_size = p->size() + ShenandoahForwarding::word_size();
+size_t obj_size = p->size();
 if (_compact_point + obj_size > _to_region->end()) {
 finish_region();
 // Object doesn't fit. Pick next empty region and start compacting there.
 ShenandoahHeapRegion* new_to_region;
 }
 // Object fits into current region, record new location:
 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
 shenandoah_assert_not_forwarded(NULL, p);
-ShenandoahForwarding::set_forwardee_raw(p, _compact_point + ShenandoahForwarding::word_size());
+_preserved_marks->push_if_necessary(p, p->mark_raw());
+p->forward_to(oop(_compact_point));
 _compact_point += obj_size;
 }
 };
 class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
 private:
+PreservedMarksSet*        const _preserved_marks;
 ShenandoahHeap*           const _heap;
 ShenandoahHeapRegionSet** const _worker_slices;
 ShenandoahRegionIterator        _heap_regions;
 ShenandoahHeapRegion* next_from_region(ShenandoahHeapRegionSet* slice) {
 return from_region;
 }
 public:
-ShenandoahPrepareForCompactionTask(ShenandoahHeapRegionSet** worker_slices) :
+ShenandoahPrepareForCompactionTask(PreservedMarksSet* preserved_marks, ShenandoahHeapRegionSet** worker_slices) :
 AbstractGangTask("Shenandoah Prepare For Compaction Task"),
+_preserved_marks(preserved_marks),
 _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
 }
 void work(uint worker_id) {
 ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
 // Sliding compaction. Walk all regions in the slice, and compact them.
 // Remember empty regions and reuse them as needed.
 ResourceMark rm;
 GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
-ShenandoahPrepareForCompactionObjectClosure cl(empty_regions, from_region);
+ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
 while (from_region != NULL) {
 cl.set_from_region(from_region);
 if (from_region->has_live()) {
 _heap->marked_object_iterate(from_region, &cl);
 }
 // detected, then sliding restarts towards that non-movable region.
 size_t to_begin = heap->num_regions();
 size_t to_end = heap->num_regions();
-for (size_t c = heap->num_regions() - 1; c > 0; c--) {
+for (size_t c = heap->num_regions(); c > 0; c--) {
-ShenandoahHeapRegion *r = heap->get_region(c);
+ShenandoahHeapRegion *r = heap->get_region(c - 1);
 if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
 // To-region candidate: record this, and continue scan
 to_begin = r->region_number();
 continue;
 }
 if (r->is_humongous_start() && r->is_move_allowed()) {
 // From-region candidate: movable humongous region
-oop old_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
+oop old_obj = oop(r->bottom());
-size_t words_size = old_obj->size() + ShenandoahForwarding::word_size();
+size_t words_size = old_obj->size();
 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
 size_t start = to_end - num_regions;
 if (start >= to_begin && start != r->region_number()) {
 // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
-ShenandoahForwarding::set_forwardee_raw(old_obj, heap->get_region(start)->bottom() + ShenandoahForwarding::word_size());
+_preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark_raw());
+old_obj->forward_to(oop(heap->get_region(start)->bottom()));
 to_end = start;
 continue;
 }
 }
 _heap(ShenandoahHeap::heap()),
 _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
 void heap_region_do(ShenandoahHeapRegion* r) {
 if (r->is_humongous_start()) {
-oop humongous_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
+oop humongous_obj = oop(r->bottom());
 if (!_ctx->is_marked(humongous_obj)) {
 assert(!r->has_live(),
 "Region " SIZE_FORMAT " is not marked, should not have live", r->region_number());
 _heap->trash_humongous_region_at(r);
 } else {
 }
 // Compute the new addresses for regular objects
 {
 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
-ShenandoahPrepareForCompactionTask prepare_task(worker_slices);
+ShenandoahPrepareForCompactionTask prepare_task(_preserved_marks, worker_slices);
 heap->workers()->run_task(&prepare_task);
 }
 // Compute the new addresses for humongous objects
 {
 inline void do_oop_work(T* p) {
 T o = RawAccess<>::oop_load(p);
 if (!CompressedOops::is_null(o)) {
 oop obj = CompressedOops::decode_not_null(o);
 assert(_ctx->is_marked(obj), "must be marked");
-oop forw = oop(ShenandoahForwarding::get_forwardee_raw(obj));
+if (obj->is_forwarded()) {
-RawAccess<IS_NOT_NULL>::oop_store(p, forw);
+oop forw = obj->forwardee();
+RawAccess<IS_NOT_NULL>::oop_store(p, forw);
+}
 }
 }
 public:
 ShenandoahAdjustPointersClosure() :
 ShenandoahAdjustPointersObjectClosure() :
 _heap(ShenandoahHeap::heap()) {
 }
 void do_object(oop p) {
 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
-HeapWord* forw = ShenandoahForwarding::get_forwardee_raw(p);
 p->oop_iterate(&_cl);
 }
 };
 class ShenandoahAdjustPointersTask : public AbstractGangTask {
 };
 class ShenandoahAdjustRootPointersTask : public AbstractGangTask {
 private:
 ShenandoahRootAdjuster* _rp;
+PreservedMarksSet* _preserved_marks;
 public:
-ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp) :
+ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp, PreservedMarksSet* preserved_marks) :
 AbstractGangTask("Shenandoah Adjust Root Pointers Task"),
-_rp(rp) {}
+_rp(rp),
+_preserved_marks(preserved_marks) {}
 void work(uint worker_id) {
 ShenandoahAdjustPointersClosure cl;
 _rp->roots_do(worker_id, &cl);
+_preserved_marks->get(worker_id)->adjust_during_full_gc();
 }
 };
 void ShenandoahMarkCompact::phase3_update_references() {
 GCTraceTime(Info, gc, phases) time("Phase 3: Adjust pointers", _gc_timer);
 {
 #if COMPILER2_OR_JVMCI
 DerivedPointerTable::clear();
 #endif
 ShenandoahRootAdjuster rp(nworkers, ShenandoahPhaseTimings::full_gc_roots);
-ShenandoahAdjustRootPointersTask task(&rp);
+ShenandoahAdjustRootPointersTask task(&rp, _preserved_marks);
 workers->run_task(&task);
 #if COMPILER2_OR_JVMCI
 DerivedPointerTable::update_pointers();
 #endif
 }
 _heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
 void do_object(oop p) {
 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
 size_t size = (size_t)p->size();
-HeapWord* compact_to = ShenandoahForwarding::get_forwardee_raw(p);
+if (p->is_forwarded()) {
 HeapWord* compact_from = (HeapWord*) p;
-if (compact_from != compact_to) {
+HeapWord* compact_to = (HeapWord*) p->forwardee();
 Copy::aligned_conjoint_words(compact_from, compact_to, size);
-}
+oop new_obj = oop(compact_to);
-oop new_obj = oop(compact_to);
+new_obj->init_mark_raw();
-ShenandoahForwarding::initialize(new_obj);
+}
 }
 };
 class ShenandoahCompactObjectsTask : public AbstractGangTask {
 private:
 // humongous regions are already compacted, and do not require further moves, which alleviates
 // sliding costs. We may consider doing this in parallel in future.
 ShenandoahHeap* heap = ShenandoahHeap::heap();
-for (size_t c = heap->num_regions() - 1; c > 0; c--) {
+for (size_t c = heap->num_regions(); c > 0; c--) {
-ShenandoahHeapRegion* r = heap->get_region(c);
+ShenandoahHeapRegion* r = heap->get_region(c - 1);
 if (r->is_humongous_start()) {
-oop old_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
+oop old_obj = oop(r->bottom());
-size_t words_size = old_obj->size() + ShenandoahForwarding::word_size();
+if (!old_obj->is_forwarded()) {
+// No need to move the object, it stays at the same slot
+continue;
+}
+size_t words_size = old_obj->size();
 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
 size_t old_start = r->region_number();
 size_t old_end   = old_start + num_regions - 1;
-size_t new_start = heap->heap_region_index_containing(ShenandoahForwarding::get_forwardee_raw(old_obj));
+size_t new_start = heap->heap_region_index_containing(old_obj->forwardee());
 size_t new_end   = new_start + num_regions - 1;
+assert(old_start != new_start, "must be real move");
-if (old_start == new_start) {
-// No need to move the object, it stays at the same slot
-continue;
-}
 assert (r->is_move_allowed(), "should be movable");
 Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(),
 heap->get_region(new_start)->bottom(),
 ShenandoahHeapRegion::region_size_words()*num_regions);
-oop new_obj = oop(heap->get_region(new_start)->bottom() + ShenandoahForwarding::word_size());
+oop new_obj = oop(heap->get_region(new_start)->bottom());
-ShenandoahForwarding::initialize(new_obj);
+new_obj->init_mark_raw();
 {
 for (size_t c = old_start; c <= old_end; c++) {
 ShenandoahHeapRegion* r = heap->get_region(c);
 r->make_regular_bypass();

changeset 55076	785a12e0f89b
parent 54924	ba1eccda5450
child 55513	be05771cdfdf
child 58678	9cf78a70fa4f