--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalAggressiveHeuristics.cpp Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalAggressiveHeuristics.cpp Mon Oct 21 15:11:43 2019 +0200
@@ -75,6 +75,9 @@
RegionData *data = get_region_data_cache(heap->num_regions());
size_t cnt = 0;
+ // About to choose the collection set, make sure we have pinned regions in correct state
+ heap->assert_pinned_region_status();
+
// Step 0. Prepare all regions
for (size_t i = 0; i < heap->num_regions(); i++) {
ShenandoahHeapRegion* r = heap->get_region(i);
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahTraversalHeuristics.cpp Mon Oct 21 15:11:43 2019 +0200
@@ -59,6 +59,9 @@
RegionData *data = get_region_data_cache(heap->num_regions());
size_t cnt = 0;
+ // About to choose the collection set, make sure we have pinned regions in correct state
+ heap->assert_pinned_region_status();
+
// Step 0. Prepare all regions
for (size_t i = 0; i < heap->num_regions(); i++) {
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp Mon Oct 21 15:11:43 2019 +0200
@@ -1507,6 +1507,14 @@
make_parsable(true);
}
+ // We are about to select the collection set, make sure it knows about
+ // current pinning status. Also, this allows trashing more regions that
+ // now have their pinning status dropped.
+ {
+ ShenandoahGCPhase phase(ShenandoahPhaseTimings::sync_pinned);
+ sync_pinned_region_status();
+ }
+
// Trash the collection set left over from previous cycle, if any.
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::trash_cset);
@@ -1784,6 +1792,7 @@
// it, we fail degeneration right away and slide into Full GC to recover.
{
+ sync_pinned_region_status();
collection_set()->clear_current_index();
ShenandoahHeapRegion* r;
@@ -2148,16 +2157,45 @@
}
oop ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
- ShenandoahHeapLocker locker(lock());
- heap_region_containing(o)->make_pinned();
+ heap_region_containing(o)->record_pin();
return o;
}
void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
+ heap_region_containing(o)->record_unpin();
+}
+
+void ShenandoahHeap::sync_pinned_region_status() {
ShenandoahHeapLocker locker(lock());
- heap_region_containing(o)->make_unpinned();
+
+ for (size_t i = 0; i < num_regions(); i++) {
+ ShenandoahHeapRegion *r = get_region(i);
+ if (r->is_active()) {
+ if (r->is_pinned()) {
+ if (r->pin_count() == 0) {
+ r->make_unpinned();
+ }
+ } else {
+ if (r->pin_count() > 0) {
+ r->make_pinned();
+ }
+ }
+ }
+ }
+
+ assert_pinned_region_status();
}
+#ifdef ASSERT
+void ShenandoahHeap::assert_pinned_region_status() {
+ for (size_t i = 0; i < num_regions(); i++) {
+ ShenandoahHeapRegion* r = get_region(i);
+ assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
+ "Region " SIZE_FORMAT " pinning status is inconsistent", i);
+ }
+}
+#endif
+
GCTimer* ShenandoahHeap::gc_timer() const {
return _gc_timer;
}
@@ -2316,6 +2354,13 @@
verifier()->verify_roots_in_to_space();
}
+ // Drop unnecessary "pinned" state from regions that does not have CP marks
+ // anymore, as this would allow trashing them below.
+ {
+ ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_sync_pinned);
+ sync_pinned_region_status();
+ }
+
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_trash_cset);
trash_cset_regions();
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp Mon Oct 21 15:11:43 2019 +0200
@@ -43,6 +43,7 @@
class ShenandoahGCStateResetter;
class ShenandoahHeuristics;
class ShenandoahMarkingContext;
+class ShenandoahMarkCompact;
class ShenandoahMode;
class ShenandoahPhaseTimings;
class ShenandoahHeap;
@@ -574,6 +575,9 @@
oop pin_object(JavaThread* thread, oop obj);
void unpin_object(JavaThread* thread, oop obj);
+ void sync_pinned_region_status();
+ void assert_pinned_region_status() NOT_DEBUG_RETURN;
+
// ---------- Allocation support
//
private:
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp Mon Oct 21 15:11:43 2019 +0200
@@ -59,7 +59,6 @@
_reserved(MemRegion(start, size_words)),
_region_number(index),
_new_top(NULL),
- _critical_pins(0),
_empty_time(os::elapsedTime()),
_state(committed ? _empty_committed : _empty_uncommitted),
_tlab_allocs(0),
@@ -69,7 +68,8 @@
_seqnum_first_alloc_gc(0),
_seqnum_last_alloc_mutator(0),
_seqnum_last_alloc_gc(0),
- _live_data(0) {
+ _live_data(0),
+ _critical_pins(0) {
ContiguousSpace::initialize(_reserved, true, committed);
}
@@ -187,25 +187,21 @@
void ShenandoahHeapRegion::make_pinned() {
_heap->assert_heaplock_owned_by_current_thread();
+ assert(pin_count() > 0, "Should have pins: " SIZE_FORMAT, pin_count());
+
switch (_state) {
case _regular:
- assert (_critical_pins == 0, "sanity");
set_state(_pinned);
case _pinned_cset:
case _pinned:
- _critical_pins++;
return;
case _humongous_start:
- assert (_critical_pins == 0, "sanity");
set_state(_pinned_humongous_start);
case _pinned_humongous_start:
- _critical_pins++;
return;
case _cset:
guarantee(_heap->cancelled_gc(), "only valid when evac has been cancelled");
- assert (_critical_pins == 0, "sanity");
_state = _pinned_cset;
- _critical_pins++;
return;
default:
report_illegal_transition("pinning");
@@ -214,32 +210,21 @@
void ShenandoahHeapRegion::make_unpinned() {
_heap->assert_heaplock_owned_by_current_thread();
+ assert(pin_count() == 0, "Should not have pins: " SIZE_FORMAT, pin_count());
+
switch (_state) {
case _pinned:
- assert (_critical_pins > 0, "sanity");
- _critical_pins--;
- if (_critical_pins == 0) {
- set_state(_regular);
- }
+ set_state(_regular);
return;
case _regular:
case _humongous_start:
- assert (_critical_pins == 0, "sanity");
return;
case _pinned_cset:
guarantee(_heap->cancelled_gc(), "only valid when evac has been cancelled");
- assert (_critical_pins > 0, "sanity");
- _critical_pins--;
- if (_critical_pins == 0) {
- set_state(_cset);
- }
+ set_state(_cset);
return;
case _pinned_humongous_start:
- assert (_critical_pins > 0, "sanity");
- _critical_pins--;
- if (_critical_pins == 0) {
- set_state(_humongous_start);
- }
+ set_state(_humongous_start);
return;
default:
report_illegal_transition("unpinning");
@@ -434,7 +419,7 @@
st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()), proper_unit_for_byte_size(get_gclab_allocs()));
st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()), proper_unit_for_byte_size(get_shared_allocs()));
st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));
- st->print("|CP " SIZE_FORMAT_W(3), _critical_pins);
+ st->print("|CP " SIZE_FORMAT_W(3), pin_count());
st->print("|SN " UINT64_FORMAT_X_W(12) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8),
seqnum_first_alloc_mutator(), seqnum_last_alloc_mutator(),
seqnum_first_alloc_gc(), seqnum_last_alloc_gc());
@@ -702,3 +687,16 @@
}
_state = to;
}
+
+void ShenandoahHeapRegion::record_pin() {
+ Atomic::add((size_t)1, &_critical_pins);
+}
+
+void ShenandoahHeapRegion::record_unpin() {
+ assert(pin_count() > 0, "Region " SIZE_FORMAT " should have non-zero pins", region_number());
+ Atomic::sub((size_t)1, &_critical_pins);
+}
+
+size_t ShenandoahHeapRegion::pin_count() const {
+ return Atomic::load(&_critical_pins);
+}
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp Mon Oct 21 15:11:43 2019 +0200
@@ -203,6 +203,10 @@
RegionState state() const { return _state; }
int state_ordinal() const { return region_state_to_ordinal(_state); }
+ void record_pin();
+ void record_unpin();
+ size_t pin_count() const;
+
private:
static size_t RegionCount;
static size_t RegionSizeBytes;
@@ -238,7 +242,6 @@
// Rarely updated fields
HeapWord* _new_top;
- size_t _critical_pins;
double _empty_time;
// Seldom updated fields
@@ -255,6 +258,7 @@
uint64_t _seqnum_last_alloc_gc;
volatile size_t _live_data;
+ volatile size_t _critical_pins;
// Claim some space at the end to protect next region
DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0);
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeuristics.cpp Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeuristics.cpp Mon Oct 21 15:11:43 2019 +0200
@@ -120,6 +120,9 @@
ShenandoahHeap* heap = ShenandoahHeap::heap();
+ // Check all pinned regions have updated status before choosing the collection set.
+ heap->assert_pinned_region_status();
+
// Step 1. Build up the region candidates we care about, rejecting losers and accepting winners right away.
size_t num_regions = heap->num_regions();
--- a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp Mon Oct 21 15:11:43 2019 +0200
@@ -128,6 +128,9 @@
// e. Set back forwarded objects bit back, in case some steps above dropped it.
heap->set_has_forwarded_objects(has_forwarded_objects);
+ // f. Sync pinned region status from the CP marks
+ heap->sync_pinned_region_status();
+
// The rest of prologue:
BiasedLocking::preserve_marks();
_preserved_marks->init(heap->workers()->active_workers());
@@ -505,6 +508,10 @@
ShenandoahHeap* heap = ShenandoahHeap::heap();
+ // About to figure out which regions can be compacted, make sure pinning status
+ // had been updated in GC prologue.
+ heap->assert_pinned_region_status();
+
{
// Trash the immediately collectible regions before computing addresses
ShenandoahTrashImmediateGarbageClosure tigcl;
--- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp Mon Oct 21 15:11:43 2019 +0200
@@ -101,6 +101,7 @@
f(purge_cldg, " CLDG") \
f(complete_liveness, " Complete Liveness") \
f(retire_tlabs, " Retire TLABs") \
+ f(sync_pinned, " Sync Pinned") \
f(trash_cset, " Trash CSet") \
f(prepare_evac, " Prepare Evacuation") \
\
@@ -161,6 +162,7 @@
f(final_update_refs_string_dedup_queue_roots, " UR: Dedup Queue Roots") \
f(final_update_refs_finish_queues, " UR: Finish Queues") \
\
+ f(final_update_refs_sync_pinned, " Sync Pinned") \
f(final_update_refs_trash_cset, " Trash CSet") \
\
f(degen_gc_gross, "Pause Degenerated GC (G)") \
@@ -193,6 +195,7 @@
f(traversal_gc_prepare, " Prepare") \
f(traversal_gc_make_parsable, " Make Parsable") \
f(traversal_gc_resize_tlabs, " Resize TLABs") \
+ f(traversal_gc_prepare_sync_pinned, " Sync Pinned") \
\
/* Per-thread timer block, should have "roots" counters in consistent order */ \
f(init_traversal_gc_work, " Work") \
@@ -264,6 +267,7 @@
f(final_traversal_update_string_dedup_queue_roots, " TU: Dedup Queue Roots") \
f(final_traversal_update_finish_queues, " TU: Finish Queues") \
\
+ f(traversal_gc_sync_pinned, " Sync Pinned") \
f(traversal_gc_cleanup, " Cleanup") \
\
f(full_gc_gross, "Pause Full GC (G)") \
--- a/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp Mon Oct 21 15:11:42 2019 +0200
+++ b/src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp Mon Oct 21 15:11:43 2019 +0200
@@ -340,9 +340,6 @@
}
void ShenandoahTraversalGC::prepare() {
- _heap->collection_set()->clear();
- assert(_heap->collection_set()->count() == 0, "collection set not clear");
-
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::traversal_gc_make_parsable);
_heap->make_parsable(true);
@@ -356,15 +353,26 @@
assert(_heap->marking_context()->is_bitmap_clear(), "need clean mark bitmap");
assert(!_heap->marking_context()->is_complete(), "should not be complete");
- ShenandoahFreeSet* free_set = _heap->free_set();
+ // About to choose the collection set, make sure we know which regions are pinned.
+ {
+ ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_prepare_sync_pinned);
+ _heap->sync_pinned_region_status();
+ }
+
ShenandoahCollectionSet* collection_set = _heap->collection_set();
+ {
+ ShenandoahHeapLocker lock(_heap->lock());
- // Find collection set
- _heap->heuristics()->choose_collection_set(collection_set);
- prepare_regions();
+ collection_set->clear();
+ assert(collection_set->count() == 0, "collection set not clear");
- // Rebuild free set
- free_set->rebuild();
+ // Find collection set
+ _heap->heuristics()->choose_collection_set(collection_set);
+ prepare_regions();
+
+ // Rebuild free set
+ _heap->free_set()->rebuild();
+ }
log_info(gc, ergo)("Collectable Garbage: " SIZE_FORMAT "%s, " SIZE_FORMAT "%s CSet, " SIZE_FORMAT " CSet regions",
byte_size_in_proper_unit(collection_set->garbage()), proper_unit_for_byte_size(collection_set->garbage()),
@@ -385,7 +393,6 @@
{
ShenandoahGCPhase phase_prepare(ShenandoahPhaseTimings::traversal_gc_prepare);
- ShenandoahHeapLocker lock(_heap->lock());
prepare();
}
@@ -613,6 +620,13 @@
// Resize metaspace
MetaspaceGC::compute_new_size();
+ // Need to see that pinned region status is updated: newly pinned regions must not
+ // be trashed. New unpinned regions should be trashed.
+ {
+ ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_sync_pinned);
+ _heap->sync_pinned_region_status();
+ }
+
// Still good? We can now trash the cset, and make final verification
{
ShenandoahGCPhase phase_cleanup(ShenandoahPhaseTimings::traversal_gc_cleanup);