8180415: Rebuild remembered sets during the concurrent cycle
Summary: In general maintain remembered sets of old regions only from the start of the concurrent cycle to the mixed gc they are used, at most until the end of the mixed phase.
Reviewed-by: sjohanss, sangheki
--- a/src/hotspot/share/gc/g1/collectionSetChooser.cpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/collectionSetChooser.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -25,6 +25,7 @@
#include "precompiled.hpp"
#include "gc/g1/collectionSetChooser.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/heapRegionRemSet.hpp"
#include "gc/shared/space.inline.hpp"
#include "runtime/atomic.hpp"
@@ -147,6 +148,8 @@
assert(!hr->is_pinned(),
"Pinned region shouldn't be added to the collection set (index %u)", hr->hrm_index());
assert(!hr->is_young(), "should not be young!");
+ assert(hr->rem_set()->is_complete(),
+ "Trying to add region %u to the collection set with incomplete remembered set", hr->hrm_index());
_regions.append(hr);
_end++;
_remaining_reclaimable_bytes += hr->reclaimable_bytes();
@@ -237,6 +240,10 @@
// before we fill them up).
if (_cset_updater.should_add(r) && !_g1h->is_old_gc_alloc_region(r)) {
_cset_updater.add_region(r);
+ } else if (r->is_old()) {
+ // Can clean out the remembered sets of all regions that we did not choose but
+ // we created the remembered set for.
+ r->rem_set()->clear(true);
}
}
return false;
@@ -276,7 +283,8 @@
assert(hr->is_marked(), "pre-condition");
assert(!hr->is_young(), "should never consider young regions");
return !hr->is_pinned() &&
- region_occupancy_low_enough_for_evac(hr->live_bytes());
+ region_occupancy_low_enough_for_evac(hr->live_bytes()) &&
+ hr->rem_set()->is_complete();
}
void CollectionSetChooser::rebuild(WorkGang* workers, uint n_regions) {
--- a/src/hotspot/share/gc/g1/collectionSetChooser.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/collectionSetChooser.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -112,6 +112,7 @@
// Determine whether to add the given region to the CSet chooser or
// not. Currently, we skip pinned regions and regions whose live
// bytes are over the threshold. Humongous regions may be reclaimed during cleanup.
+ // Regions also need a complete remembered set to be a candidate.
bool should_add(HeapRegion* hr) const;
// Returns the number candidate old regions added
--- a/src/hotspot/share/gc/g1/concurrentMarkThread.cpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/concurrentMarkThread.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -30,6 +30,7 @@
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1MMUTracker.hpp"
#include "gc/g1/g1Policy.hpp"
+#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/vm_operations_g1.hpp"
#include "gc/shared/concurrentGCPhaseManager.hpp"
#include "gc/shared/gcId.hpp"
@@ -48,19 +49,19 @@
STATIC_ASSERT(ConcurrentGCPhaseManager::UNCONSTRAINED_PHASE <
ConcurrentGCPhaseManager::IDLE_PHASE);
-#define EXPAND_CONCURRENT_PHASES(expander) \
- expander(ANY, = ConcurrentGCPhaseManager::UNCONSTRAINED_PHASE, NULL) \
- expander(IDLE, = ConcurrentGCPhaseManager::IDLE_PHASE, NULL) \
- expander(CONCURRENT_CYCLE,, "Concurrent Cycle") \
- expander(CLEAR_CLAIMED_MARKS,, "Concurrent Clear Claimed Marks") \
- expander(SCAN_ROOT_REGIONS,, "Concurrent Scan Root Regions") \
- expander(CONCURRENT_MARK,, "Concurrent Mark") \
- expander(MARK_FROM_ROOTS,, "Concurrent Mark From Roots") \
- expander(BEFORE_REMARK,, NULL) \
- expander(REMARK,, NULL) \
- expander(CREATE_LIVE_DATA,, "Concurrent Create Live Data") \
- expander(COMPLETE_CLEANUP,, "Concurrent Complete Cleanup") \
- expander(CLEANUP_FOR_NEXT_MARK,, "Concurrent Cleanup for Next Mark") \
+#define EXPAND_CONCURRENT_PHASES(expander) \
+ expander(ANY, = ConcurrentGCPhaseManager::UNCONSTRAINED_PHASE, NULL) \
+ expander(IDLE, = ConcurrentGCPhaseManager::IDLE_PHASE, NULL) \
+ expander(CONCURRENT_CYCLE,, "Concurrent Cycle") \
+ expander(CLEAR_CLAIMED_MARKS,, "Concurrent Clear Claimed Marks") \
+ expander(SCAN_ROOT_REGIONS,, "Concurrent Scan Root Regions") \
+ expander(CONCURRENT_MARK,, "Concurrent Mark") \
+ expander(MARK_FROM_ROOTS,, "Concurrent Mark From Roots") \
+ expander(BEFORE_REMARK,, NULL) \
+ expander(REMARK,, NULL) \
+ expander(REBUILD_REMEMBERED_SETS,, "Concurrent Rebuild Remembered Sets") \
+ expander(COMPLETE_CLEANUP,, "Concurrent Complete Cleanup") \
+ expander(CLEANUP_FOR_NEXT_MARK,, "Concurrent Cleanup for Next Mark") \
/* */
class G1ConcurrentPhase : public AllStatic {
@@ -108,8 +109,8 @@
_cm(cm) {}
void do_void(){
- _cm->cleanup();
- }
+ _cm->cleanup();
+ }
};
double ConcurrentMarkThread::mmu_sleep_time(G1Policy* g1_policy, bool remark) {
@@ -350,8 +351,8 @@
}
if (!cm()->has_aborted()) {
- G1ConcPhase p(G1ConcurrentPhase::CREATE_LIVE_DATA, this);
- cm()->create_live_data();
+ G1ConcPhase p(G1ConcurrentPhase::REBUILD_REMEMBERED_SETS, this);
+ cm()->rebuild_rem_set_concurrently();
}
double end_time = os::elapsedVTime();
--- a/src/hotspot/share/gc/g1/g1Allocator.cpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1Allocator.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -27,8 +27,10 @@
#include "gc/g1/g1AllocRegion.inline.hpp"
#include "gc/g1/g1EvacStats.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1Policy.hpp"
#include "gc/g1/heapRegion.inline.hpp"
#include "gc/g1/heapRegionSet.inline.hpp"
+#include "gc/g1/heapRegionType.hpp"
#include "utilities/align.hpp"
G1DefaultAllocator::G1DefaultAllocator(G1CollectedHeap* heap) :
@@ -342,6 +344,7 @@
} else {
hr->set_closed_archive();
}
+ _g1h->g1_policy()->remset_tracker()->update_at_allocate(hr);
_g1h->old_set_add(hr);
_g1h->hr_printer()->alloc(hr);
_allocated_regions.append(hr);
--- a/src/hotspot/share/gc/g1/g1CardLiveData.cpp Mon Mar 26 16:51:43 2018 +0200
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,591 +0,0 @@
-/*
- * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "gc/g1/g1CollectedHeap.inline.hpp"
-#include "gc/g1/g1ConcurrentMark.inline.hpp"
-#include "gc/g1/g1CardLiveData.inline.hpp"
-#include "gc/shared/suspendibleThreadSet.hpp"
-#include "gc/shared/workgroup.hpp"
-#include "logging/log.hpp"
-#include "memory/resourceArea.hpp"
-#include "memory/universe.hpp"
-#include "runtime/atomic.hpp"
-#include "runtime/globals.hpp"
-#include "runtime/os.hpp"
-#include "utilities/align.hpp"
-#include "utilities/bitMap.inline.hpp"
-#include "utilities/debug.hpp"
-
-G1CardLiveData::G1CardLiveData() :
- _max_capacity(0),
- _cards_per_region(0),
- _gc_timestamp_at_create(0),
- _live_regions(NULL),
- _live_regions_size_in_bits(0),
- _live_cards(NULL),
- _live_cards_size_in_bits(0) {
-}
-
-G1CardLiveData::~G1CardLiveData() {
- free_large_bitmap(_live_cards, _live_cards_size_in_bits);
- free_large_bitmap(_live_regions, _live_regions_size_in_bits);
-}
-
-G1CardLiveData::bm_word_t* G1CardLiveData::allocate_large_bitmap(size_t size_in_bits) {
- size_t size_in_words = BitMap::calc_size_in_words(size_in_bits);
-
- bm_word_t* map = MmapArrayAllocator<bm_word_t>::allocate(size_in_words, mtGC);
-
- return map;
-}
-
-void G1CardLiveData::free_large_bitmap(bm_word_t* bitmap, size_t size_in_bits) {
- MmapArrayAllocator<bm_word_t>::free(bitmap, BitMap::calc_size_in_words(size_in_bits));
-}
-
-void G1CardLiveData::initialize(size_t max_capacity, uint num_max_regions) {
- assert(max_capacity % num_max_regions == 0,
- "Given capacity must be evenly divisible by region size.");
- size_t region_size = max_capacity / num_max_regions;
- assert(region_size % (G1CardTable::card_size * BitsPerWord) == 0,
- "Region size must be evenly divisible by area covered by a single word.");
- _max_capacity = max_capacity;
- _cards_per_region = region_size / G1CardTable::card_size;
-
- _live_regions_size_in_bits = live_region_bitmap_size_in_bits();
- _live_regions = allocate_large_bitmap(_live_regions_size_in_bits);
- _live_cards_size_in_bits = live_card_bitmap_size_in_bits();
- _live_cards = allocate_large_bitmap(_live_cards_size_in_bits);
-}
-
-void G1CardLiveData::pretouch() {
- live_cards_bm().pretouch();
- live_regions_bm().pretouch();
-}
-
-size_t G1CardLiveData::live_region_bitmap_size_in_bits() const {
- return _max_capacity / (_cards_per_region << G1CardTable::card_shift);
-}
-
-size_t G1CardLiveData::live_card_bitmap_size_in_bits() const {
- return _max_capacity >> G1CardTable::card_shift;
-}
-
-// Helper class that provides functionality to generate the Live Data Count
-// information.
-class G1CardLiveDataHelper {
-private:
- BitMapView _region_bm;
- BitMapView _card_bm;
-
- // The card number of the bottom of the G1 heap.
- // Used in biasing indices into accounting card bitmaps.
- BitMap::idx_t _heap_card_bias;
-
- // Utility routine to set an exclusive range of bits on the given
- // bitmap, optimized for very small ranges.
- // There must be at least one bit to set.
- void set_card_bitmap_range(BitMap::idx_t start_idx,
- BitMap::idx_t end_idx) {
-
- // Set the exclusive bit range [start_idx, end_idx).
- assert((end_idx - start_idx) > 0, "at least one bit");
-
- // For small ranges use a simple loop; otherwise use set_range.
- // The range is made up of the cards that are spanned by an object/mem
- // region so 8 cards will allow up to object sizes up to 4K to be handled
- // using the loop.
- if ((end_idx - start_idx) <= 8) {
- for (BitMap::idx_t i = start_idx; i < end_idx; i += 1) {
- _card_bm.set_bit(i);
- }
- } else {
- _card_bm.set_range(start_idx, end_idx);
- }
- }
-
- // We cache the last mark set. This avoids setting the same bit multiple times.
- // This is particularly interesting for dense bitmaps, as this avoids doing
- // lots of work most of the time.
- BitMap::idx_t _last_marked_bit_idx;
-
- void clear_card_bitmap_range(HeapWord* start, HeapWord* end) {
- BitMap::idx_t start_idx = card_live_bitmap_index_for(start);
- BitMap::idx_t end_idx = card_live_bitmap_index_for(align_up(end, CardTable::card_size));
-
- _card_bm.clear_range(start_idx, end_idx);
- }
-
- // Mark the card liveness bitmap for the object spanning from start to end.
- void mark_card_bitmap_range(HeapWord* start, HeapWord* end) {
- BitMap::idx_t start_idx = card_live_bitmap_index_for(start);
- BitMap::idx_t end_idx = card_live_bitmap_index_for(align_up(end, CardTable::card_size));
-
- assert((end_idx - start_idx) > 0, "Trying to mark zero sized range.");
-
- if (start_idx == _last_marked_bit_idx) {
- start_idx++;
- }
- if (start_idx == end_idx) {
- return;
- }
-
- // Set the bits in the card bitmap for the cards spanned by this object.
- set_card_bitmap_range(start_idx, end_idx);
- _last_marked_bit_idx = end_idx - 1;
- }
-
- void reset_mark_cache() {
- _last_marked_bit_idx = (BitMap::idx_t)-1;
- }
-
-public:
- // Returns the index in the per-card liveness count bitmap
- // for the given address
- inline BitMap::idx_t card_live_bitmap_index_for(HeapWord* addr) {
- // Below, the term "card num" means the result of shifting an address
- // by the card shift -- address 0 corresponds to card number 0. One
- // must subtract the card num of the bottom of the heap to obtain a
- // card table index.
- BitMap::idx_t card_num = uintptr_t(addr) >> G1CardTable::card_shift;
- return card_num - _heap_card_bias;
- }
-
- // Takes a region that's not empty (i.e., it has at least one
- // live object in it and sets its corresponding bit on the region
- // bitmap to 1.
- void set_bit_for_region(HeapRegion* hr) {
- _region_bm.par_set_bit(hr->hrm_index());
- }
-
- void reset_live_data(HeapRegion* hr) {
- clear_card_bitmap_range(hr->next_top_at_mark_start(), hr->end());
- }
-
- // Mark the range of bits covered by allocations done since the last marking
- // in the given heap region, i.e. from NTAMS to top of the given region.
- // Returns if there has been some allocation in this region since the last marking.
- bool mark_allocated_since_marking(HeapRegion* hr) {
- reset_mark_cache();
-
- HeapWord* ntams = hr->next_top_at_mark_start();
- HeapWord* top = hr->top();
-
- assert(hr->bottom() <= ntams && ntams <= hr->end(), "Preconditions.");
-
- // Mark the allocated-since-marking portion...
- if (ntams < top) {
- mark_card_bitmap_range(ntams, top);
- return true;
- } else {
- return false;
- }
- }
-
- // Mark the range of bits covered by live objects on the mark bitmap between
- // bottom and NTAMS of the given region.
- // Returns the number of live bytes marked within that area for the given
- // heap region.
- size_t mark_marked_during_marking(G1CMBitMap* mark_bitmap, HeapRegion* hr) {
- reset_mark_cache();
-
- size_t marked_bytes = 0;
-
- HeapWord* ntams = hr->next_top_at_mark_start();
- HeapWord* start = hr->bottom();
-
- if (ntams <= start) {
- // Skip empty regions.
- return 0;
- }
- if (hr->is_humongous()) {
- HeapRegion* start_region = hr->humongous_start_region();
- if (mark_bitmap->is_marked(start_region->bottom())) {
- mark_card_bitmap_range(start, hr->top());
- return pointer_delta(hr->top(), start, 1);
- } else {
- // Humongous start object was actually dead.
- return 0;
- }
- }
-
- assert(start <= hr->end() && start <= ntams && ntams <= hr->end(),
- "Preconditions not met - "
- "start: " PTR_FORMAT ", ntams: " PTR_FORMAT ", end: " PTR_FORMAT,
- p2i(start), p2i(ntams), p2i(hr->end()));
-
- // Find the first marked object at or after "start".
- start = mark_bitmap->get_next_marked_addr(start, ntams);
- while (start < ntams) {
- oop obj = oop(start);
- size_t obj_size = obj->size();
- HeapWord* obj_end = start + obj_size;
-
- assert(obj_end <= hr->end(), "Humongous objects must have been handled elsewhere.");
-
- mark_card_bitmap_range(start, obj_end);
-
- // Add the size of this object to the number of marked bytes.
- marked_bytes += obj_size * HeapWordSize;
-
- // Find the next marked object after this one.
- start = mark_bitmap->get_next_marked_addr(obj_end, ntams);
- }
-
- return marked_bytes;
- }
-
- G1CardLiveDataHelper(G1CardLiveData* live_data, HeapWord* base_address) :
- _region_bm(live_data->live_regions_bm()),
- _card_bm(live_data->live_cards_bm()) {
- // Calculate the card number for the bottom of the heap. Used
- // in biasing indexes into the accounting card bitmaps.
- _heap_card_bias =
- uintptr_t(base_address) >> G1CardTable::card_shift;
- }
-};
-
-class G1CreateCardLiveDataTask: public AbstractGangTask {
- // Aggregate the counting data that was constructed concurrently
- // with marking.
- class G1CreateLiveDataClosure : public HeapRegionClosure {
- G1CardLiveDataHelper _helper;
-
- G1CMBitMap* _mark_bitmap;
-
- G1ConcurrentMark* _cm;
- public:
- G1CreateLiveDataClosure(G1CollectedHeap* g1h,
- G1ConcurrentMark* cm,
- G1CMBitMap* mark_bitmap,
- G1CardLiveData* live_data) :
- HeapRegionClosure(),
- _helper(live_data, g1h->reserved_region().start()),
- _mark_bitmap(mark_bitmap),
- _cm(cm) { }
-
- bool do_heap_region(HeapRegion* hr) {
- size_t marked_bytes = _helper.mark_marked_during_marking(_mark_bitmap, hr);
- if (marked_bytes > 0) {
- hr->add_to_marked_bytes(marked_bytes);
- assert(!hr->is_old() || marked_bytes == (_cm->liveness(hr->hrm_index()) * HeapWordSize),
- "Marked bytes " SIZE_FORMAT " for region %u do not match liveness during mark " SIZE_FORMAT,
- marked_bytes, hr->hrm_index(), _cm->liveness(hr->hrm_index()) * HeapWordSize);
- }
-
- return (_cm->do_yield_check() && _cm->has_aborted());
- }
- };
-
- G1ConcurrentMark* _cm;
- G1CardLiveData* _live_data;
- HeapRegionClaimer _hr_claimer;
-
-public:
- G1CreateCardLiveDataTask(G1CMBitMap* bitmap,
- G1CardLiveData* live_data,
- uint n_workers) :
- AbstractGangTask("G1 Create Live Data"),
- _live_data(live_data),
- _hr_claimer(n_workers) {
- }
-
- void work(uint worker_id) {
- SuspendibleThreadSetJoiner sts_join;
-
- G1CollectedHeap* g1h = G1CollectedHeap::heap();
- G1ConcurrentMark* cm = g1h->concurrent_mark();
- G1CreateLiveDataClosure cl(g1h, cm, cm->next_mark_bitmap(), _live_data);
- g1h->heap_region_par_iterate_from_worker_offset(&cl, &_hr_claimer, worker_id);
- }
-};
-
-void G1CardLiveData::create(WorkGang* workers, G1CMBitMap* mark_bitmap) {
- _gc_timestamp_at_create = G1CollectedHeap::heap()->get_gc_time_stamp();
-
- uint n_workers = workers->active_workers();
-
- G1CreateCardLiveDataTask cl(mark_bitmap,
- this,
- n_workers);
- workers->run_task(&cl);
-}
-
-class G1FinalizeCardLiveDataTask: public AbstractGangTask {
- // Finalizes the liveness counting data.
- // Sets the bits corresponding to the interval [NTAMS, top]
- // (which contains the implicitly live objects) in the
- // card liveness bitmap. Also sets the bit for each region
- // containing live data, in the region liveness bitmap.
- class G1FinalizeCardLiveDataClosure: public HeapRegionClosure {
- private:
- G1CardLiveDataHelper _helper;
-
- uint _gc_timestamp_at_create;
-
- bool has_been_reclaimed(HeapRegion* hr) const {
- return hr->get_gc_time_stamp() > _gc_timestamp_at_create;
- }
- public:
- G1FinalizeCardLiveDataClosure(G1CollectedHeap* g1h,
- G1CMBitMap* bitmap,
- G1CardLiveData* live_data) :
- HeapRegionClosure(),
- _helper(live_data, g1h->reserved_region().start()),
- _gc_timestamp_at_create(live_data->gc_timestamp_at_create()) { }
-
- bool do_heap_region(HeapRegion* hr) {
- if (has_been_reclaimed(hr)) {
- _helper.reset_live_data(hr);
- }
- bool allocated_since_marking = _helper.mark_allocated_since_marking(hr);
- if (allocated_since_marking || hr->next_marked_bytes() > 0) {
- _helper.set_bit_for_region(hr);
- }
- return false;
- }
- };
-
- G1CMBitMap* _bitmap;
-
- G1CardLiveData* _live_data;
-
- HeapRegionClaimer _hr_claimer;
-
-public:
- G1FinalizeCardLiveDataTask(G1CMBitMap* bitmap, G1CardLiveData* live_data, uint n_workers) :
- AbstractGangTask("G1 Finalize Card Live Data"),
- _bitmap(bitmap),
- _live_data(live_data),
- _hr_claimer(n_workers) {
- }
-
- void work(uint worker_id) {
- G1FinalizeCardLiveDataClosure cl(G1CollectedHeap::heap(), _bitmap, _live_data);
-
- G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&cl, &_hr_claimer, worker_id);
- }
-};
-
-void G1CardLiveData::finalize(WorkGang* workers, G1CMBitMap* mark_bitmap) {
- // Finalize the live data.
- G1FinalizeCardLiveDataTask cl(mark_bitmap,
- this,
- workers->active_workers());
- workers->run_task(&cl);
-}
-
-class G1ClearCardLiveDataTask : public AbstractGangTask {
- BitMapView _bitmap;
- size_t _num_chunks;
- size_t _cur_chunk;
-public:
- G1ClearCardLiveDataTask(const BitMapView& bitmap, size_t num_tasks) :
- AbstractGangTask("G1 Clear Card Live Data"),
- _bitmap(bitmap),
- _num_chunks(num_tasks),
- _cur_chunk(0) {
- }
-
- static size_t chunk_size() { return M; }
-
- virtual void work(uint worker_id) {
- while (true) {
- size_t to_process = Atomic::add(1u, &_cur_chunk) - 1;
- if (to_process >= _num_chunks) {
- break;
- }
-
- BitMap::idx_t start = M * BitsPerByte * to_process;
- BitMap::idx_t end = MIN2(start + M * BitsPerByte, _bitmap.size());
- _bitmap.clear_range(start, end);
- }
- }
-};
-
-void G1CardLiveData::clear(WorkGang* workers) {
- guarantee(Universe::is_fully_initialized(), "Should not call this during initialization.");
-
- size_t const num_chunks = align_up(live_cards_bm().size_in_bytes(), G1ClearCardLiveDataTask::chunk_size()) / G1ClearCardLiveDataTask::chunk_size();
- uint const num_workers = (uint)MIN2(num_chunks, (size_t)workers->active_workers());
-
- G1ClearCardLiveDataTask cl(live_cards_bm(), num_chunks);
-
- log_debug(gc, ergo)("Running %s using %u workers for " SIZE_FORMAT " work units.", cl.name(), num_workers, num_chunks);
- workers->run_task(&cl, num_workers);
-
- // The region live bitmap is always very small, even for huge heaps. Clear
- // directly.
- live_regions_bm().clear();
-}
-
-class G1VerifyCardLiveDataTask: public AbstractGangTask {
- // Heap region closure used for verifying the live count data
- // that was created concurrently and finalized during
- // the remark pause. This closure is applied to the heap
- // regions during the STW cleanup pause.
- class G1VerifyCardLiveDataClosure: public HeapRegionClosure {
- private:
- G1CollectedHeap* _g1h;
- G1CMBitMap* _mark_bitmap;
- G1CardLiveDataHelper _helper;
-
- G1CardLiveData* _act_live_data;
-
- G1CardLiveData* _exp_live_data;
-
- int _failures;
-
- // Completely recreates the live data count for the given heap region and
- // returns the number of bytes marked.
- size_t create_live_data_count(HeapRegion* hr) {
- size_t bytes_marked = _helper.mark_marked_during_marking(_mark_bitmap, hr);
- bool allocated_since_marking = _helper.mark_allocated_since_marking(hr);
- if (allocated_since_marking || bytes_marked > 0) {
- _helper.set_bit_for_region(hr);
- }
- return bytes_marked;
- }
- public:
- G1VerifyCardLiveDataClosure(G1CollectedHeap* g1h,
- G1CMBitMap* mark_bitmap,
- G1CardLiveData* act_live_data,
- G1CardLiveData* exp_live_data) :
- _g1h(g1h),
- _mark_bitmap(mark_bitmap),
- _helper(exp_live_data, g1h->reserved_region().start()),
- _act_live_data(act_live_data),
- _exp_live_data(exp_live_data),
- _failures(0) { }
-
- int failures() const { return _failures; }
-
- bool do_heap_region(HeapRegion* hr) {
- int failures = 0;
-
- // Walk the marking bitmap for this region and set the corresponding bits
- // in the expected region and card bitmaps.
- size_t exp_marked_bytes = create_live_data_count(hr);
- size_t act_marked_bytes = hr->next_marked_bytes();
- // Verify the marked bytes for this region.
-
- if (exp_marked_bytes != act_marked_bytes) {
- log_error(gc)("Expected marked bytes " SIZE_FORMAT " != actual marked bytes " SIZE_FORMAT " in region %u", exp_marked_bytes, act_marked_bytes, hr->hrm_index());
- failures += 1;
- } else if (exp_marked_bytes > HeapRegion::GrainBytes) {
- log_error(gc)("Expected marked bytes " SIZE_FORMAT " larger than possible " SIZE_FORMAT " in region %u", exp_marked_bytes, HeapRegion::GrainBytes, hr->hrm_index());
- failures += 1;
- }
-
- // Verify the bit, for this region, in the actual and expected
- // (which was just calculated) region bit maps.
- uint index = hr->hrm_index();
-
- bool expected = _exp_live_data->is_region_live(index);
- bool actual = _act_live_data->is_region_live(index);
- if (expected != actual) {
- log_error(gc)("Expected liveness %d not equal actual %d in region %u", expected, actual, hr->hrm_index());
- failures += 1;
- }
-
- // Verify that the card bit maps for the cards spanned by the current
- // region match.
- BitMap::idx_t start_idx = _helper.card_live_bitmap_index_for(hr->bottom());
- BitMap::idx_t end_idx = _helper.card_live_bitmap_index_for(hr->top());
-
- for (BitMap::idx_t i = start_idx; i < end_idx; i+=1) {
- expected = _exp_live_data->is_card_live_at(i);
- actual = _act_live_data->is_card_live_at(i);
-
- if (expected != actual) {
- log_error(gc)("Expected card liveness %d not equal actual card liveness %d at card " SIZE_FORMAT " in region %u", expected, actual, i, hr->hrm_index());
- failures += 1;
- }
- }
-
- _failures += failures;
-
- // We could stop iteration over the heap when we
- // find the first violating region by returning true.
- return false;
- }
- };
-protected:
- G1CollectedHeap* _g1h;
- G1CMBitMap* _mark_bitmap;
-
- G1CardLiveData* _act_live_data;
-
- G1CardLiveData _exp_live_data;
-
- int _failures;
-
- HeapRegionClaimer _hr_claimer;
-
-public:
- G1VerifyCardLiveDataTask(G1CMBitMap* bitmap,
- G1CardLiveData* act_live_data,
- uint n_workers)
- : AbstractGangTask("G1 Verify Card Live Data"),
- _g1h(G1CollectedHeap::heap()),
- _mark_bitmap(bitmap),
- _act_live_data(act_live_data),
- _exp_live_data(),
- _failures(0),
- _hr_claimer(n_workers) {
- assert(VerifyDuringGC, "don't call this otherwise");
- _exp_live_data.initialize(_g1h->max_capacity(), _g1h->max_regions());
- }
-
- void work(uint worker_id) {
- G1VerifyCardLiveDataClosure cl(_g1h,
- _mark_bitmap,
- _act_live_data,
- &_exp_live_data);
- _g1h->heap_region_par_iterate_from_worker_offset(&cl, &_hr_claimer, worker_id);
-
- Atomic::add(cl.failures(), &_failures);
- }
-
- int failures() const { return _failures; }
-};
-
-void G1CardLiveData::verify(WorkGang* workers, G1CMBitMap* actual_bitmap) {
- ResourceMark rm;
-
- G1VerifyCardLiveDataTask cl(actual_bitmap,
- this,
- workers->active_workers());
- workers->run_task(&cl);
-
- guarantee(cl.failures() == 0, "Unexpected accounting failures");
-}
-
-#ifndef PRODUCT
-void G1CardLiveData::verify_is_clear() {
- assert(live_cards_bm().count_one_bits() == 0, "Live cards bitmap must be clear.");
- assert(live_regions_bm().count_one_bits() == 0, "Live regions bitmap must be clear.");
-}
-#endif
--- a/src/hotspot/share/gc/g1/g1CardLiveData.hpp Mon Mar 26 16:51:43 2018 +0200
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,112 +0,0 @@
-/*
- * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_VM_GC_G1_G1CARDLIVEDATA_HPP
-#define SHARE_VM_GC_G1_G1CARDLIVEDATA_HPP
-
-#include "gc/g1/g1CollectedHeap.hpp"
-#include "utilities/bitMap.hpp"
-#include "utilities/globalDefinitions.hpp"
-
-class G1CollectedHeap;
-class G1CMBitMap;
-class WorkGang;
-
-// Information about object liveness on the Java heap on a "card" basis.
-// Can be used for various purposes, like as remembered set for completely
-// coarsened remembered sets, scrubbing remembered sets or estimating liveness.
-// This information is created as part of the concurrent marking cycle.
-class G1CardLiveData {
- friend class G1CardLiveDataHelper;
- friend class G1VerifyCardLiveDataTask;
-private:
- typedef BitMap::bm_word_t bm_word_t;
- // Store some additional information about the covered area to be able to test.
- size_t _max_capacity;
- size_t _cards_per_region;
-
- // Regions may be reclaimed while concurrently creating live data (e.g. due to humongous
- // eager reclaim). This results in wrong live data for these regions at the end.
- // So we need to somehow detect these regions, and during live data finalization completely
- // recreate their information.
- // This _gc_timestamp_at_create tracks the global timestamp when live data creation
- // has started. Any regions with a higher time stamp have been cleared after that
- // point in time, and need re-finalization.
- // Unsynchronized access to this variable is okay, since this value is only set during a
- // concurrent phase, and read only at the Cleanup safepoint. I.e. there is always
- // full memory synchronization inbetween.
- uint _gc_timestamp_at_create;
- // The per-card liveness bitmap.
- bm_word_t* _live_cards;
- size_t _live_cards_size_in_bits;
- // The per-region liveness bitmap.
- bm_word_t* _live_regions;
- size_t _live_regions_size_in_bits;
- // The bits in this bitmap contain for every card whether it contains
- // at least part of at least one live object.
- BitMapView live_cards_bm() const { return BitMapView(_live_cards, _live_cards_size_in_bits); }
- // The bits in this bitmap indicate that a given region contains some live objects.
- BitMapView live_regions_bm() const { return BitMapView(_live_regions, _live_regions_size_in_bits); }
-
- // Allocate a "large" bitmap from virtual memory with the given size in bits.
- bm_word_t* allocate_large_bitmap(size_t size_in_bits);
- void free_large_bitmap(bm_word_t* map, size_t size_in_bits);
-
- inline BitMapView live_card_bitmap(uint region);
-
- inline bool is_card_live_at(BitMap::idx_t idx) const;
-
- size_t live_region_bitmap_size_in_bits() const;
- size_t live_card_bitmap_size_in_bits() const;
-public:
- uint gc_timestamp_at_create() const { return _gc_timestamp_at_create; }
-
- inline bool is_region_live(uint region) const;
-
- inline void remove_nonlive_cards(uint region, BitMap* bm);
- inline void remove_nonlive_regions(BitMap* bm);
-
- G1CardLiveData();
- ~G1CardLiveData();
-
- void initialize(size_t max_capacity, uint num_max_regions);
- void pretouch();
-
- // Create the initial liveness data based on the marking result from the bottom
- // to the ntams of every region in the heap and the marks in the given bitmap.
- void create(WorkGang* workers, G1CMBitMap* mark_bitmap);
- // Finalize the liveness data.
- void finalize(WorkGang* workers, G1CMBitMap* mark_bitmap);
-
- // Verify that the liveness count data created concurrently matches one created
- // during this safepoint.
- void verify(WorkGang* workers, G1CMBitMap* actual_bitmap);
- // Clear all data structures, prepare for next processing.
- void clear(WorkGang* workers);
-
- void verify_is_clear() PRODUCT_RETURN;
-};
-
-#endif /* SHARE_VM_GC_G1_G1CARDLIVEDATA_HPP */
-
--- a/src/hotspot/share/gc/g1/g1CardLiveData.inline.hpp Mon Mar 26 16:51:43 2018 +0200
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,52 +0,0 @@
-/*
- * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_VM_GC_G1_G1CARDLIVEDATA_INLINE_HPP
-#define SHARE_VM_GC_G1_G1CARDLIVEDATA_INLINE_HPP
-
-#include "gc/g1/g1CardLiveData.hpp"
-#include "utilities/bitMap.inline.hpp"
-#include "utilities/globalDefinitions.hpp"
-
-inline BitMapView G1CardLiveData::live_card_bitmap(uint region) {
- return BitMapView(_live_cards + ((size_t)region * _cards_per_region >> LogBitsPerWord), _cards_per_region);
-}
-
-inline bool G1CardLiveData::is_card_live_at(BitMap::idx_t idx) const {
- return live_cards_bm().at(idx);
-}
-
-inline bool G1CardLiveData::is_region_live(uint region) const {
- return live_regions_bm().at(region);
-}
-
-inline void G1CardLiveData::remove_nonlive_cards(uint region, BitMap* bm) {
- bm->set_intersection(live_card_bitmap(region));
-}
-
-inline void G1CardLiveData::remove_nonlive_regions(BitMap* bm) {
- bm->set_intersection(live_regions_bm());
-}
-
-#endif /* SHARE_VM_GC_G1_G1CARDLIVEDATA_INLINE_HPP */
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -303,12 +303,14 @@
// that there is a single object that starts at the bottom of the
// first region.
first_hr->set_starts_humongous(obj_top, word_fill_size);
+ _g1_policy->remset_tracker()->update_at_allocate(first_hr);
// Then, if there are any, we will set up the "continues
// humongous" regions.
HeapRegion* hr = NULL;
for (uint i = first + 1; i <= last; ++i) {
hr = region_at(i);
hr->set_continues_humongous(first_hr);
+ _g1_policy->remset_tracker()->update_at_allocate(hr);
}
// Up to this point no concurrent thread would have been able to
@@ -2611,6 +2613,11 @@
return false;
}
+ // If we do not have a complete remembered set for the region, then we can
+ // not be sure that we have all references to it.
+ if (!region->rem_set()->is_complete()) {
+ return false;
+ }
// Candidate selection must satisfy the following constraints
// while concurrent marking is in progress:
//
@@ -2696,7 +2703,15 @@
assert(hrrs.n_yielded() == r->rem_set()->occupied(),
"Remembered set hash maps out of sync, cur: " SIZE_FORMAT " entries, next: " SIZE_FORMAT " entries",
hrrs.n_yielded(), r->rem_set()->occupied());
- r->rem_set()->clear_locked();
+ // We should only clear the card based remembered set here as we will not
+ // implicitly rebuild anything else during eager reclaim. Note that at the moment
+ // (and probably never) we do not enter this path if there are other kind of
+ // remembered sets for this region.
+ r->rem_set()->clear_locked(true /* only_cardset */);
+ // Clear_locked() above sets the state to Empty. However we want to continue
+ // collecting remembered set entries for humongous regions that were not
+ // reclaimed.
+ r->rem_set()->set_state_complete();
}
assert(r->rem_set()->is_empty(), "At this point any humongous candidate remembered set must be empty.");
}
@@ -4404,6 +4419,7 @@
_hot_card_cache->reset_card_counts(hr);
}
hr->hr_clear(skip_remset, true /* clear_space */, locked /* locked */);
+ _g1_policy->remset_tracker()->update_at_free(hr);
free_list->add_ordered(hr);
}
@@ -4438,29 +4454,6 @@
decrease_used(bytes);
}
-class G1ParScrubRemSetTask: public AbstractGangTask {
-protected:
- G1RemSet* _g1rs;
- HeapRegionClaimer _hrclaimer;
-
-public:
- G1ParScrubRemSetTask(G1RemSet* g1_rs, uint num_workers) :
- AbstractGangTask("G1 ScrubRS"),
- _g1rs(g1_rs),
- _hrclaimer(num_workers) {
- }
-
- void work(uint worker_id) {
- _g1rs->scrub(worker_id, &_hrclaimer);
- }
-};
-
-void G1CollectedHeap::scrub_rem_set() {
- uint num_workers = workers()->active_workers();
- G1ParScrubRemSetTask g1_par_scrub_rs_task(g1_rem_set(), num_workers);
- workers()->run_task(&g1_par_scrub_rs_task);
-}
-
class G1FreeCollectionSetTask : public AbstractGangTask {
private:
@@ -5053,6 +5046,8 @@
}
bool do_heap_region(HeapRegion* r) {
+ // After full GC, no region should have a remembered set.
+ r->rem_set()->clear(true);
if (r->is_empty()) {
// Add free regions to the free list
r->set_free();
@@ -5120,6 +5115,7 @@
set_region_short_lived_locked(new_alloc_region);
_hr_printer.alloc(new_alloc_region, !should_allocate);
_verifier->check_bitmaps("Mutator Region Allocation", new_alloc_region);
+ _g1_policy->remset_tracker()->update_at_allocate(new_alloc_region);
return new_alloc_region;
}
}
@@ -5175,6 +5171,7 @@
new_alloc_region->set_old();
_verifier->check_bitmaps("Old Region Allocation", new_alloc_region);
}
+ _g1_policy->remset_tracker()->update_at_allocate(new_alloc_region);
_hr_printer.alloc(new_alloc_region);
bool during_im = collector_state()->during_initial_mark_pause();
new_alloc_region->note_start_of_copying(during_im);
--- a/src/hotspot/share/gc/g1/g1CollectorState.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1CollectorState.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -71,6 +71,9 @@
bool _in_marking_window;
bool _in_marking_window_im;
+ // Are we going into a mixed gc phase.
+ bool _mixed_gc_pending;
+
bool _full_collection;
public:
@@ -78,6 +81,7 @@
_gcs_are_young(true),
_last_gc_was_young(false),
_last_young_gc(false),
+ _mixed_gc_pending(false),
_during_initial_mark_pause(false),
_initiate_conc_mark_if_possible(false),
@@ -91,13 +95,14 @@
// Setters
void set_gcs_are_young(bool v) { _gcs_are_young = v; }
void set_last_gc_was_young(bool v) { _last_gc_was_young = v; }
- void set_last_young_gc(bool v) { _last_young_gc = v; }
+ void set_last_young_gc(bool v) { _last_young_gc = v; _mixed_gc_pending = false;}
void set_during_initial_mark_pause(bool v) { _during_initial_mark_pause = v; }
void set_initiate_conc_mark_if_possible(bool v) { _initiate_conc_mark_if_possible = v; }
void set_during_marking(bool v) { _during_marking = v; }
void set_mark_in_progress(bool v) { _mark_in_progress = v; }
void set_in_marking_window(bool v) { _in_marking_window = v; }
void set_in_marking_window_im(bool v) { _in_marking_window_im = v; }
+ void set_mixed_gc_pending(bool v) { _mixed_gc_pending = v; }
void set_full_collection(bool v) { _full_collection = v; }
// Getters
@@ -110,6 +115,7 @@
bool mark_in_progress() const { return _mark_in_progress; }
bool in_marking_window() const { return _in_marking_window; }
bool in_marking_window_im() const { return _in_marking_window_im; }
+ bool mixed_gc_pending() const { return _mixed_gc_pending; }
bool full_collection() const { return _full_collection; }
// Composite booleans (clients worry about flickering)
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -32,7 +32,6 @@
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
-#include "gc/g1/g1CardLiveData.inline.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
#include "gc/g1/g1StringDedup.hpp"
@@ -358,6 +357,7 @@
// _finger set in set_non_marking_state
+ _worker_id_offset(DirtyCardQueueSet::num_par_ids() + G1ConcRefinementThreads),
_max_num_tasks(ParallelGCThreads),
// _num_active_tasks set in set_non_marking_state()
// _tasks set inside the constructor
@@ -384,7 +384,6 @@
_remark_weak_ref_times(),
_cleanup_times(),
_total_counting_time(0.0),
- _total_rs_scrub_time(0.0),
_accum_task_vtime(NULL),
@@ -392,7 +391,8 @@
_num_concurrent_workers(0),
_max_concurrent_workers(0),
- _region_mark_stats(NEW_C_HEAP_ARRAY(G1RegionMarkStats, _g1h->max_regions(), mtGC))
+ _region_mark_stats(NEW_C_HEAP_ARRAY(G1RegionMarkStats, _g1h->max_regions(), mtGC)),
+ _top_at_rebuild_starts(NEW_C_HEAP_ARRAY(HeapWord*, _g1h->max_regions(), mtGC))
{
_mark_bitmap_1.initialize(g1h->reserved_region(), prev_bitmap_storage);
_mark_bitmap_2.initialize(g1h->reserved_region(), next_bitmap_storage);
@@ -424,7 +424,7 @@
return;
}
- log_debug(gc)("ConcGCThreads: %u", ConcGCThreads);
+ log_debug(gc)("ConcGCThreads: %u offset %u", ConcGCThreads, _worker_id_offset);
log_debug(gc)("ParallelGCThreads: %u", ParallelGCThreads);
_num_concurrent_workers = ConcGCThreads;
@@ -514,6 +514,7 @@
uint max_regions = _g1h->max_regions();
for (uint i = 0; i < max_regions; i++) {
+ _top_at_rebuild_starts[i] = NULL;
_region_mark_stats[i].clear();
}
@@ -526,6 +527,7 @@
for (uint j = 0; j < _max_num_tasks; ++j) {
_tasks[j]->clear_mark_stats_cache(region_idx);
}
+ _top_at_rebuild_starts[region_idx] = NULL;
_region_mark_stats[region_idx].clear();
}
@@ -614,6 +616,7 @@
}
G1ConcurrentMark::~G1ConcurrentMark() {
+ FREE_C_HEAP_ARRAY(HeapWord*, _top_at_rebuild_starts);
FREE_C_HEAP_ARRAY(G1RegionMarkStats, _region_mark_stats);
// The G1ConcurrentMark instance is never freed.
ShouldNotReachHere();
@@ -712,13 +715,6 @@
clear_bitmap(_next_mark_bitmap, _concurrent_workers, true);
- // Clear the live count data. If the marking has been aborted, the abort()
- // call already did that.
- if (!has_aborted()) {
- clear_live_data(_concurrent_workers);
- DEBUG_ONLY(verify_live_data_clear());
- }
-
// Repeat the asserts from above.
guarantee(cm_thread()->during_cycle(), "invariant");
guarantee(!_g1h->collector_state()->mark_in_progress(), "invariant");
@@ -1018,6 +1014,46 @@
print_stats();
}
+class G1UpdateRemSetTrackingBeforeRebuild : public HeapRegionClosure {
+ G1CollectedHeap* _g1h;
+ G1ConcurrentMark* _cm;
+
+ uint _num_regions_selected_for_rebuild; // The number of regions actually selected for rebuild.
+
+ void update_remset_before_rebuild(HeapRegion * hr) {
+ G1RemSetTrackingPolicy* tracking_policy = _g1h->g1_policy()->remset_tracker();
+
+ size_t live_bytes = _cm->liveness(hr->hrm_index()) * HeapWordSize;
+ bool selected_for_rebuild = tracking_policy->update_before_rebuild(hr, live_bytes);
+ if (selected_for_rebuild) {
+ _num_regions_selected_for_rebuild++;
+ }
+ _cm->update_top_at_rebuild_start(hr);
+ }
+
+public:
+ G1UpdateRemSetTrackingBeforeRebuild(G1CollectedHeap* g1h, G1ConcurrentMark* cm) :
+ _g1h(g1h), _cm(cm), _num_regions_selected_for_rebuild(0) { }
+
+ virtual bool do_heap_region(HeapRegion* r) {
+ update_remset_before_rebuild(r);
+ return false;
+ }
+
+ uint num_selected_for_rebuild() const { return _num_regions_selected_for_rebuild; }
+};
+
+class G1UpdateRemSetTrackingAfterRebuild : public HeapRegionClosure {
+ G1CollectedHeap* _g1h;
+public:
+ G1UpdateRemSetTrackingAfterRebuild(G1CollectedHeap* g1h) : _g1h(g1h) { }
+
+ virtual bool do_heap_region(HeapRegion* r) {
+ _g1h->g1_policy()->remset_tracker()->update_after_rebuild(r);
+ return false;
+ }
+};
+
void G1ConcurrentMark::checkpoint_roots_final(bool clear_all_soft_refs) {
// world is stopped at this checkpoint
assert(SafepointSynchronize::is_at_safepoint(),
@@ -1032,7 +1068,7 @@
}
if (VerifyDuringGC) {
- g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "During GC (before)");
+ g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "During GC (Remark before)");
}
g1h->verifier()->check_bitmaps("Remark Start");
@@ -1053,7 +1089,7 @@
// Verify the heap w.r.t. the previous marking bitmap.
if (VerifyDuringGC) {
- g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "During GC (overflow)");
+ g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UsePrevMarking, "During GC (Remark overflow)");
}
// Clear the marking state because we will be restarting
@@ -1072,8 +1108,16 @@
flush_all_task_caches();
}
+ {
+ GCTraceTime(Debug, gc, phases)("Update Remembered Set Tracking Before Rebuild");
+ G1UpdateRemSetTrackingBeforeRebuild cl(_g1h, this);
+ g1h->heap_region_iterate(&cl);
+ log_debug(gc, remset, tracking)("Remembered Set Tracking update regions total %u, selected %u",
+ _g1h->num_regions(), cl.num_selected_for_rebuild());
+ }
+
if (VerifyDuringGC) {
- g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UseNextMarking, "During GC (after)");
+ g1h->verifier()->verify(G1HeapVerifier::G1VerifyRemark, VerifyOption_G1UseNextMarking, "During GC (Remark after)");
}
g1h->verifier()->check_bitmaps("Remark End");
assert(!restart_for_overflow(), "sanity");
@@ -1155,7 +1199,7 @@
FreeRegionList local_cleanup_list("Local Cleanup List");
HRRSCleanupTask hrrs_cleanup_task;
G1NoteEndOfConcMarkClosure g1_note_end(_g1h, &local_cleanup_list,
- &hrrs_cleanup_task);
+ &hrrs_cleanup_task);
_g1h->heap_region_par_iterate_from_worker_offset(&g1_note_end, &_hrclaimer, worker_id);
assert(g1_note_end.is_complete(), "Shouldn't have yielded!");
@@ -1204,8 +1248,8 @@
g1h->verifier()->verify_region_sets_optional();
- if (VerifyDuringGC) {
- g1h->verifier()->verify(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UsePrevMarking, "During GC (before)");
+ if (VerifyDuringGC) { // While rebuilding the remembered set we used the next marking...
+ g1h->verifier()->verify(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UseNextMarking, "During GC (Cleanup before)");
}
g1h->verifier()->check_bitmaps("Cleanup Start");
@@ -1217,13 +1261,9 @@
HeapRegionRemSet::reset_for_cleanup_tasks();
{
- GCTraceTime(Debug, gc)("Finalize Live Data");
- finalize_live_data();
- }
-
- if (VerifyDuringGC) {
- GCTraceTime(Debug, gc)("Verify Live Data");
- verify_live_data();
+ GCTraceTime(Debug, gc, phases)("Update Remembered Set Tracking After Rebuild");
+ G1UpdateRemSetTrackingAfterRebuild cl(_g1h);
+ g1h->heap_region_iterate(&cl);
}
g1h->collector_state()->set_mark_in_progress(false);
@@ -1233,7 +1273,7 @@
_total_counting_time += this_final_counting_time;
if (log_is_enabled(Trace, gc, liveness)) {
- G1PrintRegionLivenessInfoClosure cl("Post-Marking");
+ G1PrintRegionLivenessInfoClosure cl("Post-Cleanup");
_g1h->heap_region_iterate(&cl);
}
@@ -1256,18 +1296,13 @@
g1h->set_free_regions_coming();
}
- // call below, since it affects the metric by which we sort the heap
- // regions.
- if (G1ScrubRemSets) {
- double rs_scrub_start = os::elapsedTime();
- g1h->scrub_rem_set();
- _total_rs_scrub_time += (os::elapsedTime() - rs_scrub_start);
+ {
+ GCTraceTime(Debug, gc, phases)("Finalize Concurrent Mark Cleanup");
+ // This will also free any regions totally full of garbage objects,
+ // and sort the regions.
+ g1h->g1_policy()->record_concurrent_mark_cleanup_end();
}
- // this will also free any regions totally full of garbage objects,
- // and sort the regions.
- g1h->g1_policy()->record_concurrent_mark_cleanup_end();
-
// Statistics.
double end = os::elapsedTime();
_cleanup_times.add((end - start) * 1000.0);
@@ -1277,7 +1312,7 @@
Universe::update_heap_info_at_gc();
if (VerifyDuringGC) {
- g1h->verifier()->verify(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UsePrevMarking, "During GC (after)");
+ g1h->verifier()->verify(G1HeapVerifier::G1VerifyCleanup, VerifyOption_G1UsePrevMarking, "During GC (Cleanup after)");
}
g1h->verifier()->check_bitmaps("Cleanup End");
@@ -1963,28 +1998,11 @@
}
}
#endif // PRODUCT
-void G1ConcurrentMark::create_live_data() {
- _g1h->g1_rem_set()->create_card_live_data(_concurrent_workers, _next_mark_bitmap);
-}
-
-void G1ConcurrentMark::finalize_live_data() {
- _g1h->g1_rem_set()->finalize_card_live_data(_g1h->workers(), _next_mark_bitmap);
-}
-
-void G1ConcurrentMark::verify_live_data() {
- _g1h->g1_rem_set()->verify_card_live_data(_g1h->workers(), _next_mark_bitmap);
+
+void G1ConcurrentMark::rebuild_rem_set_concurrently() {
+ _g1h->g1_rem_set()->rebuild_rem_set(this, _concurrent_workers, _worker_id_offset);
}
-void G1ConcurrentMark::clear_live_data(WorkGang* workers) {
- _g1h->g1_rem_set()->clear_card_live_data(workers);
-}
-
-#ifdef ASSERT
-void G1ConcurrentMark::verify_live_data_clear() {
- _g1h->g1_rem_set()->verify_card_live_data_is_clear();
-}
-#endif
-
void G1ConcurrentMark::print_stats() {
if (!log_is_enabled(Debug, gc, stats)) {
return;
@@ -2012,14 +2030,6 @@
// since VerifyDuringGC verifies the objects marked during
// a full GC against the previous bitmap.
- {
- GCTraceTime(Debug, gc)("Clear Live Data");
- clear_live_data(_g1h->workers());
- }
- DEBUG_ONLY({
- GCTraceTime(Debug, gc)("Verify Live Data Clear");
- verify_live_data_clear();
- })
// Empty mark stack
reset_marking_state();
for (uint i = 0; i < _max_num_tasks; ++i) {
@@ -2065,10 +2075,6 @@
print_ms_time_info(" ", "cleanups", _cleanup_times);
log.trace(" Finalize live data total time = %8.2f s (avg = %8.2f ms).",
_total_counting_time, (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
- if (G1ScrubRemSets) {
- log.trace(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
- _total_rs_scrub_time, (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
- }
log.trace(" Total stop_world time = %8.2f s.",
(_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -312,6 +312,7 @@
// always pointing to the end of the
// last claimed region
+ uint _worker_id_offset;
uint _max_num_tasks; // Maximum number of marking tasks
uint _num_active_tasks; // Number of tasks currently active
G1CMTask** _tasks; // Task queue array (max_worker_id length)
@@ -360,7 +361,6 @@
NumberSeq _remark_weak_ref_times;
NumberSeq _cleanup_times;
double _total_counting_time;
- double _total_rs_scrub_time;
double* _accum_task_vtime; // Accumulated task vtime
@@ -455,12 +455,22 @@
void clear_statistics_in_region(uint region_idx);
// Region statistics gathered during marking.
G1RegionMarkStats* _region_mark_stats;
+ // Top pointer for each region at the start of the rebuild remembered set process
+ // for regions which remembered sets need to be rebuilt. A NULL for a given region
+ // means that this region does not be scanned during the rebuilding remembered
+ // set phase at all.
+ HeapWord** _top_at_rebuild_starts;
public:
void add_to_liveness(uint worker_id, oop const obj, size_t size);
// Liveness of the given region as determined by concurrent marking, i.e. the amount of
// live words between bottom and nTAMS.
size_t liveness(uint region) { return _region_mark_stats[region]._live_words; }
+ // Sets the internal top_at_region_start for the given region to current top of the region.
+ inline void update_top_at_rebuild_start(HeapRegion* r);
+ // TARS for the given region during remembered set rebuilding.
+ inline HeapWord* top_at_rebuild_start(uint region) const;
+
// Notification for eagerly reclaimed regions to clean up.
void humongous_object_eagerly_reclaimed(HeapRegion* r);
// Manipulation of the global mark stack.
@@ -606,21 +616,8 @@
G1OldTracer* gc_tracer_cm() const { return _gc_tracer_cm; }
private:
- // Clear (Reset) all liveness count data.
- void clear_live_data(WorkGang* workers);
-
-#ifdef ASSERT
- // Verify all of the above data structures that they are in initial state.
- void verify_live_data_clear();
-#endif
-
- // Aggregates the per-card liveness data based on the current marking. Also sets
- // the amount of marked bytes for each region.
- void create_live_data();
-
- void finalize_live_data();
-
- void verify_live_data();
+ // Rebuilds the remembered sets for chosen regions in parallel and concurrently to the application.
+ void rebuild_rem_set_concurrently();
};
// A class representing a marking task.
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -29,7 +29,10 @@
#include "gc/g1/g1ConcurrentMark.hpp"
#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp"
+#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
+#include "gc/g1/g1RemSetTrackingPolicy.hpp"
+#include "gc/g1/heapRegionRemSet.hpp"
#include "gc/g1/heapRegion.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
#include "gc/shared/taskqueue.inline.hpp"
@@ -163,6 +166,27 @@
return mr.word_size();
}
+inline HeapWord* G1ConcurrentMark::top_at_rebuild_start(uint region) const {
+ assert(region < _g1h->max_regions(), "Tried to access TARS for region %u out of bounds", region);
+ return _top_at_rebuild_starts[region];
+}
+
+inline void G1ConcurrentMark::update_top_at_rebuild_start(HeapRegion* r) {
+ uint const region = r->hrm_index();
+ assert(region < _g1h->max_regions(), "Tried to access TARS for region %u out of bounds", region);
+ assert(_top_at_rebuild_starts[region] == NULL,
+ "TARS for region %u has already been set to " PTR_FORMAT " should be NULL",
+ region, p2i(_top_at_rebuild_starts[region]));
+ G1RemSetTrackingPolicy* tracker = _g1h->g1_policy()->remset_tracker();
+ if (tracker->needs_scan_for_rebuild(r)) {
+ _top_at_rebuild_starts[region] = r->top();
+ } else {
+ // We could leave the TARS for this region at NULL, but we would not catch
+ // accidental double assignment then.
+ _top_at_rebuild_starts[region] = r->bottom();
+ }
+}
+
inline void G1CMTask::update_liveness(oop const obj, const size_t obj_size) {
_mark_stats_cache.add_live_words(_g1h->addr_to_region((HeapWord*)obj), obj_size);
}
--- a/src/hotspot/share/gc/g1/g1EvacFailure.cpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1EvacFailure.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -240,6 +240,7 @@
size_t live_bytes = remove_self_forward_ptr_by_walking_hr(hr, during_initial_mark);
hr->rem_set()->clean_strong_code_roots(hr);
+ hr->rem_set()->clear_locked(true);
hr->note_self_forwarding_removal_end(live_bytes);
}
--- a/src/hotspot/share/gc/g1/g1FullCollector.cpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1FullCollector.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -213,7 +213,7 @@
void G1FullCollector::phase3_adjust_pointers() {
// Adjust the pointers to reflect the new locations
- GCTraceTime(Info, gc, phases) info("Phase 3: Adjust pointers and remembered sets", scope()->timer());
+ GCTraceTime(Info, gc, phases) info("Phase 3: Adjust pointers", scope()->timer());
G1FullGCAdjustTask task(this);
run_task(&task);
--- a/src/hotspot/share/gc/g1/g1FullCollector.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1FullCollector.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
--- a/src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -37,13 +37,12 @@
#include "utilities/ticks.inline.hpp"
class G1AdjustLiveClosure : public StackObj {
- G1AdjustAndRebuildClosure* _adjust_closure;
+ G1AdjustClosure* _adjust_closure;
public:
- G1AdjustLiveClosure(G1AdjustAndRebuildClosure* cl) :
+ G1AdjustLiveClosure(G1AdjustClosure* cl) :
_adjust_closure(cl) { }
size_t apply(oop object) {
- _adjust_closure->update_compaction_delta(object);
return object->oop_iterate_size(_adjust_closure);
}
};
@@ -57,10 +56,9 @@
_worker_id(worker_id) { }
bool do_heap_region(HeapRegion* r) {
- G1AdjustAndRebuildClosure cl(_worker_id);
+ G1AdjustClosure cl;
if (r->is_humongous()) {
oop obj = oop(r->humongous_start_region()->bottom());
- cl.update_compaction_delta(obj);
obj->oop_iterate(&cl, MemRegion(r->bottom(), r->top()));
} else if (r->is_open_archive()) {
// Only adjust the open archive regions, the closed ones
@@ -79,7 +77,7 @@
};
G1FullGCAdjustTask::G1FullGCAdjustTask(G1FullCollector* collector) :
- G1FullGCTask("G1 Adjust and Rebuild", collector),
+ G1FullGCTask("G1 Adjust", collector),
_root_processor(G1CollectedHeap::heap(), collector->workers()),
_hrclaimer(collector->workers()),
_adjust(),
@@ -115,5 +113,5 @@
// Now adjust pointers region by region
G1AdjustRegionClosure blk(collector()->mark_bitmap(), worker_id);
G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&blk, &_hrclaimer, worker_id);
- log_task("Adjust and Rebuild task", worker_id, start);
+ log_task("Adjust task", worker_id, start);
}
--- a/src/hotspot/share/gc/g1/g1FullGCOopClosures.cpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1FullGCOopClosures.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -51,32 +51,6 @@
do_cld_nv(cld);
}
-G1AdjustAndRebuildClosure::G1AdjustAndRebuildClosure(uint worker_id) :
- _worker_id(worker_id),
- _compaction_delta(0),
- _g1h(G1CollectedHeap::heap()) { }
-
-void G1AdjustAndRebuildClosure::update_compaction_delta(oop obj) {
- if (G1ArchiveAllocator::is_open_archive_object(obj)) {
- _compaction_delta = 0;
- return;
- }
- oop forwardee = obj->forwardee();
- if (forwardee == NULL) {
- // Object not moved.
- _compaction_delta = 0;
- } else {
- // Object moved to forwardee, calculate delta.
- _compaction_delta = calculate_compaction_delta(obj, forwardee);
- }
-}
-
-void G1AdjustClosure::do_oop(oop* p) { adjust_pointer(p); }
-void G1AdjustClosure::do_oop(narrowOop* p) { adjust_pointer(p); }
-
-void G1AdjustAndRebuildClosure::do_oop(oop* p) { do_oop_nv(p); }
-void G1AdjustAndRebuildClosure::do_oop(narrowOop* p) { do_oop_nv(p); }
-
void G1FollowStackClosure::do_void() { _marker->drain_stack(); }
void G1FullKeepAliveClosure::do_oop(oop* p) { do_oop_work(p); }
--- a/src/hotspot/share/gc/g1/g1FullGCOopClosures.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1FullGCOopClosures.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -79,43 +79,16 @@
void do_cld_nv(ClassLoaderData* cld);
};
-class G1AdjustClosure : public OopClosure {
+class G1AdjustClosure : public ExtendedOopClosure {
+ template <class T> static inline void adjust_pointer(T* p);
public:
- template <class T> static inline oop adjust_pointer(T* p);
- virtual void do_oop(oop* p);
- virtual void do_oop(narrowOop* p);
-};
-
-class G1AdjustAndRebuildClosure : public ExtendedOopClosure {
- uint _worker_id;
- size_t _compaction_delta;
- G1CollectedHeap* _g1h;
-
- inline size_t calculate_compaction_delta(oop current, oop forwardee);
- template <class T> inline T* add_compaction_delta(T* p);
-
-public:
- G1AdjustAndRebuildClosure(uint worker_id);
-
- void update_compaction_delta(oop obj);
-
- template <class T> inline void add_reference(T* from_field, oop reference, uint worker_id);
- template <class T> void do_oop_nv(T* p);
+ template <class T> void do_oop_nv(T* p) { adjust_pointer(p); }
virtual void do_oop(oop* p);
virtual void do_oop(narrowOop* p);
virtual ReferenceIterationMode reference_iteration_mode() { return DO_FIELDS; }
};
-class G1AdjustObjectClosure {
- G1AdjustAndRebuildClosure* _closure;
-
-public:
- G1AdjustObjectClosure(G1AdjustAndRebuildClosure* cl) : _closure(cl) { }
-
- inline int adjust_object(oop obj);
-};
-
class G1VerifyOopClosure: public OopClosure {
private:
G1CollectedHeap* _g1h;
--- a/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -51,18 +51,17 @@
_marker->follow_cld(cld);
}
-template <class T> inline oop G1AdjustClosure::adjust_pointer(T* p) {
+template <class T> inline void G1AdjustClosure::adjust_pointer(T* p) {
T heap_oop = RawAccess<>::oop_load(p);
if (CompressedOops::is_null(heap_oop)) {
- // NULL reference, return NULL.
- return NULL;
+ return;
}
oop obj = CompressedOops::decode_not_null(heap_oop);
assert(Universe::heap()->is_in(obj), "should be in heap");
if (G1ArchiveAllocator::is_archive_object(obj)) {
- // Never forwarding archive objects, return current reference.
- return obj;
+ // We never forward archive objects.
+ return;
}
oop forwardee = obj->forwardee();
@@ -73,50 +72,16 @@
(UseBiasedLocking && obj->has_bias_pattern()), // Will be restored by BiasedLocking
"Must have correct prototype or be preserved, obj: " PTR_FORMAT ", mark: " PTR_FORMAT ", prototype: " PTR_FORMAT,
p2i(obj), p2i(obj->mark()), p2i(markOopDesc::prototype_for_object(obj)));
- return obj;
- }
-
- // Forwarded, update and return new reference.
- assert(Universe::heap()->is_in_reserved(forwardee), "should be in object space");
- RawAccess<OOP_NOT_NULL>::oop_store(p, forwardee);
- return forwardee;
-}
-
-template <class T>
-inline void G1AdjustAndRebuildClosure::add_reference(T* from_field, oop reference, uint worker_id) {
- if (HeapRegion::is_in_same_region(from_field, reference)) {
- return;
- }
- _g1h->heap_region_containing(reference)->rem_set()->add_reference(from_field, worker_id);
-}
-
-inline size_t G1AdjustAndRebuildClosure::calculate_compaction_delta(oop current, oop forwardee) {
- return pointer_delta((HeapWord*)forwardee, (HeapWord*)current);
-}
-
-template <class T>
-inline T* G1AdjustAndRebuildClosure::add_compaction_delta(T* p) {
- return (T*)((HeapWord*)p + _compaction_delta);
-}
-
-template<typename T>
-void G1AdjustAndRebuildClosure::do_oop_nv(T* p) {
- oop new_reference = G1AdjustClosure::adjust_pointer(p);
- if (new_reference == NULL) {
return;
}
- // Update p using the calculated compaction delta to
- // get the new field address.
- T* new_field = add_compaction_delta(p);
- // Update the remembered set.
- add_reference(new_field, new_reference, _worker_id);
+ // Forwarded, just update.
+ assert(Universe::heap()->is_in_reserved(forwardee), "should be in object space");
+ RawAccess<OOP_NOT_NULL>::oop_store(p, forwardee);
}
-inline int G1AdjustObjectClosure::adjust_object(oop obj) {
- _closure->update_compaction_delta(obj);
- return obj->oop_iterate_size(_closure);
-}
+inline void G1AdjustClosure::do_oop(oop* p) { do_oop_nv(p); }
+inline void G1AdjustClosure::do_oop(narrowOop* p) { do_oop_nv(p); }
inline bool G1IsAliveClosure::do_object_b(oop p) {
return _bitmap->is_marked(p) || G1ArchiveAllocator::is_closed_archive_object(p);
--- a/src/hotspot/share/gc/g1/g1HeapVerifier.cpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1HeapVerifier.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -312,6 +312,9 @@
}
bool do_heap_region(HeapRegion* r) {
+ guarantee(!r->is_young() || r->rem_set()->is_complete(), "Remembered set for Young region %u must be complete, is %s", r->hrm_index(), r->rem_set()->get_state_str());
+ // Humongous and old regions regions might be of any state, so can't check here.
+ guarantee(!r->is_free() || !r->rem_set()->is_tracked(), "Remembered set for free region %u must be untracked, is %s", r->hrm_index(), r->rem_set()->get_state_str());
// For archive regions, verify there are no heap pointers to
// non-pinned regions. For all others, verify liveness info.
if (r->is_closed_archive()) {
@@ -453,7 +456,7 @@
}
if (failures) {
- log_error(gc, verify)("Heap after failed verification:");
+ log_error(gc, verify)("Heap after failed verification (kind %d):", vo);
// It helps to have the per-region information in the output to
// help us track down what went wrong. This is why we call
// print_extended_on() instead of print_on().
--- a/src/hotspot/share/gc/g1/g1OopClosures.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1OopClosures.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -208,4 +208,18 @@
virtual void do_oop(oop* p) { do_oop_nv(p); }
};
+class G1RebuildRemSetClosure : public ExtendedOopClosure {
+ G1CollectedHeap* _g1;
+ uint _worker_id;
+public:
+ G1RebuildRemSetClosure(G1CollectedHeap* g1, uint worker_id) : _g1(g1), _worker_id(worker_id) {
+ }
+
+ template <class T> void do_oop_nv(T* p);
+ virtual void do_oop(oop* p) { do_oop_nv(p); }
+ virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
+ // This closure needs special handling for InstanceRefKlass.
+ virtual ReferenceIterationMode reference_iteration_mode() { return DO_DISCOVERED_AND_DISCOVERY; }
+};
+
#endif // SHARE_VM_GC_G1_G1OOPCLOSURES_HPP
--- a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -143,10 +143,12 @@
return;
}
- HeapRegion* to = _g1->heap_region_containing(obj);
+ HeapRegionRemSet* to_rem_set = _g1->heap_region_containing(obj)->rem_set();
- assert(to->rem_set() != NULL, "Need per-region 'into' remsets.");
- to->rem_set()->add_reference(p, _worker_i);
+ assert(to_rem_set != NULL, "Need per-region 'into' remsets.");
+ if (to_rem_set->is_tracked()) {
+ to_rem_set->add_reference(p, _worker_i);
+ }
}
template <class T>
@@ -267,4 +269,20 @@
}
}
}
+
+template <class T> void G1RebuildRemSetClosure::do_oop_nv(T* p) {
+ oop const obj = RawAccess<MO_VOLATILE>::oop_load(p);
+ if (obj == NULL) {
+ return;
+ }
+
+ if (HeapRegion::is_in_same_region(p, obj)) {
+ return;
+ }
+
+ HeapRegion* to = _g1->heap_region_containing(obj);
+ HeapRegionRemSet* rem_set = to->rem_set();
+ rem_set->add_reference(p, _worker_id);
+}
+
#endif // SHARE_VM_GC_G1_G1OOPCLOSURES_INLINE_HPP
--- a/src/hotspot/share/gc/g1/g1Policy.cpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1Policy.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -49,6 +49,7 @@
G1Policy::G1Policy(STWGCTimer* gc_timer) :
_predictor(G1ConfidencePercent / 100.0),
_analytics(new G1Analytics(&_predictor)),
+ _remset_tracker(),
_mmu_tracker(new G1MMUTrackerQueue(GCPauseIntervalMillis / 1000.0, MaxGCPauseMillis / 1000.0)),
_ihop_control(create_ihop_control(&_predictor)),
_policy_counters(new GCPolicyCounters("GarbageFirst", 1, 2)),
@@ -66,7 +67,8 @@
_tenuring_threshold(MaxTenuringThreshold),
_max_survivor_regions(0),
_survivors_age_table(true),
- _collection_pause_end_millis(os::javaTimeNanos() / NANOSECS_PER_MILLISEC) { }
+ _collection_pause_end_millis(os::javaTimeNanos() / NANOSECS_PER_MILLISEC) {
+}
G1Policy::~G1Policy() {
delete _ihop_control;
@@ -413,6 +415,7 @@
_full_collection_start_sec = os::elapsedTime();
// Release the future to-space so that it is available for compaction into.
collector_state()->set_full_collection(true);
+ cset_chooser()->clear();
}
void G1Policy::record_full_collection_end() {
@@ -443,7 +446,6 @@
_survivor_surv_rate_group->reset();
update_young_list_max_and_target_length();
update_rs_lengths_prediction();
- cset_chooser()->clear();
_bytes_allocated_in_old_since_last_gc = 0;
@@ -500,13 +502,7 @@
}
void G1Policy::record_concurrent_mark_cleanup_completed() {
- bool should_continue_with_reclaim = next_gc_should_be_mixed("request last young-only gc",
- "skip last young-only gc");
- collector_state()->set_last_young_gc(should_continue_with_reclaim);
- // We skip the marking phase.
- if (!should_continue_with_reclaim) {
- abort_time_to_mixed_tracking();
- }
+ collector_state()->set_last_young_gc(collector_state()->mixed_gc_pending());
collector_state()->set_in_marking_window(false);
}
@@ -537,6 +533,7 @@
}
bool G1Policy::about_to_start_mixed_phase() const {
+ guarantee(_g1->concurrent_mark()->cm_thread()->during_cycle() || !collector_state()->mixed_gc_pending(), "Pending mixed phase when CM is idle!");
return _g1->concurrent_mark()->cm_thread()->during_cycle() || collector_state()->last_young_gc();
}
@@ -619,28 +616,22 @@
}
if (collector_state()->last_young_gc()) {
- // This is supposed to to be the "last young GC" before we start
- // doing mixed GCs. Here we decide whether to start mixed GCs or not.
assert(!last_pause_included_initial_mark, "The last young GC is not allowed to be an initial mark GC");
-
- if (next_gc_should_be_mixed("start mixed GCs",
- "do not start mixed GCs")) {
- collector_state()->set_gcs_are_young(false);
- } else {
- // We aborted the mixed GC phase early.
- abort_time_to_mixed_tracking();
- }
-
+ // This has been the "last young GC" before we start doing mixed GCs. We already
+ // decided to start mixed GCs much earlier, so there is nothing to do except
+ // advancing the state.
+ collector_state()->set_gcs_are_young(false);
collector_state()->set_last_young_gc(false);
}
if (!collector_state()->last_gc_was_young()) {
- // This is a mixed GC. Here we decide whether to continue doing
+ // This is a mixed GC. Here we decide whether to continue doing more
// mixed GCs or not.
if (!next_gc_should_be_mixed("continue mixed GCs",
"do not continue mixed GCs")) {
collector_state()->set_gcs_are_young(true);
+ clear_collection_set_candidates();
maybe_start_marking();
}
}
@@ -971,6 +962,11 @@
collector_state()->set_gcs_are_young(true);
collector_state()->set_last_young_gc(false);
+ // We might have ended up coming here about to start a mixed phase with a collection set
+ // active. The following remark might change the change the "evacuation efficiency" of
+ // the regions in this set, leading to failing asserts later.
+ // Since the concurrent cycle will recreate the collection set anyway, simply drop it here.
+ clear_collection_set_candidates();
abort_time_to_mixed_tracking();
initiate_conc_mark();
log_debug(gc, ergo)("Initiate concurrent cycle (user requested concurrent cycle)");
@@ -995,6 +991,13 @@
void G1Policy::record_concurrent_mark_cleanup_end() {
cset_chooser()->rebuild(_g1->workers(), _g1->num_regions());
+ bool mixed_gc_pending = next_gc_should_be_mixed("request mixed gcs", "request young-only gcs");
+ if (!mixed_gc_pending) {
+ clear_collection_set_candidates();
+ abort_time_to_mixed_tracking();
+ }
+ collector_state()->set_mixed_gc_pending(mixed_gc_pending);
+
double end_sec = os::elapsedTime();
double elapsed_time_ms = (end_sec - _mark_cleanup_start_sec) * 1000.0;
_analytics->report_concurrent_mark_cleanup_times_ms(elapsed_time_ms);
@@ -1007,6 +1010,21 @@
return percent_of(reclaimable_bytes, _g1->capacity());
}
+class G1ClearCollectionSetCandidateRemSets : public HeapRegionClosure {
+ virtual bool do_heap_region(HeapRegion* r) {
+ r->rem_set()->clear_locked(true /* only_cardset */);
+ return false;
+ }
+};
+
+void G1Policy::clear_collection_set_candidates() {
+ // Clear remembered sets of remaining candidate regions and the actual candidate
+ // list.
+ G1ClearCollectionSetCandidateRemSets cl;
+ cset_chooser()->iterate(&cl);
+ cset_chooser()->clear();
+}
+
void G1Policy::maybe_start_marking() {
if (need_to_start_conc_mark("end of GC")) {
// Note: this might have already been set, if during the last
--- a/src/hotspot/share/gc/g1/g1Policy.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1Policy.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -30,6 +30,7 @@
#include "gc/g1/g1InCSetState.hpp"
#include "gc/g1/g1InitialMarkToMixedTimeTracker.hpp"
#include "gc/g1/g1MMUTracker.hpp"
+#include "gc/g1/g1RemSetTrackingPolicy.hpp"
#include "gc/g1/g1Predictions.hpp"
#include "gc/g1/g1YoungGenSizer.hpp"
#include "gc/shared/gcCause.hpp"
@@ -62,6 +63,7 @@
G1Predictions _predictor;
G1Analytics* _analytics;
+ G1RemSetTrackingPolicy _remset_tracker;
G1MMUTracker* _mmu_tracker;
G1IHOPControl* _ihop_control;
@@ -107,6 +109,8 @@
const G1Predictions& predictor() const { return _predictor; }
const G1Analytics* analytics() const { return const_cast<const G1Analytics*>(_analytics); }
+ G1RemSetTrackingPolicy* remset_tracker() { return &_remset_tracker; }
+
// Add the given number of bytes to the total number of allocated bytes in the old gen.
void add_bytes_allocated_in_old_since_last_gc(size_t bytes) { _bytes_allocated_in_old_since_last_gc += bytes; }
@@ -254,6 +258,7 @@
jlong collection_pause_end_millis() { return _collection_pause_end_millis; }
private:
+ void clear_collection_set_candidates();
// Sets up marking if proper conditions are met.
void maybe_start_marking();
--- a/src/hotspot/share/gc/g1/g1RemSet.cpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1RemSet.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -40,6 +40,7 @@
#include "gc/shared/suspendibleThreadSet.hpp"
#include "memory/iterator.hpp"
#include "memory/resourceArea.hpp"
+#include "oops/access.inline.hpp"
#include "oops/oop.inline.hpp"
#include "utilities/align.hpp"
#include "utilities/globalDefinitions.hpp"
@@ -298,20 +299,12 @@
}
uint G1RemSet::num_par_rem_sets() {
- return MAX2(DirtyCardQueueSet::num_par_ids() + G1ConcurrentRefine::max_num_threads(), ParallelGCThreads);
+ return DirtyCardQueueSet::num_par_ids() + G1ConcurrentRefine::max_num_threads() + MAX2(ConcGCThreads, ParallelGCThreads);
}
void G1RemSet::initialize(size_t capacity, uint max_regions) {
G1FromCardCache::initialize(num_par_rem_sets(), max_regions);
_scan_state->initialize(max_regions);
- {
- GCTraceTime(Debug, gc, marking)("Initialize Card Live Data");
- _card_live_data.initialize(capacity, max_regions);
- }
- if (G1PretouchAuxiliaryMemory) {
- GCTraceTime(Debug, gc, marking)("Pre-Touch Card Live Data");
- _card_live_data.pretouch();
- }
}
G1ScanRSForRegionClosure::G1ScanRSForRegionClosure(G1RemSetScanState* scan_state,
@@ -514,27 +507,6 @@
phase_times->record_clear_ct_time((os::elapsedTime() - start) * 1000.0);
}
-class G1ScrubRSClosure: public HeapRegionClosure {
- G1CollectedHeap* _g1h;
- G1CardLiveData* _live_data;
-public:
- G1ScrubRSClosure(G1CardLiveData* live_data) :
- _g1h(G1CollectedHeap::heap()),
- _live_data(live_data) { }
-
- bool do_heap_region(HeapRegion* r) {
- if (!r->is_continues_humongous()) {
- r->rem_set()->scrub(_live_data);
- }
- return false;
- }
-};
-
-void G1RemSet::scrub(uint worker_num, HeapRegionClaimer *hrclaimer) {
- G1ScrubRSClosure scrub_cl(&_card_live_data);
- _g1->heap_region_par_iterate_from_worker_offset(&scrub_cl, hrclaimer, worker_num);
-}
-
inline void check_card_ptr(jbyte* card_ptr, G1CardTable* ct) {
#ifdef ASSERT
G1CollectedHeap* g1 = G1CollectedHeap::heap();
@@ -750,24 +722,173 @@
}
}
-void G1RemSet::create_card_live_data(WorkGang* workers, G1CMBitMap* mark_bitmap) {
- _card_live_data.create(workers, mark_bitmap);
-}
+class G1RebuildRemSetTask: public AbstractGangTask {
+ // Aggregate the counting data that was constructed concurrently
+ // with marking.
+ class G1RebuildRemSetHeapRegionClosure : public HeapRegionClosure {
+ G1ConcurrentMark* _cm;
+ G1RebuildRemSetClosure _update_cl;
+
+ void scan_for_references(oop const obj, MemRegion mr) {
+ obj->oop_iterate(&_update_cl, mr);
+ }
+
+ void scan_for_references(oop const obj) {
+ obj->oop_iterate(&_update_cl);
+ }
+
+ // A humongous object is live (with respect to the scanning) either
+ // a) it is marked on the bitmap as such
+ // b) its TARS is larger than nTAMS, i.e. has been allocated during marking.
+ bool is_humongous_live(oop const humongous_obj, HeapWord* ntams, HeapWord* tars) const {
+ return _cm->next_mark_bitmap()->is_marked(humongous_obj) || (tars > ntams);
+ }
+
+ // Rebuilds the remembered sets by scanning the objects that were allocated before
+ // rebuild start in the given region, applying the given closure to each of these objects.
+ // Uses the bitmap to get live objects in the area from [bottom, nTAMS), and all
+ // objects from [nTAMS, TARS).
+ // Returns the number of bytes marked in that region between bottom and nTAMS.
+ size_t rebuild_rem_set_in_region(G1CMBitMap* const mark_bitmap, HeapRegion* hr, HeapWord* const top_at_rebuild_start) {
+ size_t marked_bytes = 0;
+
+ HeapWord* start = hr->bottom();
+ HeapWord* const ntams = hr->next_top_at_mark_start();
+
+ if (top_at_rebuild_start <= start) {
+ return 0;
+ }
+
+ if (hr->is_humongous()) {
+ oop const humongous_obj = oop(hr->humongous_start_region()->bottom());
+ log_debug(gc,remset)("Humongous obj region %u marked %d start " PTR_FORMAT " region start " PTR_FORMAT " TAMS " PTR_FORMAT " TARS " PTR_FORMAT,
+ hr->hrm_index(), _cm->next_mark_bitmap()->is_marked(humongous_obj),
+ p2i(humongous_obj), p2i(hr->bottom()), p2i(hr->next_top_at_mark_start()), p2i(top_at_rebuild_start));
+ if (is_humongous_live(humongous_obj, ntams, top_at_rebuild_start)) {
+ // We need to scan both [bottom, nTAMS) and [nTAMS, top_at_rebuild_start);
+ // however in case of humongous objects it is sufficient to scan the encompassing
+ // area (top_at_rebuild_start is always larger or equal to nTAMS) as one of the
+ // two areas will be zero sized. I.e. nTAMS is either
+ // the same as bottom or top(_at_rebuild_start). There is no way ntams has a different
+ // value: this would mean that nTAMS points somewhere into the object.
+ assert(hr->top() == hr->next_top_at_mark_start() || hr->top() == top_at_rebuild_start,
+ "More than one object in the humongous region?");
+ scan_for_references(humongous_obj, MemRegion(start, top_at_rebuild_start));
+ return ntams != start ? pointer_delta(hr->next_top_at_mark_start(), start, 1) : 0;
+ } else {
+ return 0;
+ }
+ }
+
+ assert(start <= hr->end() && start <= ntams &&
+ ntams <= top_at_rebuild_start && top_at_rebuild_start <= hr->end(),
+ "Inconsistency between bottom, nTAMS, TARS, end - "
+ "start: " PTR_FORMAT ", nTAMS: " PTR_FORMAT ", TARS: " PTR_FORMAT ", end: " PTR_FORMAT,
+ p2i(start), p2i(ntams), p2i(top_at_rebuild_start), p2i(hr->end()));
+
+ // Iterate live objects between bottom and nTAMS.
+ start = mark_bitmap->get_next_marked_addr(start, ntams);
+ while (start < ntams) {
+ oop obj = oop(start);
+ assert(oopDesc::is_oop(obj), "Address " PTR_FORMAT " below nTAMS is not an oop", p2i(start));
+ size_t obj_size = obj->size();
+ HeapWord* obj_end = start + obj_size;
+
+ assert(obj_end <= hr->end(), "Humongous objects must have been handled elsewhere.");
+
+ scan_for_references(obj);
+
+ // Add the size of this object to the number of marked bytes.
+ marked_bytes += obj_size;
+
+ // Find the next marked object after this one.
+ start = mark_bitmap->get_next_marked_addr(obj_end, ntams);
+ }
-void G1RemSet::finalize_card_live_data(WorkGang* workers, G1CMBitMap* mark_bitmap) {
- _card_live_data.finalize(workers, mark_bitmap);
-}
+ // Finally process live objects (all of them) between nTAMS and top_at_rebuild_start.
+ // Objects between top_at_rebuild_start and top are implicitly managed by concurrent refinement.
+ while (start < top_at_rebuild_start) {
+ oop obj = oop(start);
+ assert(oopDesc::is_oop(obj),
+ "Address " PTR_FORMAT " above nTAMS is not an oop (TARS " PTR_FORMAT " region %u)",
+ p2i(start), p2i(top_at_rebuild_start), hr->hrm_index());
+ size_t obj_size = obj->size();
+ HeapWord* obj_end = start + obj_size;
+
+ assert(obj_end <= hr->end(), "Humongous objects must have been handled elsewhere.");
+
+ scan_for_references(obj);
+ start = obj_end;
+ }
+ return marked_bytes * HeapWordSize;
+ }
+ public:
+ G1RebuildRemSetHeapRegionClosure(G1CollectedHeap* g1h,
+ G1ConcurrentMark* cm,
+ uint worker_id) :
+ HeapRegionClosure(),
+ _cm(cm),
+ _update_cl(g1h, worker_id) { }
-void G1RemSet::verify_card_live_data(WorkGang* workers, G1CMBitMap* bitmap) {
- _card_live_data.verify(workers, bitmap);
+ bool do_heap_region(HeapRegion* hr) {
+ if (_cm->has_aborted()) {
+ return true;
+ }
+ uint const region_idx = hr->hrm_index();
+ HeapWord* const top_at_rebuild_start = _cm->top_at_rebuild_start(region_idx);
+ // TODO: smaller increments to do yield checks with
+ size_t marked_bytes = rebuild_rem_set_in_region(_cm->next_mark_bitmap(), hr, top_at_rebuild_start);
+ log_trace(gc, remset, tracking)("Rebuilt region %u " SIZE_FORMAT " marked bytes " SIZE_FORMAT " "
+ "bot " PTR_FORMAT " nTAMS " PTR_FORMAT " TARS " PTR_FORMAT,
+ region_idx,
+ _cm->liveness(region_idx) * HeapWordSize,
+ marked_bytes,
+ p2i(hr->bottom()),
+ p2i(hr->next_top_at_mark_start()),
+ p2i(top_at_rebuild_start));
+ if (marked_bytes > 0) {
+ hr->add_to_marked_bytes(marked_bytes);
+ assert(!hr->is_old() || marked_bytes == (_cm->liveness(hr->hrm_index()) * HeapWordSize),
+ "Marked bytes " SIZE_FORMAT " for region %u do not match liveness during mark " SIZE_FORMAT,
+ marked_bytes, hr->hrm_index(), _cm->liveness(hr->hrm_index()) * HeapWordSize);
+ }
+ _cm->do_yield_check();
+ // Abort state may have changed after the yield check.
+ return _cm->has_aborted();
+ }
+ };
+
+ HeapRegionClaimer _hr_claimer;
+ G1ConcurrentMark* _cm;
+
+ uint _worker_id_offset;
+public:
+ G1RebuildRemSetTask(G1ConcurrentMark* cm,
+ uint n_workers,
+ uint worker_id_offset) :
+ AbstractGangTask("G1 Rebuild Remembered Set"),
+ _cm(cm),
+ _hr_claimer(n_workers),
+ _worker_id_offset(worker_id_offset) {
+ }
+
+ void work(uint worker_id) {
+ SuspendibleThreadSetJoiner sts_join;
+
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+
+ G1RebuildRemSetHeapRegionClosure cl(g1h, _cm, _worker_id_offset + worker_id);
+ g1h->heap_region_par_iterate_from_worker_offset(&cl, &_hr_claimer, worker_id);
+ }
+};
+
+void G1RemSet::rebuild_rem_set(G1ConcurrentMark* cm,
+ WorkGang* workers,
+ uint worker_id_offset) {
+ uint num_workers = workers->active_workers();
+
+ G1RebuildRemSetTask cl(cm,
+ num_workers,
+ worker_id_offset);
+ workers->run_task(&cl, num_workers);
}
-
-void G1RemSet::clear_card_live_data(WorkGang* workers) {
- _card_live_data.clear(workers);
-}
-
-#ifdef ASSERT
-void G1RemSet::verify_card_live_data_is_clear() {
- _card_live_data.verify_is_clear();
-}
-#endif
--- a/src/hotspot/share/gc/g1/g1RemSet.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1RemSet.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -26,8 +26,8 @@
#define SHARE_VM_GC_G1_G1REMSET_HPP
#include "gc/g1/dirtyCardQueue.hpp"
-#include "gc/g1/g1CardLiveData.hpp"
#include "gc/g1/g1CardTable.hpp"
+#include "gc/g1/g1OopClosures.hpp"
#include "gc/g1/g1RemSetSummary.hpp"
#include "gc/g1/heapRegion.hpp"
#include "memory/allocation.hpp"
@@ -41,6 +41,7 @@
class G1BlockOffsetTable;
class CodeBlobClosure;
class G1CollectedHeap;
+class G1CMBitMap;
class G1HotCardCache;
class G1RemSetScanState;
class G1ParScanThreadState;
@@ -55,7 +56,6 @@
class G1RemSet: public CHeapObj<mtGC> {
private:
G1RemSetScanState* _scan_state;
- G1CardLiveData _card_live_data;
G1RemSetSummary _prev_period_summary;
@@ -114,9 +114,6 @@
G1RemSetScanState* scan_state() const { return _scan_state; }
- // Eliminates any remembered set entries that correspond to dead heap ranges.
- void scrub(uint worker_num, HeapRegionClaimer* hrclaimer);
-
// Refine the card corresponding to "card_ptr". Safe to be called concurrently
// to the mutator.
void refine_card_concurrently(jbyte* card_ptr,
@@ -135,18 +132,9 @@
size_t num_conc_refined_cards() const { return _num_conc_refined_cards; }
- void create_card_live_data(WorkGang* workers, G1CMBitMap* mark_bitmap);
- void finalize_card_live_data(WorkGang* workers, G1CMBitMap* mark_bitmap);
-
- // Verify that the liveness count data created concurrently matches one created
- // during this safepoint.
- void verify_card_live_data(WorkGang* workers, G1CMBitMap* actual_bitmap);
-
- void clear_card_live_data(WorkGang* workers);
-
-#ifdef ASSERT
- void verify_card_live_data_is_clear();
-#endif
+ // Rebuilds the remembered set by scanning from bottom to TARS for all regions
+ // using the given work gang.
+ void rebuild_rem_set(G1ConcurrentMark* cm, WorkGang* workers, uint worker_id_offset);
};
class G1ScanRSForRegionClosure : public HeapRegionClosure {
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -0,0 +1,142 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/g1/collectionSetChooser.hpp"
+#include "gc/g1/g1RemSetTrackingPolicy.hpp"
+#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/heapRegionRemSet.hpp"
+#include "runtime/safepoint.hpp"
+
+bool G1RemSetTrackingPolicy::is_interesting_humongous_region(HeapRegion* r) const {
+ return r->is_starts_humongous() && oop(r->bottom())->is_typeArray();
+}
+
+bool G1RemSetTrackingPolicy::needs_scan_for_rebuild(HeapRegion* r) const {
+ // All non-young and non-closed archive regions need to be scanned for references;
+ // At every gc we gather references to other regions in young, and closed archive
+ // regions by definition do not have references going outside the closed archive.
+ return !(r->is_young() || r->is_closed_archive());
+}
+
+void G1RemSetTrackingPolicy::update_at_allocate(HeapRegion* r) {
+ if (r->is_young()) {
+ // Always collect remembered set for young regions.
+ r->rem_set()->set_state_complete();
+ } else if (r->is_humongous()) {
+ // Collect remembered sets for humongous regions by default to allow eager reclaim.
+ r->rem_set()->set_state_complete();
+ } else if (r->is_archive()) {
+ // Archive regions never move ever. So never build remembered sets for them.
+ r->rem_set()->set_state_empty();
+ } else if (r->is_old()) {
+ // By default, do not create remembered set for new old regions.
+ r->rem_set()->set_state_empty();
+ } else {
+ guarantee(false, "Unhandled region %u with heap region type %s", r->hrm_index(), r->get_type_str());
+ }
+}
+
+void G1RemSetTrackingPolicy::update_at_free(HeapRegion* r) {
+ r->rem_set()->set_state_empty();
+}
+
+bool G1RemSetTrackingPolicy::update_before_rebuild(HeapRegion* r, size_t live_bytes) {
+ assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
+
+ bool selected_for_rebuild = false;
+
+ // Only consider updating the remembered set for old gen regions - excluding archive regions
+ // which never move (but are "Old" regions).
+ if (r->is_old_or_humongous() && !r->is_archive()) {
+ size_t between_ntams_and_top = (r->top() - r->next_top_at_mark_start()) * HeapWordSize;
+ size_t total_live_bytes = live_bytes + between_ntams_and_top;
+ // Completely free regions after rebuild are of no interest wrt rebuilding the
+ // remembered set.
+ assert(!r->rem_set()->is_updating(), "Remembered set of region %u is updating before rebuild", r->hrm_index());
+ // To be of interest for rebuilding the remembered set the following must apply:
+ // - They must contain some live data in them.
+ // - We always try to update the remembered sets of humongous regions containing
+ // type arrays if they are empty as they might have been reset after full gc.
+ // - Only need to rebuild non-complete remembered sets.
+ // - Otherwise only add those old gen regions which occupancy is low enough that there
+ // is a chance that we will ever evacuate them in the mixed gcs.
+ if ((total_live_bytes > 0) &&
+ (is_interesting_humongous_region(r) || CollectionSetChooser::region_occupancy_low_enough_for_evac(total_live_bytes)) &&
+ !r->rem_set()->is_tracked()) {
+
+ r->rem_set()->set_state_updating();
+ selected_for_rebuild = true;
+ }
+ log_trace(gc, remset, tracking)("Before rebuild region %u "
+ "(ntams: " PTR_FORMAT ") "
+ "total_live_bytes " SIZE_FORMAT " "
+ "selected %s "
+ "(live_bytes " SIZE_FORMAT " "
+ "next_marked " SIZE_FORMAT " "
+ "marked " SIZE_FORMAT " "
+ "type %s)",
+ r->hrm_index(),
+ p2i(r->next_top_at_mark_start()),
+ total_live_bytes,
+ BOOL_TO_STR(selected_for_rebuild),
+ live_bytes,
+ r->next_marked_bytes(),
+ r->marked_bytes(),
+ r->get_type_str());
+ }
+
+ return selected_for_rebuild;
+}
+
+void G1RemSetTrackingPolicy::update_after_rebuild(HeapRegion* r) {
+ assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
+
+ if (r->is_old_or_humongous()) {
+ if (r->rem_set()->is_updating()) {
+ r->rem_set()->set_state_complete();
+ }
+ // We can drop remembered sets of humongous regions that have a too large remembered set:
+ // We will never try to eagerly reclaim or move them anyway until the next concurrent
+ // cycle as e.g. remembered set entries will always be added.
+ if (r->is_humongous() && !G1CollectedHeap::heap()->is_potential_eager_reclaim_candidate(r)) {
+ r->rem_set()->clear_locked(true /* only_cardset */);
+ }
+ assert(!r->is_continues_humongous() || r->rem_set()->is_empty(), "Continues humongous object remsets should be empty");
+ G1ConcurrentMark* cm = G1CollectedHeap::heap()->concurrent_mark();
+ log_trace(gc, remset, tracking)("After rebuild region %u "
+ "(ntams " PTR_FORMAT " "
+ "liveness " SIZE_FORMAT " "
+ "next_marked_bytes " SIZE_FORMAT " "
+ "remset occ " SIZE_FORMAT " "
+ "size " SIZE_FORMAT ")",
+ r->hrm_index(),
+ p2i(r->next_top_at_mark_start()),
+ cm->liveness(r->hrm_index()) * HeapWordSize,
+ r->next_marked_bytes(),
+ r->rem_set()->occupied_locked(),
+ r->rem_set()->mem_size());
+ }
+}
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_G1_G1REMSETTRACKINGPOLICY_HPP
+#define SHARE_VM_GC_G1_G1REMSETTRACKINGPOLICY_HPP
+
+#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/heapRegionType.hpp"
+#include "memory/allocation.hpp"
+
+// The remembered set tracking policy determines for a given region the state of
+// the remembered set, ie. when it should be tracked, and if/when the remembered
+// set is complete.
+class G1RemSetTrackingPolicy : public CHeapObj<mtGC> {
+private:
+ // Is the given region an interesting humongous region to start remembered set tracking
+ // for?
+ bool is_interesting_humongous_region(HeapRegion* r) const;
+public:
+ // Do we need to scan the given region to get all outgoing references for remembered
+ // set rebuild?
+ bool needs_scan_for_rebuild(HeapRegion* r) const;
+ // Update remembered set tracking state at allocation of the region. May be
+ // called at any time. The caller makes sure that the changes to the remembered
+ // set state are visible to other threads.
+ void update_at_allocate(HeapRegion* r);
+ // Update remembered set tracking state before we are going to rebuild remembered
+ // sets. Called at safepoint in the remark pause.
+ bool update_before_rebuild(HeapRegion* r, size_t live_bytes);
+ // Update remembered set tracking state after rebuild is complete, i.e. the cleanup
+ // pause. Called at safepoint.
+ void update_after_rebuild(HeapRegion* r);
+ // Update remembered set tracking state when the region is freed.
+ void update_at_free(HeapRegion* r);
+};
+
+#endif /* SHARE_VM_GC_G1_G1REMSETTRACKINGPOLICY_HPP */
+
--- a/src/hotspot/share/gc/g1/g1_globals.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1_globals.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -184,9 +184,6 @@
"-1 means print all.") \
range(-1, max_jint) \
\
- develop(bool, G1ScrubRemSets, true, \
- "When true, do RS scrubbing after cleanup.") \
- \
product(uintx, G1ReservePercent, 10, \
"It determines the minimum reserve we should have in the heap " \
"to minimize the probability of promotion failure.") \
--- a/src/hotspot/share/gc/g1/g1_specialized_oop_closures.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/g1_specialized_oop_closures.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -42,7 +42,9 @@
class G1RootRegionScanClosure;
class G1MarkAndPushClosure;
-class G1AdjustAndRebuildClosure;
+class G1AdjustClosure;
+
+class G1RebuildRemSetClosure;
#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_G1(f) \
f(G1ScanEvacuatedObjClosure,_nv) \
@@ -50,10 +52,11 @@
f(G1ScanObjsDuringScanRSClosure,_nv) \
f(G1ConcurrentRefineOopClosure,_nv) \
f(G1CMOopClosure,_nv) \
- f(G1RootRegionScanClosure,_nv)
+ f(G1RootRegionScanClosure,_nv) \
+ f(G1RebuildRemSetClosure,_nv)
#define SPECIALIZED_OOP_OOP_ITERATE_CLOSURES_G1FULL(f) \
f(G1MarkAndPushClosure,_nv) \
- f(G1AdjustAndRebuildClosure,_nv)
+ f(G1AdjustClosure,_nv)
#endif // SHARE_VM_GC_G1_G1_SPECIALIZED_OOP_CLOSURES_HPP
--- a/src/hotspot/share/gc/g1/heapRegion.cpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/heapRegion.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -453,8 +453,8 @@
st->print("| ");
}
st->print("|TS%3u", _gc_time_stamp);
- st->print_cr("|TAMS " PTR_FORMAT ", " PTR_FORMAT "|",
- p2i(prev_top_at_mark_start()), p2i(next_top_at_mark_start()));
+ st->print_cr("|TAMS " PTR_FORMAT ", " PTR_FORMAT "| %s ",
+ p2i(prev_top_at_mark_start()), p2i(next_top_at_mark_start()), rem_set()->get_state_str());
}
class G1VerificationClosure : public OopClosure {
@@ -527,7 +527,8 @@
p2i(p), p2i(_containing_obj), p2i(from->bottom()), p2i(from->end()));
LogStream ls(log.error());
print_object(&ls, _containing_obj);
- log.error("points to obj " PTR_FORMAT " not in the heap", p2i(obj));
+ HeapRegion* const to = _g1h->heap_region_containing(obj);
+ log.error("points to obj " PTR_FORMAT " in region " HR_FORMAT " remset %s", p2i(obj), HR_FORMAT_PARAMS(to), to->rem_set()->get_state_str());
} else {
HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
HeapRegion* to = _g1h->heap_region_containing((HeapWord*)obj);
@@ -572,7 +573,8 @@
HeapRegion* to = _g1h->heap_region_containing(obj);
if (from != NULL && to != NULL &&
from != to &&
- !to->is_pinned()) {
+ !to->is_pinned() &&
+ to->rem_set()->is_complete()) {
jbyte cv_obj = *_ct->byte_for_const(_containing_obj);
jbyte cv_field = *_ct->byte_for_const(p);
const jbyte dirty = G1CardTable::dirty_card_val();
@@ -595,7 +597,7 @@
ResourceMark rm;
LogStream ls(log.error());
_containing_obj->print_on(&ls);
- log.error("points to obj " PTR_FORMAT " in region " HR_FORMAT, p2i(obj), HR_FORMAT_PARAMS(to));
+ log.error("points to obj " PTR_FORMAT " in region " HR_FORMAT " remset %s", p2i(obj), HR_FORMAT_PARAMS(to), to->rem_set()->get_state_str());
if (oopDesc::is_oop(obj)) {
obj->print_on(&ls);
}
--- a/src/hotspot/share/gc/g1/heapRegionRemSet.cpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/heapRegionRemSet.cpp Mon Mar 26 16:51:43 2018 +0200
@@ -26,7 +26,6 @@
#include "gc/g1/g1BlockOffsetTable.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentRefine.hpp"
-#include "gc/g1/g1CardLiveData.inline.hpp"
#include "gc/g1/heapRegionManager.inline.hpp"
#include "gc/g1/heapRegionRemSet.hpp"
#include "gc/shared/space.inline.hpp"
@@ -40,6 +39,8 @@
#include "utilities/globalDefinitions.hpp"
#include "utilities/growableArray.hpp"
+const char* HeapRegionRemSet::_state_strings[] = {"Untracked", "Updating", "Complete"};
+
class PerRegionTable: public CHeapObj<mtGC> {
friend class OtherRegionsTable;
friend class HeapRegionRemSetIterator;
@@ -64,10 +65,6 @@
// We need access in order to union things into the base table.
BitMap* bm() { return &_bm; }
- void recount_occupied() {
- _occupied = (jint) bm()->count_one_bits();
- }
-
PerRegionTable(HeapRegion* hr) :
_hr(hr),
_occupied(0),
@@ -142,11 +139,6 @@
add_reference_work(from, /*parallel*/ false);
}
- void scrub(G1CardLiveData* live_data) {
- live_data->remove_nonlive_cards(hr()->hrm_index(), &_bm);
- recount_occupied();
- }
-
void add_card(CardIdx_t from_card_index) {
add_card_work(from_card_index, /*parallel*/ true);
}
@@ -436,7 +428,7 @@
assert(prt != NULL, "Inv");
prt->add_reference(from);
- assert(contains_reference(from), "We just added " PTR_FORMAT " to the PRT", p2i(from));
+ assert(contains_reference(from), "We just added " PTR_FORMAT " to the PRT (%d)", p2i(from), prt->contains_reference(from));
}
PerRegionTable*
@@ -509,56 +501,6 @@
return max;
}
-void OtherRegionsTable::scrub(G1CardLiveData* live_data) {
- // First eliminated garbage regions from the coarse map.
- log_develop_trace(gc, remset, scrub)("Scrubbing region %u:", _hr->hrm_index());
-
- log_develop_trace(gc, remset, scrub)(" Coarse map: before = " SIZE_FORMAT "...", _n_coarse_entries);
- if (_n_coarse_entries > 0) {
- live_data->remove_nonlive_regions(&_coarse_map);
- _n_coarse_entries = _coarse_map.count_one_bits();
- }
- log_develop_trace(gc, remset, scrub)(" after = " SIZE_FORMAT ".", _n_coarse_entries);
-
- // Now do the fine-grained maps.
- for (size_t i = 0; i < _max_fine_entries; i++) {
- PerRegionTable* cur = _fine_grain_regions[i];
- PerRegionTable** prev = &_fine_grain_regions[i];
- while (cur != NULL) {
- PerRegionTable* nxt = cur->collision_list_next();
- // If the entire region is dead, eliminate.
- log_develop_trace(gc, remset, scrub)(" For other region %u:", cur->hr()->hrm_index());
- if (!live_data->is_region_live(cur->hr()->hrm_index())) {
- *prev = nxt;
- cur->set_collision_list_next(NULL);
- _n_fine_entries--;
- log_develop_trace(gc, remset, scrub)(" deleted via region map.");
- unlink_from_all(cur);
- PerRegionTable::free(cur);
- } else {
- // Do fine-grain elimination.
- log_develop_trace(gc, remset, scrub)(" occ: before = %4d.", cur->occupied());
- cur->scrub(live_data);
- log_develop_trace(gc, remset, scrub)(" after = %4d.", cur->occupied());
- // Did that empty the table completely?
- if (cur->occupied() == 0) {
- *prev = nxt;
- cur->set_collision_list_next(NULL);
- _n_fine_entries--;
- unlink_from_all(cur);
- PerRegionTable::free(cur);
- } else {
- prev = cur->collision_list_next_addr();
- }
- }
- cur = nxt;
- }
- }
- // Since we may have deleted a from_card_cache entry from the RS, clear
- // the FCC.
- clear_fcc();
-}
-
bool OtherRegionsTable::occupancy_less_or_equal_than(size_t limit) const {
if (limit <= (size_t)G1RSetSparseRegionEntries) {
return occ_coarse() == 0 && _first_all_fine_prts == NULL && occ_sparse() <= limit;
@@ -692,6 +634,7 @@
: _bot(bot),
_m(Mutex::leaf, FormatBuffer<128>("HeapRegionRemSet lock #%u", hr->hrm_index()), true, Monitor::_safepoint_check_never),
_code_roots(),
+ _state(Untracked),
_other_regions(hr, &_m) {
}
@@ -713,21 +656,20 @@
SparsePRT::cleanup_all();
}
-void HeapRegionRemSet::clear() {
+void HeapRegionRemSet::clear(bool only_cardset) {
MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag);
- clear_locked();
+ clear_locked(only_cardset);
}
-void HeapRegionRemSet::clear_locked() {
- _code_roots.clear();
+void HeapRegionRemSet::clear_locked(bool only_cardset) {
+ if (!only_cardset) {
+ _code_roots.clear();
+ }
_other_regions.clear();
+ set_state_empty();
assert(occupied_locked() == 0, "Should be clear.");
}
-void HeapRegionRemSet::scrub(G1CardLiveData* live_data) {
- _other_regions.scrub(live_data);
-}
-
// Code roots support
//
// The code root set is protected by two separate locking schemes
--- a/src/hotspot/share/gc/g1/heapRegionRemSet.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/gc/g1/heapRegionRemSet.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -123,15 +123,14 @@
bool contains_reference_locked(OopOrNarrowOopStar from) const;
+public:
// Clear the from_card_cache entries for this region.
void clear_fcc();
-public:
// Create a new remembered set for the given heap region. The given mutex should
// be used to ensure consistency.
OtherRegionsTable(HeapRegion* hr, Mutex* m);
- // For now. Could "expand" some tables in the future, so that this made
- // sense.
+ // Adds the reference from "from to this remembered set.
void add_reference(OopOrNarrowOopStar from, uint tid);
// Returns whether the remembered set contains the given reference.
@@ -141,11 +140,6 @@
// that is less or equal than the given occupancy.
bool occupancy_less_or_equal_than(size_t limit) const;
- // Removes any entries shown by the given bitmaps to contain only dead
- // objects. Not thread safe.
- // Set bits in the bitmaps indicate that the given region or card is live.
- void scrub(G1CardLiveData* live_data);
-
// Returns whether this remembered set (and all sub-sets) does not contain any entry.
bool is_empty() const;
@@ -217,24 +211,62 @@
static jint n_coarsenings() { return OtherRegionsTable::n_coarsenings(); }
+private:
+ enum RemSetState {
+ Untracked,
+ Updating,
+ Complete
+ };
+
+ RemSetState _state;
+
+ static const char* _state_strings[];
+public:
+
+ const char* get_state_str() const { return _state_strings[_state]; }
+
+ bool is_tracked() { return _state != Untracked; }
+ bool is_updating() { return _state == Updating; }
+ bool is_complete() { return _state == Complete; }
+
+ void set_state_empty() {
+ guarantee(SafepointSynchronize::is_at_safepoint() || !is_tracked(), "Should only set to Untracked during safepoint but is %s.", get_state_str());
+ if (_state == Untracked) {
+ return;
+ }
+ _other_regions.clear_fcc();
+ _state = Untracked;
+ }
+
+ void set_state_updating() {
+ guarantee(SafepointSynchronize::is_at_safepoint() && !is_tracked(), "Should only set to Updating from Untracked during safepoint but is %s", get_state_str());
+ _other_regions.clear_fcc();
+ _state = Updating;
+ }
+
+ void set_state_complete() {
+ _other_regions.clear_fcc();
+ _state = Complete;
+ }
+
// Used in the sequential case.
void add_reference(OopOrNarrowOopStar from) {
- _other_regions.add_reference(from, 0);
+ add_reference(from, 0);
}
// Used in the parallel case.
void add_reference(OopOrNarrowOopStar from, uint tid) {
+ RemSetState state = _state;
+ if (state == Untracked) {
+ return;
+ }
_other_regions.add_reference(from, tid);
}
- // Removes any entries in the remembered set shown by the given card live data to
- // contain only dead objects. Not thread safe.
- void scrub(G1CardLiveData* live_data);
-
// The region is being reclaimed; clear its remset, and any mention of
// entries for this region in other remsets.
- void clear();
- void clear_locked();
+ void clear(bool only_cardset = false);
+ void clear_locked(bool only_cardset = false);
// The actual # of bytes this hr_remset takes up.
// Note also includes the strong code root set.
--- a/src/hotspot/share/logging/logPrefix.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/logging/logPrefix.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -72,6 +72,7 @@
LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, plab)) \
LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, region)) \
LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, remset)) \
+ LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, remset, tracking)) \
LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, ref)) \
LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, ref, start)) \
LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, start)) \
--- a/src/hotspot/share/logging/logTag.hpp Mon Mar 26 16:51:43 2018 +0200
+++ b/src/hotspot/share/logging/logTag.hpp Mon Mar 26 16:51:43 2018 +0200
@@ -124,7 +124,6 @@
LOG_TAG(resolve) \
LOG_TAG(safepoint) \
LOG_TAG(scavenge) \
- LOG_TAG(scrub) \
LOG_TAG(smr) \
LOG_TAG(stacktrace) \
LOG_TAG(stackwalk) \
@@ -145,6 +144,7 @@
LOG_TAG(tlab) \
LOG_TAG(time) \
LOG_TAG(timer) \
+ LOG_TAG(tracking) \
LOG_TAG(update) \
LOG_TAG(unload) /* Trace unloading of classes */ \
LOG_TAG(unshareable) \
--- a/test/hotspot/jtreg/gc/concurrent_phase_control/TestConcurrentPhaseControlG1.java Mon Mar 26 16:51:43 2018 +0200
+++ b/test/hotspot/jtreg/gc/concurrent_phase_control/TestConcurrentPhaseControlG1.java Mon Mar 26 16:51:43 2018 +0200
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -53,7 +53,7 @@
{"MARK_FROM_ROOTS", "Concurrent Mark From Roots"},
{"BEFORE_REMARK", null},
{"REMARK", "Pause Remark"},
- {"CREATE_LIVE_DATA", "Concurrent Create Live Data"},
+ {"REBUILD_REMEMBERED_SETS", "Concurrent Rebuild Remembered Sets"},
// "COMPLETE_CLEANUP", -- optional phase, not reached by this test
{"CLEANUP_FOR_NEXT_MARK", "Concurrent Cleanup for Next Mark"},
// Clear request
--- a/test/hotspot/jtreg/gc/concurrent_phase_control/TestConcurrentPhaseControlG1Basics.java Mon Mar 26 16:51:43 2018 +0200
+++ b/test/hotspot/jtreg/gc/concurrent_phase_control/TestConcurrentPhaseControlG1Basics.java Mon Mar 26 16:51:43 2018 +0200
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -53,7 +53,7 @@
"MARK_FROM_ROOTS",
"BEFORE_REMARK",
"REMARK",
- "CREATE_LIVE_DATA",
+ "REBUILD_REMEMBERED_SETS",
"COMPLETE_CLEANUP",
"CLEANUP_FOR_NEXT_MARK",
};