8226232: Move merge heap roots code out from G1RemSetScanState
authortschatzl
Wed, 17 Jul 2019 16:33:19 +0200
changeset 55720 dec1d532c000
parent 55719 4888ccfc234e
child 55721 6891a4438ae6
8226232: Move merge heap roots code out from G1RemSetScanState Reviewed-by: sangheki, kbarrett
src/hotspot/share/gc/g1/g1RemSet.cpp
src/hotspot/share/gc/g1/g1RemSet.hpp
--- a/src/hotspot/share/gc/g1/g1RemSet.cpp	Wed Jul 17 23:22:48 2019 +0900
+++ b/src/hotspot/share/gc/g1/g1RemSet.cpp	Wed Jul 17 16:33:19 2019 +0200
@@ -183,259 +183,6 @@
     }
   };
 
-  // Returns whether the given region contains cards we need to scan. The remembered
-  // set and other sources may contain cards that
-  // - are in uncommitted regions
-  // - are located in the collection set
-  // - are located in free regions
-  // as we do not clean up remembered sets before merging heap roots.
-  bool contains_cards_to_process(uint const region_idx) const {
-    HeapRegion* hr = G1CollectedHeap::heap()->region_at_or_null(region_idx);
-    return (hr != NULL && !hr->in_collection_set() && hr->is_old_or_humongous_or_archive());
-  }
-
-  class G1MergeCardSetClosure : public HeapRegionClosure {
-    G1RemSetScanState* _scan_state;
-    G1CardTable* _ct;
-
-    uint _merged_sparse;
-    uint _merged_fine;
-    uint _merged_coarse;
-
-    // Returns if the region contains cards we need to scan. If so, remember that
-    // region in the current set of dirty regions.
-    bool remember_if_interesting(uint const region_idx) {
-      if (!_scan_state->contains_cards_to_process(region_idx)) {
-        return false;
-      }
-      _scan_state->add_dirty_region(region_idx);
-      return true;
-    }
-  public:
-    G1MergeCardSetClosure(G1RemSetScanState* scan_state) :
-      _scan_state(scan_state),
-      _ct(G1CollectedHeap::heap()->card_table()),
-      _merged_sparse(0),
-      _merged_fine(0),
-      _merged_coarse(0) { }
-
-    void next_coarse_prt(uint const region_idx) {
-      if (!remember_if_interesting(region_idx)) {
-        return;
-      }
-
-      _merged_coarse++;
-
-      size_t region_base_idx = (size_t)region_idx << HeapRegion::LogCardsPerRegion;
-      _ct->mark_region_dirty(region_base_idx, HeapRegion::CardsPerRegion);
-      _scan_state->set_chunk_region_dirty(region_base_idx);
-    }
-
-    void next_fine_prt(uint const region_idx, BitMap* bm) {
-      if (!remember_if_interesting(region_idx)) {
-        return;
-      }
-
-      _merged_fine++;
-
-      size_t const region_base_idx = (size_t)region_idx << HeapRegion::LogCardsPerRegion;
-      BitMap::idx_t cur = bm->get_next_one_offset(0);
-      while (cur != bm->size()) {
-        _ct->mark_clean_as_dirty(region_base_idx + cur);
-        _scan_state->set_chunk_dirty(region_base_idx + cur);
-        cur = bm->get_next_one_offset(cur + 1);
-      }
-    }
-
-    void next_sparse_prt(uint const region_idx, SparsePRTEntry::card_elem_t* cards, uint const num_cards) {
-      if (!remember_if_interesting(region_idx)) {
-        return;
-      }
-
-      _merged_sparse++;
-
-      size_t const region_base_idx = (size_t)region_idx << HeapRegion::LogCardsPerRegion;
-      for (uint i = 0; i < num_cards; i++) {
-        size_t card_idx = region_base_idx + cards[i];
-        _ct->mark_clean_as_dirty(card_idx);
-        _scan_state->set_chunk_dirty(card_idx);
-      }
-    }
-
-    virtual bool do_heap_region(HeapRegion* r) {
-      assert(r->in_collection_set() || r->is_starts_humongous(), "must be");
-
-      HeapRegionRemSet* rem_set = r->rem_set();
-      if (!rem_set->is_empty()) {
-        rem_set->iterate_prts(*this);
-      }
-
-      return false;
-    }
-
-    size_t merged_sparse() const { return _merged_sparse; }
-    size_t merged_fine() const { return _merged_fine; }
-    size_t merged_coarse() const { return _merged_coarse; }
-  };
-
-  // Visitor for the remembered sets of humongous candidate regions to merge their
-  // remembered set into the card table.
-  class G1FlushHumongousCandidateRemSets : public HeapRegionClosure {
-    G1MergeCardSetClosure _cl;
-
-  public:
-    G1FlushHumongousCandidateRemSets(G1RemSetScanState* scan_state) : _cl(scan_state) { }
-
-    virtual bool do_heap_region(HeapRegion* r) {
-      G1CollectedHeap* g1h = G1CollectedHeap::heap();
-
-      if (!r->is_starts_humongous() ||
-          !g1h->region_attr(r->hrm_index()).is_humongous() ||
-          r->rem_set()->is_empty()) {
-        return false;
-      }
-
-      guarantee(r->rem_set()->occupancy_less_or_equal_than(G1RSetSparseRegionEntries),
-                "Found a not-small remembered set here. This is inconsistent with previous assumptions.");
-
-      _cl.do_heap_region(r);
-
-      // We should only clear the card based remembered set here as we will not
-      // implicitly rebuild anything else during eager reclaim. Note that at the moment
-      // (and probably never) we do not enter this path if there are other kind of
-      // remembered sets for this region.
-      r->rem_set()->clear_locked(true /* only_cardset */);
-      // Clear_locked() above sets the state to Empty. However we want to continue
-      // collecting remembered set entries for humongous regions that were not
-      // reclaimed.
-      r->rem_set()->set_state_complete();
-#ifdef ASSERT
-      G1HeapRegionAttr region_attr = g1h->region_attr(r->hrm_index());
-      assert(region_attr.needs_remset_update(), "must be");
-#endif
-      assert(r->rem_set()->is_empty(), "At this point any humongous candidate remembered set must be empty.");
-
-      return false;
-    }
-
-    size_t merged_sparse() const { return _cl.merged_sparse(); }
-    size_t merged_fine() const { return _cl.merged_fine(); }
-    size_t merged_coarse() const { return _cl.merged_coarse(); }
-  };
-
-  // Visitor for the log buffer entries to merge them into the card table.
-  class G1MergeLogBufferCardsClosure : public G1CardTableEntryClosure {
-    G1RemSetScanState* _scan_state;
-    G1CardTable* _ct;
-
-    size_t _cards_dirty;
-    size_t _cards_skipped;
-  public:
-    G1MergeLogBufferCardsClosure(G1CollectedHeap* g1h, G1RemSetScanState* scan_state) :
-      _scan_state(scan_state), _ct(g1h->card_table()), _cards_dirty(0), _cards_skipped(0)
-    {}
-
-    bool do_card_ptr(CardValue* card_ptr, uint worker_i) {
-      // The only time we care about recording cards that
-      // contain references that point into the collection set
-      // is during RSet updating within an evacuation pause.
-      // In this case worker_id should be the id of a GC worker thread.
-      assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause");
-
-      uint const region_idx = _ct->region_idx_for(card_ptr);
-
-      // The second clause must come after - the log buffers might contain cards to uncommited
-      // regions.
-      // This code may count duplicate entries in the log buffers (even if rare) multiple
-      // times.
-      if (_scan_state->contains_cards_to_process(region_idx) && (*card_ptr == G1CardTable::dirty_card_val())) {
-        _scan_state->add_dirty_region(region_idx);
-        _scan_state->set_chunk_dirty(_ct->index_for_cardvalue(card_ptr));
-        _cards_dirty++;
-      } else {
-        // We may have had dirty cards in the (initial) collection set (or the
-        // young regions which are always in the initial collection set). We do
-        // not fix their cards here: we already added these regions to the set of
-        // regions to clear the card table at the end during the prepare() phase.
-        _cards_skipped++;
-      }
-      return true;
-    }
-
-    size_t cards_dirty() const { return _cards_dirty; }
-    size_t cards_skipped() const { return _cards_skipped; }
-  };
-
-  class G1MergeHeapRootsTask : public AbstractGangTask {
-    HeapRegionClaimer _hr_claimer;
-    G1RemSetScanState* _scan_state;
-    bool _remembered_set_only;
-
-    G1GCPhaseTimes::GCParPhases _merge_phase;
-
-    volatile bool _fast_reclaim_handled;
-
-  public:
-    G1MergeHeapRootsTask(G1RemSetScanState* scan_state, uint num_workers, bool remembered_set_only, G1GCPhaseTimes::GCParPhases merge_phase) :
-      AbstractGangTask("G1 Merge Heap Roots"),
-      _hr_claimer(num_workers),
-      _scan_state(scan_state),
-      _remembered_set_only(remembered_set_only),
-      _merge_phase(merge_phase),
-      _fast_reclaim_handled(false) { }
-
-    virtual void work(uint worker_id) {
-      G1CollectedHeap* g1h = G1CollectedHeap::heap();
-      G1GCPhaseTimes* p = g1h->phase_times();
-
-      // We schedule flushing the remembered sets of humongous fast reclaim candidates
-      // onto the card table first to allow the remaining parallelized tasks hide it.
-      if (!_remembered_set_only &&
-          p->fast_reclaim_humongous_candidates() > 0 &&
-          !_fast_reclaim_handled &&
-          !Atomic::cmpxchg(true, &_fast_reclaim_handled, false)) {
-
-        G1FlushHumongousCandidateRemSets cl(_scan_state);
-        g1h->heap_region_iterate(&cl);
-
-        p->record_or_add_thread_work_item(_merge_phase, worker_id, cl.merged_sparse(), G1GCPhaseTimes::MergeRSMergedSparse);
-        p->record_or_add_thread_work_item(_merge_phase, worker_id, cl.merged_fine(), G1GCPhaseTimes::MergeRSMergedFine);
-        p->record_or_add_thread_work_item(_merge_phase, worker_id, cl.merged_coarse(), G1GCPhaseTimes::MergeRSMergedCoarse);
-      }
-
-      // Merge remembered sets of current candidates.
-      {
-        G1GCParPhaseTimesTracker x(p, _merge_phase, worker_id, !_remembered_set_only /* must_record */);
-        G1MergeCardSetClosure cl(_scan_state);
-        g1h->collection_set_iterate_increment_from(&cl, &_hr_claimer, worker_id);
-
-        p->record_or_add_thread_work_item(_merge_phase, worker_id, cl.merged_sparse(), G1GCPhaseTimes::MergeRSMergedSparse);
-        p->record_or_add_thread_work_item(_merge_phase, worker_id, cl.merged_fine(), G1GCPhaseTimes::MergeRSMergedFine);
-        p->record_or_add_thread_work_item(_merge_phase, worker_id, cl.merged_coarse(), G1GCPhaseTimes::MergeRSMergedCoarse);
-      }
-
-      // Apply closure to log entries in the HCC.
-      if (!_remembered_set_only && G1HotCardCache::default_use_cache()) {
-        assert(_merge_phase == G1GCPhaseTimes::MergeRS, "Wrong merge phase");
-        G1GCParPhaseTimesTracker x(p, G1GCPhaseTimes::MergeHCC, worker_id);
-        G1MergeLogBufferCardsClosure cl(g1h, _scan_state);
-        g1h->iterate_hcc_closure(&cl, worker_id);
-      }
-
-      // Now apply the closure to all remaining log entries.
-      if (!_remembered_set_only) {
-        assert(_merge_phase == G1GCPhaseTimes::MergeRS, "Wrong merge phase");
-        G1GCParPhaseTimesTracker x(p, G1GCPhaseTimes::MergeLB, worker_id);
-
-        G1MergeLogBufferCardsClosure cl(g1h, _scan_state);
-        g1h->iterate_dirty_card_closure(&cl, worker_id);
-
-        p->record_thread_work_item(G1GCPhaseTimes::MergeLB, worker_id, cl.cards_dirty(), G1GCPhaseTimes::MergeLBDirtyCards);
-        p->record_thread_work_item(G1GCPhaseTimes::MergeLB, worker_id, cl.cards_skipped(), G1GCPhaseTimes::MergeLBSkippedCards);
-      }
-    }
-  };
-
   // Creates a snapshot of the current _top values at the start of collection to
   // filter out card marks that we do not want to scan.
   class G1ResetScanTopClosure : public HeapRegionClosure {
@@ -571,61 +318,46 @@
     }
 
     _all_dirty_regions = new G1DirtyRegions(_max_regions);
+    _next_dirty_regions = new G1DirtyRegions(_max_regions);
 
     G1ResetScanTopClosure cl(this);
     G1CollectedHeap::heap()->heap_region_iterate(&cl);
-
-    _next_dirty_regions = new G1DirtyRegions(_max_regions);
   }
 
-  void print_merge_heap_roots_stats() {
-    size_t num_scan_chunks = 0;
-    for (uint i = 0; i < _max_regions * _scan_chunks_per_region; i++) {
-      if (_region_scan_chunks[i]) {
-        num_scan_chunks++;
-      }
-    }
-    size_t num_visited_cards = num_scan_chunks * CardsPerChunk;
-    size_t total_dirty_region_cards = _next_dirty_regions->size() * HeapRegion::CardsPerRegion;
+  void prepare_for_merge_heap_roots() {
+    _all_dirty_regions->merge(_next_dirty_regions);
 
-    G1CollectedHeap* g1h = G1CollectedHeap::heap();
-    size_t total_old_region_cards =
-      (g1h->num_regions() - (g1h->num_free_regions() - g1h->collection_set()->cur_length())) * HeapRegion::CardsPerRegion;
+    _next_dirty_regions->reset();
+    for (size_t i = 0; i < _max_regions; i++) {
+      _card_table_scan_state[i] = 0;
+    }
 
-    log_debug(gc,remset)("Visited cards " SIZE_FORMAT " Total dirty " SIZE_FORMAT " (%.2lf%%) Total old " SIZE_FORMAT " (%.2lf%%)",
-                         num_visited_cards,
-                         total_dirty_region_cards,
-                         percent_of(num_visited_cards, total_dirty_region_cards),
-                         total_old_region_cards,
-                         percent_of(num_visited_cards, total_old_region_cards));
+    ::memset(_region_scan_chunks, false, _max_regions * _scan_chunks_per_region * sizeof(*_region_scan_chunks));
   }
 
-  void merge_heap_roots(WorkGang* workers, bool remembered_set_only, G1GCPhaseTimes::GCParPhases merge_phase) {
-    {
-      _all_dirty_regions->merge(_next_dirty_regions);
-      _next_dirty_regions->reset();
-      for (size_t i = 0; i < _max_regions; i++) {
-        _card_table_scan_state[i] = 0;
-      }
-
-      ::memset(_region_scan_chunks, false, _max_regions * _scan_chunks_per_region * sizeof(*_region_scan_chunks));
-    }
-
-    size_t const increment_length = G1CollectedHeap::heap()->collection_set()->increment_length();
+  // Returns whether the given region contains cards we need to scan. The remembered
+  // set and other sources may contain cards that
+  // - are in uncommitted regions
+  // - are located in the collection set
+  // - are located in free regions
+  // as we do not clean up remembered sets before merging heap roots.
+  bool contains_cards_to_process(uint const region_idx) const {
+    HeapRegion* hr = G1CollectedHeap::heap()->region_at_or_null(region_idx);
+    return (hr != NULL && !hr->in_collection_set() && hr->is_old_or_humongous_or_archive());
+  }
 
-    uint const num_workers = !remembered_set_only ? workers->active_workers() :
-                                                    MIN2(workers->active_workers(), (uint)increment_length);
+  size_t num_visited_cards() const {
+    size_t result = 0;
+    for (uint i = 0; i < _max_regions * _scan_chunks_per_region; i++) {
+      if (_region_scan_chunks[i]) {
+        result++;
+      }
+    }
+    return result * CardsPerChunk;
+  }
 
-    {
-      G1MergeHeapRootsTask cl(this, num_workers, remembered_set_only, merge_phase);
-      log_debug(gc, ergo)("Running %s using %u workers for " SIZE_FORMAT " regions",
-                          cl.name(), num_workers, increment_length);
-      workers->run_task(&cl, num_workers);
-    }
-
-    if (log_is_enabled(Debug, gc, remset)) {
-      print_merge_heap_roots_stats();
-    }
+  size_t num_cards_in_dirty_regions() const {
+    return _next_dirty_regions->size() * HeapRegion::CardsPerRegion;
   }
 
   void set_chunk_region_dirty(size_t const region_card_idx) {
@@ -1169,8 +901,288 @@
   _scan_state->prepare();
 }
 
+class G1MergeHeapRootsTask : public AbstractGangTask {
+
+  // Visitor for remembered sets, dropping entries onto the card table.
+  class G1MergeCardSetClosure : public HeapRegionClosure {
+    G1RemSetScanState* _scan_state;
+    G1CardTable* _ct;
+
+    uint _merged_sparse;
+    uint _merged_fine;
+    uint _merged_coarse;
+
+    // Returns if the region contains cards we need to scan. If so, remember that
+    // region in the current set of dirty regions.
+    bool remember_if_interesting(uint const region_idx) {
+      if (!_scan_state->contains_cards_to_process(region_idx)) {
+        return false;
+      }
+      _scan_state->add_dirty_region(region_idx);
+      return true;
+    }
+  public:
+    G1MergeCardSetClosure(G1RemSetScanState* scan_state) :
+      _scan_state(scan_state),
+      _ct(G1CollectedHeap::heap()->card_table()),
+      _merged_sparse(0),
+      _merged_fine(0),
+      _merged_coarse(0) { }
+
+    void next_coarse_prt(uint const region_idx) {
+      if (!remember_if_interesting(region_idx)) {
+        return;
+      }
+
+      _merged_coarse++;
+
+      size_t region_base_idx = (size_t)region_idx << HeapRegion::LogCardsPerRegion;
+      _ct->mark_region_dirty(region_base_idx, HeapRegion::CardsPerRegion);
+      _scan_state->set_chunk_region_dirty(region_base_idx);
+    }
+
+    void next_fine_prt(uint const region_idx, BitMap* bm) {
+      if (!remember_if_interesting(region_idx)) {
+        return;
+      }
+
+      _merged_fine++;
+
+      size_t const region_base_idx = (size_t)region_idx << HeapRegion::LogCardsPerRegion;
+      BitMap::idx_t cur = bm->get_next_one_offset(0);
+      while (cur != bm->size()) {
+        _ct->mark_clean_as_dirty(region_base_idx + cur);
+        _scan_state->set_chunk_dirty(region_base_idx + cur);
+        cur = bm->get_next_one_offset(cur + 1);
+      }
+    }
+
+    void next_sparse_prt(uint const region_idx, SparsePRTEntry::card_elem_t* cards, uint const num_cards) {
+      if (!remember_if_interesting(region_idx)) {
+        return;
+      }
+
+      _merged_sparse++;
+
+      size_t const region_base_idx = (size_t)region_idx << HeapRegion::LogCardsPerRegion;
+      for (uint i = 0; i < num_cards; i++) {
+        size_t card_idx = region_base_idx + cards[i];
+        _ct->mark_clean_as_dirty(card_idx);
+        _scan_state->set_chunk_dirty(card_idx);
+      }
+    }
+
+    virtual bool do_heap_region(HeapRegion* r) {
+      assert(r->in_collection_set() || r->is_starts_humongous(), "must be");
+
+      HeapRegionRemSet* rem_set = r->rem_set();
+      if (!rem_set->is_empty()) {
+        rem_set->iterate_prts(*this);
+      }
+
+      return false;
+    }
+
+    size_t merged_sparse() const { return _merged_sparse; }
+    size_t merged_fine() const { return _merged_fine; }
+    size_t merged_coarse() const { return _merged_coarse; }
+  };
+
+  // Visitor for the remembered sets of humongous candidate regions to merge their
+  // remembered set into the card table.
+  class G1FlushHumongousCandidateRemSets : public HeapRegionClosure {
+    G1MergeCardSetClosure _cl;
+
+  public:
+    G1FlushHumongousCandidateRemSets(G1RemSetScanState* scan_state) : _cl(scan_state) { }
+
+    virtual bool do_heap_region(HeapRegion* r) {
+      G1CollectedHeap* g1h = G1CollectedHeap::heap();
+
+      if (!r->is_starts_humongous() ||
+          !g1h->region_attr(r->hrm_index()).is_humongous() ||
+          r->rem_set()->is_empty()) {
+        return false;
+      }
+
+      guarantee(r->rem_set()->occupancy_less_or_equal_than(G1RSetSparseRegionEntries),
+                "Found a not-small remembered set here. This is inconsistent with previous assumptions.");
+
+      _cl.do_heap_region(r);
+
+      // We should only clear the card based remembered set here as we will not
+      // implicitly rebuild anything else during eager reclaim. Note that at the moment
+      // (and probably never) we do not enter this path if there are other kind of
+      // remembered sets for this region.
+      r->rem_set()->clear_locked(true /* only_cardset */);
+      // Clear_locked() above sets the state to Empty. However we want to continue
+      // collecting remembered set entries for humongous regions that were not
+      // reclaimed.
+      r->rem_set()->set_state_complete();
+#ifdef ASSERT
+      G1HeapRegionAttr region_attr = g1h->region_attr(r->hrm_index());
+      assert(region_attr.needs_remset_update(), "must be");
+#endif
+      assert(r->rem_set()->is_empty(), "At this point any humongous candidate remembered set must be empty.");
+
+      return false;
+    }
+
+    size_t merged_sparse() const { return _cl.merged_sparse(); }
+    size_t merged_fine() const { return _cl.merged_fine(); }
+    size_t merged_coarse() const { return _cl.merged_coarse(); }
+  };
+
+  // Visitor for the log buffer entries to merge them into the card table.
+  class G1MergeLogBufferCardsClosure : public G1CardTableEntryClosure {
+    G1RemSetScanState* _scan_state;
+    G1CardTable* _ct;
+
+    size_t _cards_dirty;
+    size_t _cards_skipped;
+  public:
+    G1MergeLogBufferCardsClosure(G1CollectedHeap* g1h, G1RemSetScanState* scan_state) :
+      _scan_state(scan_state), _ct(g1h->card_table()), _cards_dirty(0), _cards_skipped(0)
+    {}
+
+    bool do_card_ptr(CardValue* card_ptr, uint worker_i) {
+      // The only time we care about recording cards that
+      // contain references that point into the collection set
+      // is during RSet updating within an evacuation pause.
+      // In this case worker_id should be the id of a GC worker thread.
+      assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause");
+
+      uint const region_idx = _ct->region_idx_for(card_ptr);
+
+      // The second clause must come after - the log buffers might contain cards to uncommited
+      // regions.
+      // This code may count duplicate entries in the log buffers (even if rare) multiple
+      // times.
+      if (_scan_state->contains_cards_to_process(region_idx) && (*card_ptr == G1CardTable::dirty_card_val())) {
+        _scan_state->add_dirty_region(region_idx);
+        _scan_state->set_chunk_dirty(_ct->index_for_cardvalue(card_ptr));
+        _cards_dirty++;
+      } else {
+        // We may have had dirty cards in the (initial) collection set (or the
+        // young regions which are always in the initial collection set). We do
+        // not fix their cards here: we already added these regions to the set of
+        // regions to clear the card table at the end during the prepare() phase.
+        _cards_skipped++;
+      }
+      return true;
+    }
+
+    size_t cards_dirty() const { return _cards_dirty; }
+    size_t cards_skipped() const { return _cards_skipped; }
+  };
+
+  HeapRegionClaimer _hr_claimer;
+  G1RemSetScanState* _scan_state;
+  bool _remembered_set_only;
+
+  G1GCPhaseTimes::GCParPhases _merge_phase;
+
+  volatile bool _fast_reclaim_handled;
+
+public:
+  G1MergeHeapRootsTask(G1RemSetScanState* scan_state, uint num_workers, bool remembered_set_only, G1GCPhaseTimes::GCParPhases merge_phase) :
+    AbstractGangTask("G1 Merge Heap Roots"),
+    _hr_claimer(num_workers),
+    _scan_state(scan_state),
+    _remembered_set_only(remembered_set_only),
+    _merge_phase(merge_phase),
+    _fast_reclaim_handled(false) { }
+
+  virtual void work(uint worker_id) {
+    G1CollectedHeap* g1h = G1CollectedHeap::heap();
+    G1GCPhaseTimes* p = g1h->phase_times();
+
+    // We schedule flushing the remembered sets of humongous fast reclaim candidates
+    // onto the card table first to allow the remaining parallelized tasks hide it.
+    if (!_remembered_set_only &&
+        p->fast_reclaim_humongous_candidates() > 0 &&
+        !_fast_reclaim_handled &&
+        !Atomic::cmpxchg(true, &_fast_reclaim_handled, false)) {
+
+      G1FlushHumongousCandidateRemSets cl(_scan_state);
+      g1h->heap_region_iterate(&cl);
+
+      p->record_or_add_thread_work_item(_merge_phase, worker_id, cl.merged_sparse(), G1GCPhaseTimes::MergeRSMergedSparse);
+      p->record_or_add_thread_work_item(_merge_phase, worker_id, cl.merged_fine(), G1GCPhaseTimes::MergeRSMergedFine);
+      p->record_or_add_thread_work_item(_merge_phase, worker_id, cl.merged_coarse(), G1GCPhaseTimes::MergeRSMergedCoarse);
+    }
+
+    // Merge remembered sets of current candidates.
+    {
+      G1GCParPhaseTimesTracker x(p, _merge_phase, worker_id, !_remembered_set_only /* must_record */);
+      G1MergeCardSetClosure cl(_scan_state);
+      g1h->collection_set_iterate_increment_from(&cl, &_hr_claimer, worker_id);
+
+      p->record_or_add_thread_work_item(_merge_phase, worker_id, cl.merged_sparse(), G1GCPhaseTimes::MergeRSMergedSparse);
+      p->record_or_add_thread_work_item(_merge_phase, worker_id, cl.merged_fine(), G1GCPhaseTimes::MergeRSMergedFine);
+      p->record_or_add_thread_work_item(_merge_phase, worker_id, cl.merged_coarse(), G1GCPhaseTimes::MergeRSMergedCoarse);
+    }
+
+    // Apply closure to log entries in the HCC.
+    if (!_remembered_set_only && G1HotCardCache::default_use_cache()) {
+      assert(_merge_phase == G1GCPhaseTimes::MergeRS, "Wrong merge phase");
+      G1GCParPhaseTimesTracker x(p, G1GCPhaseTimes::MergeHCC, worker_id);
+      G1MergeLogBufferCardsClosure cl(g1h, _scan_state);
+      g1h->iterate_hcc_closure(&cl, worker_id);
+    }
+
+    // Now apply the closure to all remaining log entries.
+    if (!_remembered_set_only) {
+      assert(_merge_phase == G1GCPhaseTimes::MergeRS, "Wrong merge phase");
+      G1GCParPhaseTimesTracker x(p, G1GCPhaseTimes::MergeLB, worker_id);
+
+      G1MergeLogBufferCardsClosure cl(g1h, _scan_state);
+      g1h->iterate_dirty_card_closure(&cl, worker_id);
+
+      p->record_thread_work_item(G1GCPhaseTimes::MergeLB, worker_id, cl.cards_dirty(), G1GCPhaseTimes::MergeLBDirtyCards);
+      p->record_thread_work_item(G1GCPhaseTimes::MergeLB, worker_id, cl.cards_skipped(), G1GCPhaseTimes::MergeLBSkippedCards);
+    }
+  }
+};
+
+void G1RemSet::print_merge_heap_roots_stats() {
+  size_t num_visited_cards = _scan_state->num_visited_cards();
+
+  size_t total_dirty_region_cards = _scan_state->num_cards_in_dirty_regions();
+
+  G1CollectedHeap* g1h = G1CollectedHeap::heap();
+  size_t total_old_region_cards =
+    (g1h->num_regions() - (g1h->num_free_regions() - g1h->collection_set()->cur_length())) * HeapRegion::CardsPerRegion;
+
+  log_debug(gc,remset)("Visited cards " SIZE_FORMAT " Total dirty " SIZE_FORMAT " (%.2lf%%) Total old " SIZE_FORMAT " (%.2lf%%)",
+                       num_visited_cards,
+                       total_dirty_region_cards,
+                       percent_of(num_visited_cards, total_dirty_region_cards),
+                       total_old_region_cards,
+                       percent_of(num_visited_cards, total_old_region_cards));
+}
+
 void G1RemSet::merge_heap_roots(bool remembered_set_only, G1GCPhaseTimes::GCParPhases merge_phase) {
-  _scan_state->merge_heap_roots(_g1h->workers(), remembered_set_only, merge_phase);
+  {
+    _scan_state->prepare_for_merge_heap_roots();
+  }
+
+  WorkGang* workers = G1CollectedHeap::heap()->workers();
+  size_t const increment_length = G1CollectedHeap::heap()->collection_set()->increment_length();
+
+  uint const num_workers = !remembered_set_only ? workers->active_workers() :
+                                                  MIN2(workers->active_workers(), (uint)increment_length);
+
+  {
+    G1MergeHeapRootsTask cl(_scan_state, num_workers, remembered_set_only, merge_phase);
+    log_debug(gc, ergo)("Running %s using %u workers for " SIZE_FORMAT " regions",
+                        cl.name(), num_workers, increment_length);
+    workers->run_task(&cl, num_workers);
+  }
+
+  if (log_is_enabled(Debug, gc, remset)) {
+    print_merge_heap_roots_stats();
+  }
 }
 
 void G1RemSet::prepare_for_scan_heap_roots(uint region_idx) {
--- a/src/hotspot/share/gc/g1/g1RemSet.hpp	Wed Jul 17 23:22:48 2019 +0900
+++ b/src/hotspot/share/gc/g1/g1RemSet.hpp	Wed Jul 17 16:33:19 2019 +0200
@@ -67,6 +67,7 @@
   G1Policy*              _g1p;
   G1HotCardCache*        _hot_card_cache;
 
+  void print_merge_heap_roots_stats();
 public:
 
   typedef CardTable::CardValue CardValue;