/*

 * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "gc/g1/g1BarrierSet.hpp"

#include "gc/g1/g1BlockOffsetTable.inline.hpp"

#include "gc/g1/g1CardTable.inline.hpp"

#include "gc/g1/g1CollectedHeap.inline.hpp"

#include "gc/g1/g1ConcurrentRefine.hpp"

#include "gc/g1/g1DirtyCardQueue.hpp"

#include "gc/g1/g1FromCardCache.hpp"

#include "gc/g1/g1GCPhaseTimes.hpp"

#include "gc/g1/g1HotCardCache.hpp"

#include "gc/g1/g1OopClosures.inline.hpp"

#include "gc/g1/g1RootClosures.hpp"

#include "gc/g1/g1RemSet.hpp"

#include "gc/g1/heapRegion.inline.hpp"

#include "gc/g1/heapRegionManager.inline.hpp"

#include "gc/g1/heapRegionRemSet.hpp"

#include "gc/shared/gcTraceTime.inline.hpp"

#include "gc/shared/suspendibleThreadSet.hpp"

#include "jfr/jfrEvents.hpp"

#include "memory/iterator.hpp"

#include "memory/resourceArea.hpp"

#include "oops/access.inline.hpp"

#include "oops/oop.inline.hpp"

#include "runtime/os.hpp"

#include "utilities/align.hpp"

#include "utilities/globalDefinitions.hpp"

#include "utilities/stack.inline.hpp"

#include "utilities/ticks.hpp"

// Collects information about the overall remembered set scan progress during an evacuation.

class G1RemSetScanState : public CHeapObj<mtGC> {

private:

  class G1ClearCardTableTask : public AbstractGangTask {

    G1CollectedHeap* _g1h;

    uint* _dirty_region_list;

    size_t _num_dirty_regions;

    size_t _chunk_length;

    size_t volatile _cur_dirty_regions;

  public:

    G1ClearCardTableTask(G1CollectedHeap* g1h,

                         uint* dirty_region_list,

                         size_t num_dirty_regions,

                         size_t chunk_length) :

      AbstractGangTask("G1 Clear Card Table Task"),

      _g1h(g1h),

      _dirty_region_list(dirty_region_list),

      _num_dirty_regions(num_dirty_regions),

      _chunk_length(chunk_length),

      _cur_dirty_regions(0) {

      assert(chunk_length > 0, "must be");

    }

    static size_t chunk_size() { return M; }

    void work(uint worker_id) {

      while (_cur_dirty_regions < _num_dirty_regions) {

        size_t next = Atomic::add(_chunk_length, &_cur_dirty_regions) - _chunk_length;

        size_t max = MIN2(next + _chunk_length, _num_dirty_regions);

        for (size_t i = next; i < max; i++) {

          HeapRegion* r = _g1h->region_at(_dirty_region_list[i]);

          if (!r->is_survivor()) {

            r->clear_cardtable();

          }

        }

      }

    }

  };

  size_t _max_regions;

  // Scan progress for the remembered set of a single region. Transitions from

  // Unclaimed -> Claimed -> Complete.

  // At each of the transitions the thread that does the transition needs to perform

  // some special action once. This is the reason for the extra "Claimed" state.

  typedef jint G1RemsetIterState;

  static const G1RemsetIterState Unclaimed = 0; // The remembered set has not been scanned yet.

  static const G1RemsetIterState Claimed = 1;   // The remembered set is currently being scanned.

  static const G1RemsetIterState Complete = 2;  // The remembered set has been completely scanned.

  G1RemsetIterState volatile* _iter_states;

  // The current location where the next thread should continue scanning in a region's

  // remembered set.

  size_t volatile* _iter_claims;

  // Temporary buffer holding the regions we used to store remembered set scan duplicate

  // information. These are also called "dirty". Valid entries are from [0.._cur_dirty_region)

  uint* _dirty_region_buffer;

  typedef jbyte IsDirtyRegionState;

  static const IsDirtyRegionState Clean = 0;

  static const IsDirtyRegionState Dirty = 1;

  // Holds a flag for every region whether it is in the _dirty_region_buffer already

  // to avoid duplicates. Uses jbyte since there are no atomic instructions for bools.

  IsDirtyRegionState* _in_dirty_region_buffer;

  size_t _cur_dirty_region;

  // Creates a snapshot of the current _top values at the start of collection to

  // filter out card marks that we do not want to scan.

  class G1ResetScanTopClosure : public HeapRegionClosure {

  private:

    HeapWord** _scan_top;

  public:

    G1ResetScanTopClosure(HeapWord** scan_top) : _scan_top(scan_top) { }

    virtual bool do_heap_region(HeapRegion* r) {

      uint hrm_index = r->hrm_index();

      if (!r->in_collection_set() && r->is_old_or_humongous_or_archive() && !r->is_empty()) {

        _scan_top[hrm_index] = r->top();

      } else {

        _scan_top[hrm_index] = NULL;

      }

      return false;

    }

  };

  // For each region, contains the maximum top() value to be used during this garbage

  // collection. Subsumes common checks like filtering out everything but old and

  // humongous regions outside the collection set.

  // This is valid because we are not interested in scanning stray remembered set

  // entries from free or archive regions.

  HeapWord** _scan_top;

public:

  G1RemSetScanState() :

    _max_regions(0),

    _iter_states(NULL),

    _iter_claims(NULL),

    _dirty_region_buffer(NULL),

    _in_dirty_region_buffer(NULL),

    _cur_dirty_region(0),

    _scan_top(NULL) {

  }

  ~G1RemSetScanState() {

    if (_iter_states != NULL) {

      FREE_C_HEAP_ARRAY(G1RemsetIterState, _iter_states);

    }

    if (_iter_claims != NULL) {

      FREE_C_HEAP_ARRAY(size_t, _iter_claims);

    }

    if (_dirty_region_buffer != NULL) {

      FREE_C_HEAP_ARRAY(uint, _dirty_region_buffer);

    }

    if (_in_dirty_region_buffer != NULL) {

      FREE_C_HEAP_ARRAY(IsDirtyRegionState, _in_dirty_region_buffer);

    }

    if (_scan_top != NULL) {

      FREE_C_HEAP_ARRAY(HeapWord*, _scan_top);

    }

  }

  void initialize(uint max_regions) {

    assert(_iter_states == NULL, "Must not be initialized twice");

    assert(_iter_claims == NULL, "Must not be initialized twice");

    _max_regions = max_regions;

    _iter_states = NEW_C_HEAP_ARRAY(G1RemsetIterState, max_regions, mtGC);

    _iter_claims = NEW_C_HEAP_ARRAY(size_t, max_regions, mtGC);

    _dirty_region_buffer = NEW_C_HEAP_ARRAY(uint, max_regions, mtGC);

    _in_dirty_region_buffer = NEW_C_HEAP_ARRAY(IsDirtyRegionState, max_regions, mtGC);

    _scan_top = NEW_C_HEAP_ARRAY(HeapWord*, max_regions, mtGC);

  }

  void reset() {

    for (uint i = 0; i < _max_regions; i++) {

      _iter_states[i] = Unclaimed;

      _scan_top[i] = NULL;

    }

    G1ResetScanTopClosure cl(_scan_top);

    G1CollectedHeap::heap()->heap_region_iterate(&cl);

    memset((void*)_iter_claims, 0, _max_regions * sizeof(size_t));

    memset(_in_dirty_region_buffer, Clean, _max_regions * sizeof(IsDirtyRegionState));

    _cur_dirty_region = 0;

  }

  // Attempt to claim the remembered set of the region for iteration. Returns true

  // if this call caused the transition from Unclaimed to Claimed.

  inline bool claim_iter(uint region) {

    assert(region < _max_regions, "Tried to access invalid region %u", region);

    if (_iter_states[region] != Unclaimed) {

      return false;

    }

    G1RemsetIterState res = Atomic::cmpxchg(Claimed, &_iter_states[region], Unclaimed);

    return (res == Unclaimed);

  }

  // Try to atomically sets the iteration state to "complete". Returns true for the

  // thread that caused the transition.

  inline bool set_iter_complete(uint region) {

    if (iter_is_complete(region)) {

      return false;

    }

    G1RemsetIterState res = Atomic::cmpxchg(Complete, &_iter_states[region], Claimed);

    return (res == Claimed);

  }

  // Returns true if the region's iteration is complete.

  inline bool iter_is_complete(uint region) const {

    assert(region < _max_regions, "Tried to access invalid region %u", region);

    return _iter_states[region] == Complete;

  }

  // The current position within the remembered set of the given region.

  inline size_t iter_claimed(uint region) const {

    assert(region < _max_regions, "Tried to access invalid region %u", region);

    return _iter_claims[region];

  }

  // Claim the next block of cards within the remembered set of the region with

  // step size.

  inline size_t iter_claimed_next(uint region, size_t step) {

    return Atomic::add(step, &_iter_claims[region]) - step;

  }

  void add_dirty_region(uint region) {

    if (_in_dirty_region_buffer[region] == Dirty) {

      return;

    }

    bool marked_as_dirty = Atomic::cmpxchg(Dirty, &_in_dirty_region_buffer[region], Clean) == Clean;

    if (marked_as_dirty) {

      size_t allocated = Atomic::add(1u, &_cur_dirty_region) - 1;

      _dirty_region_buffer[allocated] = region;

    }

  }

  HeapWord* scan_top(uint region_idx) const {

    return _scan_top[region_idx];

  }

  // Clear the card table of "dirty" regions.

  void clear_card_table(WorkGang* workers) {

    if (_cur_dirty_region == 0) {

      return;

    }

    size_t const num_chunks = align_up(_cur_dirty_region * HeapRegion::CardsPerRegion, G1ClearCardTableTask::chunk_size()) / G1ClearCardTableTask::chunk_size();

    uint const num_workers = (uint)MIN2(num_chunks, (size_t)workers->active_workers());

    size_t const chunk_length = G1ClearCardTableTask::chunk_size() / HeapRegion::CardsPerRegion;

    // Iterate over the dirty cards region list.

    G1ClearCardTableTask cl(G1CollectedHeap::heap(), _dirty_region_buffer, _cur_dirty_region, chunk_length);

    log_debug(gc, ergo)("Running %s using %u workers for " SIZE_FORMAT " "

                        "units of work for " SIZE_FORMAT " regions.",

                        cl.name(), num_workers, num_chunks, _cur_dirty_region);

    workers->run_task(&cl, num_workers);

#ifndef PRODUCT

    G1CollectedHeap::heap()->verifier()->verify_card_table_cleanup();

#endif

  }

};

G1RemSet::G1RemSet(G1CollectedHeap* g1h,

                   G1CardTable* ct,

                   G1HotCardCache* hot_card_cache) :

  _scan_state(new G1RemSetScanState()),

  _prev_period_summary(),

  _g1h(g1h),

  _num_conc_refined_cards(0),

  _ct(ct),

  _g1p(_g1h->g1_policy()),

  _hot_card_cache(hot_card_cache) {

}

G1RemSet::~G1RemSet() {

  if (_scan_state != NULL) {

    delete _scan_state;

  }

}

uint G1RemSet::num_par_rem_sets() {

  return G1DirtyCardQueueSet::num_par_ids() + G1ConcurrentRefine::max_num_threads() + MAX2(ConcGCThreads, ParallelGCThreads);

}

void G1RemSet::initialize(size_t capacity, uint max_regions) {

  G1FromCardCache::initialize(num_par_rem_sets(), max_regions);

  _scan_state->initialize(max_regions);

}

G1ScanRSForRegionClosure::G1ScanRSForRegionClosure(G1RemSetScanState* scan_state,

                                                   G1ScanObjsDuringScanRSClosure* scan_obj_on_card,

                                                   G1ParScanThreadState* pss,

                                                   G1GCPhaseTimes::GCParPhases phase,

                                                   uint worker_i) :

  _g1h(G1CollectedHeap::heap()),

  _ct(_g1h->card_table()),

  _pss(pss),

  _scan_objs_on_card_cl(scan_obj_on_card),

  _scan_state(scan_state),

  _phase(phase),

  _worker_i(worker_i),

  _cards_scanned(0),

  _cards_claimed(0),

  _cards_skipped(0),

  _rem_set_root_scan_time(),

  _rem_set_trim_partially_time(),

  _strong_code_root_scan_time(),

  _strong_code_trim_partially_time() {

}

void G1ScanRSForRegionClosure::claim_card(size_t card_index, const uint region_idx_for_card){

  _ct->set_card_claimed(card_index);

  _scan_state->add_dirty_region(region_idx_for_card);

}

void G1ScanRSForRegionClosure::scan_card(MemRegion mr, uint region_idx_for_card) {

  HeapRegion* const card_region = _g1h->region_at(region_idx_for_card);

  assert(!card_region->is_young(), "Should not scan card in young region %u", region_idx_for_card);

  card_region->oops_on_card_seq_iterate_careful<true>(mr, _scan_objs_on_card_cl);

  _scan_objs_on_card_cl->trim_queue_partially();

  _cards_scanned++;

}

void G1ScanRSForRegionClosure::scan_rem_set_roots(HeapRegion* r) {

  EventGCPhaseParallel event;

  uint const region_idx = r->hrm_index();

  if (_scan_state->claim_iter(region_idx)) {

    // If we ever free the collection set concurrently, we should also

    // clear the card table concurrently therefore we won't need to

    // add regions of the collection set to the dirty cards region.

    _scan_state->add_dirty_region(region_idx);

  }

  if (r->rem_set()->cardset_is_empty()) {

    return;

  }

  // We claim cards in blocks so as to reduce the contention.

  size_t const block_size = G1RSetScanBlockSize;

  HeapRegionRemSetIterator iter(r->rem_set());

  size_t card_index;

  size_t claimed_card_block = _scan_state->iter_claimed_next(region_idx, block_size);

  for (size_t current_card = 0; iter.has_next(card_index); current_card++) {

    if (current_card >= claimed_card_block + block_size) {

      claimed_card_block = _scan_state->iter_claimed_next(region_idx, block_size);

    }

    if (current_card < claimed_card_block) {

      _cards_skipped++;

      continue;

    }

    _cards_claimed++;

    HeapWord* const card_start = _g1h->bot()->address_for_index_raw(card_index);

    uint const region_idx_for_card = _g1h->addr_to_region(card_start);

#ifdef ASSERT

    HeapRegion* hr = _g1h->region_at_or_null(region_idx_for_card);

    assert(hr == NULL || hr->is_in_reserved(card_start),

           "Card start " PTR_FORMAT " to scan outside of region %u", p2i(card_start), _g1h->region_at(region_idx_for_card)->hrm_index());

#endif

    HeapWord* const top = _scan_state->scan_top(region_idx_for_card);

    if (card_start >= top) {

      continue;

    }

    // If the card is dirty, then G1 will scan it during Update RS.

    if (_ct->is_card_claimed(card_index) || _ct->is_card_dirty(card_index)) {

      continue;

    }

    // We claim lazily (so races are possible but they're benign), which reduces the

    // number of duplicate scans (the rsets of the regions in the cset can intersect).

    // Claim the card after checking bounds above: the remembered set may contain

    // random cards into current survivor, and we would then have an incorrectly

    // claimed card in survivor space. Card table clear does not reset the card table

    // of survivor space regions.

    claim_card(card_index, region_idx_for_card);

    MemRegion const mr(card_start, MIN2(card_start + BOTConstants::N_words, top));

    scan_card(mr, region_idx_for_card);

  }

  event.commit(GCId::current(), _worker_i, G1GCPhaseTimes::phase_name(_phase));

}

void G1ScanRSForRegionClosure::scan_strong_code_roots(HeapRegion* r) {

  EventGCPhaseParallel event;

  // We pass a weak code blobs closure to the remembered set scanning because we want to avoid

  // treating the nmethods visited to act as roots for concurrent marking.

  // We only want to make sure that the oops in the nmethods are adjusted with regard to the

  // objects copied by the current evacuation.

  r->strong_code_roots_do(_pss->closures()->weak_codeblobs());

  event.commit(GCId::current(), _worker_i, G1GCPhaseTimes::phase_name(G1GCPhaseTimes::CodeRoots));

}

bool G1ScanRSForRegionClosure::do_heap_region(HeapRegion* r) {

  assert(r->in_collection_set(),

         "Should only be called on elements of the collection set but region %u is not.",

         r->hrm_index());

  uint const region_idx = r->hrm_index();

  // Do an early out if we know we are complete.

  if (_scan_state->iter_is_complete(region_idx)) {

    return false;

  }

  {

    G1EvacPhaseWithTrimTimeTracker timer(_pss, _rem_set_root_scan_time, _rem_set_trim_partially_time);

    scan_rem_set_roots(r);

  }

  if (_scan_state->set_iter_complete(region_idx)) {

    G1EvacPhaseWithTrimTimeTracker timer(_pss, _strong_code_root_scan_time, _strong_code_trim_partially_time);

    // Scan the strong code root list attached to the current region

    scan_strong_code_roots(r);

  }

  return false;

}

void G1RemSet::scan_rem_set(G1ParScanThreadState* pss, uint worker_i) {

  G1ScanObjsDuringScanRSClosure scan_cl(_g1h, pss);

  G1ScanRSForRegionClosure cl(_scan_state, &scan_cl, pss, G1GCPhaseTimes::ScanRS, worker_i);

  _g1h->collection_set_iterate_from(&cl, worker_i);

  G1GCPhaseTimes* p = _g1p->phase_times();

  p->record_time_secs(G1GCPhaseTimes::ScanRS, worker_i, cl.rem_set_root_scan_time().seconds());

  p->add_time_secs(G1GCPhaseTimes::ObjCopy, worker_i, cl.rem_set_trim_partially_time().seconds());

  p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_scanned(), G1GCPhaseTimes::ScanRSScannedCards);

  p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_claimed(), G1GCPhaseTimes::ScanRSClaimedCards);

  p->record_thread_work_item(G1GCPhaseTimes::ScanRS, worker_i, cl.cards_skipped(), G1GCPhaseTimes::ScanRSSkippedCards);

  p->record_time_secs(G1GCPhaseTimes::CodeRoots, worker_i, cl.strong_code_root_scan_time().seconds());

  p->add_time_secs(G1GCPhaseTimes::ObjCopy, worker_i, cl.strong_code_root_trim_partially_time().seconds());

}

// Closure used for updating rem sets. Only called during an evacuation pause.

class G1RefineCardClosure: public G1CardTableEntryClosure {

  G1RemSet* _g1rs;

  G1ScanObjsDuringUpdateRSClosure* _update_rs_cl;

  size_t _cards_scanned;

  size_t _cards_skipped;

public:

  G1RefineCardClosure(G1CollectedHeap* g1h, G1ScanObjsDuringUpdateRSClosure* update_rs_cl) :

    _g1rs(g1h->g1_rem_set()), _update_rs_cl(update_rs_cl), _cards_scanned(0), _cards_skipped(0)

  {}

  bool do_card_ptr(jbyte* card_ptr, uint worker_i) {

    // The only time we care about recording cards that

    // contain references that point into the collection set

    // is during RSet updating within an evacuation pause.

    // In this case worker_i should be the id of a GC worker thread.

    assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause");

    bool card_scanned = _g1rs->refine_card_during_gc(card_ptr, _update_rs_cl);

    if (card_scanned) {

      _update_rs_cl->trim_queue_partially();

      _cards_scanned++;

    } else {

      _cards_skipped++;