/*

 * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "gc/g1/g1BarrierSet.hpp"

#include "gc/g1/g1BlockOffsetTable.inline.hpp"

#include "gc/g1/g1CardTable.inline.hpp"

#include "gc/g1/g1CollectedHeap.inline.hpp"

#include "gc/g1/g1ConcurrentRefine.hpp"

#include "gc/g1/g1DirtyCardQueue.hpp"

#include "gc/g1/g1FromCardCache.hpp"

#include "gc/g1/g1GCPhaseTimes.hpp"

#include "gc/g1/g1HotCardCache.hpp"

#include "gc/g1/g1OopClosures.inline.hpp"

#include "gc/g1/g1RootClosures.hpp"

#include "gc/g1/g1RemSet.hpp"

#include "gc/g1/g1SharedDirtyCardQueue.hpp"

#include "gc/g1/heapRegion.inline.hpp"

#include "gc/g1/heapRegionManager.inline.hpp"

#include "gc/g1/heapRegionRemSet.hpp"

#include "gc/shared/gcTraceTime.inline.hpp"

#include "gc/shared/suspendibleThreadSet.hpp"

#include "jfr/jfrEvents.hpp"

#include "memory/iterator.hpp"

#include "memory/resourceArea.hpp"

#include "oops/access.inline.hpp"

#include "oops/oop.inline.hpp"

#include "runtime/os.hpp"

#include "utilities/align.hpp"

#include "utilities/globalDefinitions.hpp"

#include "utilities/stack.inline.hpp"

#include "utilities/ticks.hpp"

// Collects information about the overall remembered set scan progress during an evacuation.

class G1RemSetScanState : public CHeapObj<mtGC> {

private:

  class G1ClearCardTableTask : public AbstractGangTask {

    G1CollectedHeap* _g1h;

    uint* _dirty_region_list;

    size_t _num_dirty_regions;

    size_t _chunk_length;

    size_t volatile _cur_dirty_regions;

  public:

    G1ClearCardTableTask(G1CollectedHeap* g1h,

                         uint* dirty_region_list,

                         size_t num_dirty_regions,

                         size_t chunk_length) :

      AbstractGangTask("G1 Clear Card Table Task"),

      _g1h(g1h),

      _dirty_region_list(dirty_region_list),

      _num_dirty_regions(num_dirty_regions),

      _chunk_length(chunk_length),

      _cur_dirty_regions(0) {

      assert(chunk_length > 0, "must be");

    }

    static size_t chunk_size() { return M; }

    void work(uint worker_id) {

      while (_cur_dirty_regions < _num_dirty_regions) {

        size_t next = Atomic::add(_chunk_length, &_cur_dirty_regions) - _chunk_length;

        size_t max = MIN2(next + _chunk_length, _num_dirty_regions);

        for (size_t i = next; i < max; i++) {

          HeapRegion* r = _g1h->region_at(_dirty_region_list[i]);

          if (!r->is_survivor()) {

            r->clear_cardtable();

          }

        }

      }

    }

  };

  size_t _max_regions;

  // Scan progress for the remembered set of a single region. Transitions from

  // Unclaimed -> Claimed -> Complete.

  // At each of the transitions the thread that does the transition needs to perform

  // some special action once. This is the reason for the extra "Claimed" state.

  typedef jint G1RemsetIterState;

  static const G1RemsetIterState Unclaimed = 0; // The remembered set has not been scanned yet.

  static const G1RemsetIterState Claimed = 1;   // The remembered set is currently being scanned.

  static const G1RemsetIterState Complete = 2;  // The remembered set has been completely scanned.

  G1RemsetIterState volatile* _iter_states;

  // The current location where the next thread should continue scanning in a region's

  // remembered set.

  size_t volatile* _iter_claims;

  // Temporary buffer holding the regions we used to store remembered set scan duplicate

  // information. These are also called "dirty". Valid entries are from [0.._cur_dirty_region)

  uint* _dirty_region_buffer;

  // Flag for every region whether it is in the _dirty_region_buffer already

  // to avoid duplicates.

  bool volatile* _in_dirty_region_buffer;

  size_t _cur_dirty_region;

  // Creates a snapshot of the current _top values at the start of collection to

  // filter out card marks that we do not want to scan.

  class G1ResetScanTopClosure : public HeapRegionClosure {

  private:

    HeapWord** _scan_top;

  public:

    G1ResetScanTopClosure(HeapWord** scan_top) : _scan_top(scan_top) { }

    virtual bool do_heap_region(HeapRegion* r) {

      uint hrm_index = r->hrm_index();

      if (!r->in_collection_set() && r->is_old_or_humongous_or_archive() && !r->is_empty()) {

        _scan_top[hrm_index] = r->top();

      } else {

        _scan_top[hrm_index] = NULL;

      }

      return false;

    }

  };

  // For each region, contains the maximum top() value to be used during this garbage

  // collection. Subsumes common checks like filtering out everything but old and

  // humongous regions outside the collection set.

  // This is valid because we are not interested in scanning stray remembered set

  // entries from free or archive regions.

  HeapWord** _scan_top;

public:

  G1RemSetScanState() :

    _max_regions(0),

    _iter_states(NULL),

    _iter_claims(NULL),

    _dirty_region_buffer(NULL),

    _in_dirty_region_buffer(NULL),

    _cur_dirty_region(0),

    _scan_top(NULL) {

  }

  ~G1RemSetScanState() {

    if (_iter_states != NULL) {

      FREE_C_HEAP_ARRAY(G1RemsetIterState, _iter_states);

    }

    if (_iter_claims != NULL) {

      FREE_C_HEAP_ARRAY(size_t, _iter_claims);

    }

    if (_dirty_region_buffer != NULL) {

      FREE_C_HEAP_ARRAY(uint, _dirty_region_buffer);

    }

    if (_in_dirty_region_buffer != NULL) {

      FREE_C_HEAP_ARRAY(bool, _in_dirty_region_buffer);

    }

    if (_scan_top != NULL) {

      FREE_C_HEAP_ARRAY(HeapWord*, _scan_top);

    }

  }

  void initialize(uint max_regions) {

    assert(_iter_states == NULL, "Must not be initialized twice");

    assert(_iter_claims == NULL, "Must not be initialized twice");

    _max_regions = max_regions;

    _iter_states = NEW_C_HEAP_ARRAY(G1RemsetIterState, max_regions, mtGC);

    _iter_claims = NEW_C_HEAP_ARRAY(size_t, max_regions, mtGC);

    _dirty_region_buffer = NEW_C_HEAP_ARRAY(uint, max_regions, mtGC);

    _in_dirty_region_buffer = NEW_C_HEAP_ARRAY(bool, max_regions, mtGC);

    _scan_top = NEW_C_HEAP_ARRAY(HeapWord*, max_regions, mtGC);

  }

  void reset() {

    for (uint i = 0; i < _max_regions; i++) {

      _iter_states[i] = Unclaimed;

      clear_scan_top(i);

    }

    G1ResetScanTopClosure cl(_scan_top);

    G1CollectedHeap::heap()->heap_region_iterate(&cl);

    memset((void*)_iter_claims, 0, _max_regions * sizeof(size_t));

    memset((void*)_in_dirty_region_buffer, false, _max_regions * sizeof(bool));

    _cur_dirty_region = 0;

  }

  // Attempt to claim the remembered set of the region for iteration. Returns true

  // if this call caused the transition from Unclaimed to Claimed.

  inline bool claim_iter(uint region) {

    assert(region < _max_regions, "Tried to access invalid region %u", region);

    if (_iter_states[region] != Unclaimed) {

      return false;

    }

    G1RemsetIterState res = Atomic::cmpxchg(Claimed, &_iter_states[region], Unclaimed);

    return (res == Unclaimed);

  }

  // Try to atomically sets the iteration state to "complete". Returns true for the

  // thread that caused the transition.

  inline bool set_iter_complete(uint region) {

    if (iter_is_complete(region)) {

      return false;

    }

    G1RemsetIterState res = Atomic::cmpxchg(Complete, &_iter_states[region], Claimed);

    return (res == Claimed);

  }

  // Returns true if the region's iteration is complete.

  inline bool iter_is_complete(uint region) const {

    assert(region < _max_regions, "Tried to access invalid region %u", region);

    return _iter_states[region] == Complete;

  }

  // The current position within the remembered set of the given region.

  inline size_t iter_claimed(uint region) const {

    assert(region < _max_regions, "Tried to access invalid region %u", region);

    return _iter_claims[region];

  }

  // Claim the next block of cards within the remembered set of the region with

  // step size.

  inline size_t iter_claimed_next(uint region, size_t step) {

    return Atomic::add(step, &_iter_claims[region]) - step;

  }

  void add_dirty_region(uint region) {

    if (_in_dirty_region_buffer[region]) {

      return;

    }

    if (!Atomic::cmpxchg(true, &_in_dirty_region_buffer[region], false)) {

      size_t allocated = Atomic::add(1u, &_cur_dirty_region) - 1;

      _dirty_region_buffer[allocated] = region;

    }

  }

  HeapWord* scan_top(uint region_idx) const {

    return _scan_top[region_idx];

  }

  void clear_scan_top(uint region_idx) {

    _scan_top[region_idx] = NULL;

  }

  // Clear the card table of "dirty" regions.

  void clear_card_table(WorkGang* workers) {

    if (_cur_dirty_region == 0) {

      return;

    }

    size_t const num_chunks = align_up(_cur_dirty_region * HeapRegion::CardsPerRegion, G1ClearCardTableTask::chunk_size()) / G1ClearCardTableTask::chunk_size();

    uint const num_workers = (uint)MIN2(num_chunks, (size_t)workers->active_workers());

    size_t const chunk_length = G1ClearCardTableTask::chunk_size() / HeapRegion::CardsPerRegion;

    // Iterate over the dirty cards region list.

    G1ClearCardTableTask cl(G1CollectedHeap::heap(), _dirty_region_buffer, _cur_dirty_region, chunk_length);

    log_debug(gc, ergo)("Running %s using %u workers for " SIZE_FORMAT " "

                        "units of work for " SIZE_FORMAT " regions.",

                        cl.name(), num_workers, num_chunks, _cur_dirty_region);

    workers->run_task(&cl, num_workers);

#ifndef PRODUCT

    G1CollectedHeap::heap()->verifier()->verify_card_table_cleanup();

#endif

  }

};

G1RemSet::G1RemSet(G1CollectedHeap* g1h,

                   G1CardTable* ct,

                   G1HotCardCache* hot_card_cache) :

  _scan_state(new G1RemSetScanState()),

  _prev_period_summary(),

  _g1h(g1h),

  _num_conc_refined_cards(0),

  _ct(ct),

  _g1p(_g1h->policy()),

  _hot_card_cache(hot_card_cache) {

}

G1RemSet::~G1RemSet() {

  if (_scan_state != NULL) {

    delete _scan_state;

  }

}

uint G1RemSet::num_par_rem_sets() {

  return G1DirtyCardQueueSet::num_par_ids() + G1ConcurrentRefine::max_num_threads() + MAX2(ConcGCThreads, ParallelGCThreads);

}

void G1RemSet::initialize(size_t capacity, uint max_regions) {

  G1FromCardCache::initialize(num_par_rem_sets(), max_regions);

  _scan_state->initialize(max_regions);

}

  }

  }

    }

    }

  }

    return false;

  }