/*

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "gc_implementation/g1/concurrentG1Refine.hpp"

#include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp"

#include "gc_implementation/g1/g1CollectedHeap.inline.hpp"

#include "gc_implementation/g1/heapRegionRemSet.hpp"

#include "gc_implementation/g1/heapRegionManager.inline.hpp"

#include "memory/allocation.hpp"

#include "memory/padded.inline.hpp"

#include "memory/space.inline.hpp"

#include "oops/oop.inline.hpp"

#include "runtime/atomic.inline.hpp"

#include "utilities/bitMap.inline.hpp"

#include "utilities/globalDefinitions.hpp"

#include "utilities/growableArray.hpp"

PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC

class PerRegionTable: public CHeapObj<mtGC> {

  friend class OtherRegionsTable;

  friend class HeapRegionRemSetIterator;

  HeapRegion*     _hr;

  BitMap          _bm;

  jint            _occupied;

  // next pointer for free/allocated 'all' list

  PerRegionTable* _next;

  // prev pointer for the allocated 'all' list

  PerRegionTable* _prev;

  // next pointer in collision list

  PerRegionTable * _collision_list_next;

  // Global free list of PRTs

  static PerRegionTable* _free_list;

protected:

  // We need access in order to union things into the base table.

  BitMap* bm() { return &_bm; }

  void recount_occupied() {

    _occupied = (jint) bm()->count_one_bits();

  }

  PerRegionTable(HeapRegion* hr) :

    _hr(hr),

    _occupied(0),

    _bm(HeapRegion::CardsPerRegion, false /* in-resource-area */),

    _collision_list_next(NULL), _next(NULL), _prev(NULL)

  {}

  void add_card_work(CardIdx_t from_card, bool par) {

    if (!_bm.at(from_card)) {

      if (par) {

        if (_bm.par_at_put(from_card, 1)) {

          Atomic::inc(&_occupied);

        }

      } else {

        _bm.at_put(from_card, 1);

        _occupied++;

      }

    }

  }

  void add_reference_work(OopOrNarrowOopStar from, bool par) {

    // Must make this robust in case "from" is not in "_hr", because of

    // concurrency.

    if (G1TraceHeapRegionRememberedSet) {

      gclog_or_tty->print_cr("    PRT::Add_reference_work(" PTR_FORMAT "->" PTR_FORMAT").",

                             from,

                             UseCompressedOops

                             ? (void *)oopDesc::load_decode_heap_oop((narrowOop*)from)

                             : (void *)oopDesc::load_decode_heap_oop((oop*)from));

    }

    HeapRegion* loc_hr = hr();

    // If the test below fails, then this table was reused concurrently

    // with this operation.  This is OK, since the old table was coarsened,

    // and adding a bit to the new table is never incorrect.

    // If the table used to belong to a continues humongous region and is

    // now reused for the corresponding start humongous region, we need to

    // make sure that we detect this. Thus, we call is_in_reserved_raw()

    // instead of just is_in_reserved() here.

    if (loc_hr->is_in_reserved_raw(from)) {

      size_t hw_offset = pointer_delta((HeapWord*)from, loc_hr->bottom());

      CardIdx_t from_card = (CardIdx_t)

          hw_offset >> (CardTableModRefBS::card_shift - LogHeapWordSize);

      assert(0 <= from_card && (size_t)from_card < HeapRegion::CardsPerRegion,

             "Must be in range.");

      add_card_work(from_card, par);

    }

  }

public:

  HeapRegion* hr() const { return _hr; }

  jint occupied() const {

    // Overkill, but if we ever need it...

    // guarantee(_occupied == _bm.count_one_bits(), "Check");

    return _occupied;

  }

  void init(HeapRegion* hr, bool clear_links_to_all_list) {

    if (clear_links_to_all_list) {

      set_next(NULL);

      set_prev(NULL);

    }

    _hr = hr;

    _collision_list_next = NULL;

    _occupied = 0;

    _bm.clear();

  }

  void add_reference(OopOrNarrowOopStar from) {

    add_reference_work(from, /*parallel*/ true);

  }

  void seq_add_reference(OopOrNarrowOopStar from) {

    add_reference_work(from, /*parallel*/ false);

  }

  void scrub(CardTableModRefBS* ctbs, BitMap* card_bm) {

    HeapWord* hr_bot = hr()->bottom();

    size_t hr_first_card_index = ctbs->index_for(hr_bot);

    bm()->set_intersection_at_offset(*card_bm, hr_first_card_index);

    recount_occupied();

  }

  void add_card(CardIdx_t from_card_index) {

    add_card_work(from_card_index, /*parallel*/ true);

  }

  void seq_add_card(CardIdx_t from_card_index) {

    add_card_work(from_card_index, /*parallel*/ false);

  }

  // (Destructively) union the bitmap of the current table into the given

  // bitmap (which is assumed to be of the same size.)

  void union_bitmap_into(BitMap* bm) {

    bm->set_union(_bm);

  }

  // Mem size in bytes.

  size_t mem_size() const {

    return sizeof(PerRegionTable) + _bm.size_in_words() * HeapWordSize;

  }

  // Requires "from" to be in "hr()".

  bool contains_reference(OopOrNarrowOopStar from) const {

    assert(hr()->is_in_reserved(from), "Precondition.");

    size_t card_ind = pointer_delta(from, hr()->bottom(),

                                    CardTableModRefBS::card_size);

    return _bm.at(card_ind);

  }

  // Bulk-free the PRTs from prt to last, assumes that they are

  // linked together using their _next field.

  static void bulk_free(PerRegionTable* prt, PerRegionTable* last) {

    while (true) {

      PerRegionTable* fl = _free_list;

      last->set_next(fl);

      PerRegionTable* res = (PerRegionTable*) Atomic::cmpxchg_ptr(prt, &_free_list, fl);

      if (res == fl) {

        return;

      }

    }

    ShouldNotReachHere();

  }

  static void free(PerRegionTable* prt) {

    bulk_free(prt, prt);

  }

  // Returns an initialized PerRegionTable instance.

  static PerRegionTable* alloc(HeapRegion* hr) {

    PerRegionTable* fl = _free_list;

    while (fl != NULL) {

      PerRegionTable* nxt = fl->next();

      PerRegionTable* res =

        (PerRegionTable*)

        Atomic::cmpxchg_ptr(nxt, &_free_list, fl);

      if (res == fl) {

        fl->init(hr, true);

        return fl;

      } else {

        fl = _free_list;

      }

    }

    assert(fl == NULL, "Loop condition.");

    return new PerRegionTable(hr);

  }

  PerRegionTable* next() const { return _next; }

  void set_next(PerRegionTable* next) { _next = next; }

  PerRegionTable* prev() const { return _prev; }

  void set_prev(PerRegionTable* prev) { _prev = prev; }

  // Accessor and Modification routines for the pointer for the

  // singly linked collision list that links the PRTs within the

  // OtherRegionsTable::_fine_grain_regions hash table.

  //

  // It might be useful to also make the collision list doubly linked

  // to avoid iteration over the collisions list during scrubbing/deletion.

  // OTOH there might not be many collisions.

  PerRegionTable* collision_list_next() const {

    return _collision_list_next;

  }

  void set_collision_list_next(PerRegionTable* next) {

    _collision_list_next = next;

  }

  PerRegionTable** collision_list_next_addr() {

    return &_collision_list_next;

  }

  static size_t fl_mem_size() {

    PerRegionTable* cur = _free_list;

    size_t res = 0;

    while (cur != NULL) {

      res += cur->mem_size();

      cur = cur->next();

    }

    return res;

  }

  static void test_fl_mem_size();

};

PerRegionTable* PerRegionTable::_free_list = NULL;

size_t OtherRegionsTable::_max_fine_entries = 0;

size_t OtherRegionsTable::_mod_max_fine_entries_mask = 0;

size_t OtherRegionsTable::_fine_eviction_stride = 0;

size_t OtherRegionsTable::_fine_eviction_sample_size = 0;

OtherRegionsTable::OtherRegionsTable(HeapRegion* hr, Mutex* m) :

  _g1h(G1CollectedHeap::heap()),

  _hr(hr), _m(m),

  _coarse_map(G1CollectedHeap::heap()->max_regions(),

              false /* in-resource-area */),

  _fine_grain_regions(NULL),

  _first_all_fine_prts(NULL), _last_all_fine_prts(NULL),

  _n_fine_entries(0), _n_coarse_entries(0),

  _fine_eviction_start(0),

  _sparse_table(hr)

{

  typedef PerRegionTable* PerRegionTablePtr;

  if (_max_fine_entries == 0) {

    assert(_mod_max_fine_entries_mask == 0, "Both or none.");

    size_t max_entries_log = (size_t)log2_long((jlong)G1RSetRegionEntries);

    _max_fine_entries = (size_t)1 << max_entries_log;

    _mod_max_fine_entries_mask = _max_fine_entries - 1;

    assert(_fine_eviction_sample_size == 0

           && _fine_eviction_stride == 0, "All init at same time.");

    _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log);

    _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size;

  }

  _fine_grain_regions = NEW_C_HEAP_ARRAY3(PerRegionTablePtr, _max_fine_entries,

                        mtGC, CURRENT_PC, AllocFailStrategy::RETURN_NULL);

  if (_fine_grain_regions == NULL) {

    vm_exit_out_of_memory(sizeof(void*)*_max_fine_entries, OOM_MALLOC_ERROR,

                          "Failed to allocate _fine_grain_entries.");

  }

  for (size_t i = 0; i < _max_fine_entries; i++) {

    _fine_grain_regions[i] = NULL;

  }

}

void OtherRegionsTable::link_to_all(PerRegionTable* prt) {

  // We always append to the beginning of the list for convenience;

  // the order of entries in this list does not matter.

  if (_first_all_fine_prts != NULL) {

    assert(_first_all_fine_prts->prev() == NULL, "invariant");

    _first_all_fine_prts->set_prev(prt);

    prt->set_next(_first_all_fine_prts);

  } else {

    // this is the first element we insert. Adjust the "last" pointer

    _last_all_fine_prts = prt;

    assert(prt->next() == NULL, "just checking");

  }

  // the new element is always the first element without a predecessor

  prt->set_prev(NULL);

  _first_all_fine_prts = prt;

  assert(prt->prev() == NULL, "just checking");

  assert(_first_all_fine_prts == prt, "just checking");

  assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) ||

         (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL),

         "just checking");

  assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL,

         "just checking");

  assert(_first_all_fine_prts == NULL || _first_all_fine_prts->prev() == NULL,

         "just checking");

}

void OtherRegionsTable::unlink_from_all(PerRegionTable* prt) {

  if (prt->prev() != NULL) {

    assert(_first_all_fine_prts != prt, "just checking");

    prt->prev()->set_next(prt->next());

    // removing the last element in the list?

    if (_last_all_fine_prts == prt) {

      _last_all_fine_prts = prt->prev();

    }

  } else {

    assert(_first_all_fine_prts == prt, "just checking");

    _first_all_fine_prts = prt->next();

    // list is empty now?

    if (_first_all_fine_prts == NULL) {

      _last_all_fine_prts = NULL;

    }

  }

  if (prt->next() != NULL) {

    prt->next()->set_prev(prt->prev());

  }

  prt->set_next(NULL);

  prt->set_prev(NULL);

  assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) ||

         (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL),

         "just checking");

  assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL,

         "just checking");

  assert(_first_all_fine_prts == NULL || _first_all_fine_prts->prev() == NULL,

         "just checking");

}

int**  FromCardCache::_cache = NULL;

uint   FromCardCache::_max_regions = 0;

size_t FromCardCache::_static_mem_size = 0;

void FromCardCache::initialize(uint n_par_rs, uint max_num_regions) {

  guarantee(_cache == NULL, "Should not call this multiple times");

  _max_regions = max_num_regions;

  _cache = Padded2DArray<int, mtGC>::create_unfreeable(n_par_rs,

                                                       _max_regions,

                                                       &_static_mem_size);

  invalidate(0, _max_regions);

}

void FromCardCache::invalidate(uint start_idx, size_t new_num_regions) {

  guarantee((size_t)start_idx + new_num_regions <= max_uintx,

            err_msg("Trying to invalidate beyond maximum region, from %u size "SIZE_FORMAT,

                    start_idx, new_num_regions));

  for (uint i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) {

    uint end_idx = (start_idx + (uint)new_num_regions);

    assert(end_idx <= _max_regions, "Must be within max.");

    for (uint j = start_idx; j < end_idx; j++) {

      set(i, j, InvalidCard);

    }

  }

}

#ifndef PRODUCT

void FromCardCache::print(outputStream* out) {

  for (uint i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) {

    for (uint j = 0; j < _max_regions; j++) {

      out->print_cr("_from_card_cache[%u][%u] = %d.",

                    i, j, at(i, j));

    }

  }

}

#endif

void FromCardCache::clear(uint region_idx) {

  uint num_par_remsets = HeapRegionRemSet::num_par_rem_sets();

  for (uint i = 0; i < num_par_remsets; i++) {

    set(i, region_idx, InvalidCard);

  }

}

void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, uint tid) {

  uint cur_hrm_ind = _hr->hrm_index();

  if (G1TraceHeapRegionRememberedSet) {

    gclog_or_tty->print_cr("ORT::add_reference_work(" PTR_FORMAT "->" PTR_FORMAT ").",

                                                    from,

                                                    UseCompressedOops

                                                    ? (void *)oopDesc::load_decode_heap_oop((narrowOop*)from)

                                                    : (void *)oopDesc::load_decode_heap_oop((oop*)from));

  }

  int from_card = (int)(uintptr_t(from) >> CardTableModRefBS::card_shift);

  if (G1TraceHeapRegionRememberedSet) {

    gclog_or_tty->print_cr("Table for [" PTR_FORMAT "...): card %d (cache = %d)",

                  _hr->bottom(), from_card,

                  FromCardCache::at(tid, cur_hrm_ind));

  }

  if (FromCardCache::contains_or_replace(tid, cur_hrm_ind, from_card)) {

    if (G1TraceHeapRegionRememberedSet) {

      gclog_or_tty->print_cr("  from-card cache hit.");

    }

    assert(contains_reference(from), "We just added it!");

    return;

  }

  // Note that this may be a continued H region.

  HeapRegion* from_hr = _g1h->heap_region_containing_raw(from);

  RegionIdx_t from_hrm_ind = (RegionIdx_t) from_hr->hrm_index();

  // If the region is already coarsened, return.

  if (_coarse_map.at(from_hrm_ind)) {

    if (G1TraceHeapRegionRememberedSet) {

      gclog_or_tty->print_cr("  coarse map hit.");

    }

    assert(contains_reference(from), "We just added it!");

    return;

  }

  // Otherwise find a per-region table to add it to.

  size_t ind = from_hrm_ind & _mod_max_fine_entries_mask;

  PerRegionTable* prt = find_region_table(ind, from_hr);

  if (prt == NULL) {

    MutexLockerEx x(_m, Mutex::_no_safepoint_check_flag);

    // Confirm that it's really not there...

    prt = find_region_table(ind, from_hr);

    if (prt == NULL) {

      uintptr_t from_hr_bot_card_index =

        uintptr_t(from_hr->bottom())

          >> CardTableModRefBS::card_shift;

      CardIdx_t card_index = from_card - from_hr_bot_card_index;

      assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion,

             "Must be in range.");

      if (G1HRRSUseSparseTable &&

          _sparse_table.add_card(from_hrm_ind, card_index)) {

        if (G1RecordHRRSOops) {

          HeapRegionRemSet::record(_hr, from);

          if (G1TraceHeapRegionRememberedSet) {

            gclog_or_tty->print("   Added card " PTR_FORMAT " to region "

                                "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n",

                                align_size_down(uintptr_t(from),

                                                CardTableModRefBS::card_size),

                                _hr->bottom(), from);

          }

        }

        if (G1TraceHeapRegionRememberedSet) {

          gclog_or_tty->print_cr("   added card to sparse table.");

        }

        assert(contains_reference_locked(from), "We just added it!");

        return;

      } else {

        if (G1TraceHeapRegionRememberedSet) {

          gclog_or_tty->print_cr("   [tid %u] sparse table entry "

                        "overflow(f: %d, t: %u)",

                        tid, from_hrm_ind, cur_hrm_ind);

        }

      }

      if (_n_fine_entries == _max_fine_entries) {

        prt = delete_region_table();

        // There is no need to clear the links to the 'all' list here:

        // prt will be reused immediately, i.e. remain in the 'all' list.

        prt->init(from_hr, false /* clear_links_to_all_list */);

      } else {

        prt = PerRegionTable::alloc(from_hr);

        link_to_all(prt);

      }