/*

 * Copyright (c) 2001, 2016, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "gc/g1/concurrentG1Refine.hpp"

#include "gc/g1/g1BlockOffsetTable.inline.hpp"

#include "gc/g1/g1CollectedHeap.inline.hpp"

#include "gc/g1/g1CardLiveData.inline.hpp"

#include "gc/g1/heapRegionManager.inline.hpp"

#include "gc/g1/heapRegionRemSet.hpp"

#include "gc/shared/space.inline.hpp"

#include "memory/allocation.hpp"

#include "memory/padded.inline.hpp"

#include "oops/oop.inline.hpp"

#include "runtime/atomic.inline.hpp"

#include "utilities/bitMap.inline.hpp"

#include "utilities/globalDefinitions.hpp"

#include "utilities/growableArray.hpp"

class PerRegionTable: public CHeapObj<mtGC> {

  friend class OtherRegionsTable;

  friend class HeapRegionRemSetIterator;

  HeapRegion*     _hr;

  BitMap          _bm;

  jint            _occupied;

  // next pointer for free/allocated 'all' list

  PerRegionTable* _next;

  // prev pointer for the allocated 'all' list

  PerRegionTable* _prev;

  // next pointer in collision list

  PerRegionTable * _collision_list_next;

  // Global free list of PRTs

  static PerRegionTable* _free_list;

protected:

  // We need access in order to union things into the base table.

  BitMap* bm() { return &_bm; }

  void recount_occupied() {

    _occupied = (jint) bm()->count_one_bits();

  }

  PerRegionTable(HeapRegion* hr) :

    _hr(hr),

    _occupied(0),

    _bm(HeapRegion::CardsPerRegion, false /* in-resource-area */),

    _collision_list_next(NULL), _next(NULL), _prev(NULL)

  {}

  void add_card_work(CardIdx_t from_card, bool par) {

    if (!_bm.at(from_card)) {

      if (par) {

        if (_bm.par_at_put(from_card, 1)) {

          Atomic::inc(&_occupied);

        }

      } else {

        _bm.at_put(from_card, 1);

        _occupied++;

      }

    }

  }

  void add_reference_work(OopOrNarrowOopStar from, bool par) {

    // Must make this robust in case "from" is not in "_hr", because of

    // concurrency.

    HeapRegion* loc_hr = hr();

    // If the test below fails, then this table was reused concurrently

    // with this operation.  This is OK, since the old table was coarsened,

    // and adding a bit to the new table is never incorrect.

    // If the table used to belong to a continues humongous region and is

    // now reused for the corresponding start humongous region, we need to

    // make sure that we detect this. Thus, we call is_in_reserved_raw()

    // instead of just is_in_reserved() here.

    if (loc_hr->is_in_reserved(from)) {

      size_t hw_offset = pointer_delta((HeapWord*)from, loc_hr->bottom());

      CardIdx_t from_card = (CardIdx_t)

          hw_offset >> (CardTableModRefBS::card_shift - LogHeapWordSize);

      assert(0 <= from_card && (size_t)from_card < HeapRegion::CardsPerRegion,

             "Must be in range.");

      add_card_work(from_card, par);

    }

  }

public:

  HeapRegion* hr() const {

    return (HeapRegion*) OrderAccess::load_ptr_acquire(&_hr);

  }

  jint occupied() const {

    // Overkill, but if we ever need it...

    // guarantee(_occupied == _bm.count_one_bits(), "Check");

    return _occupied;

  }

  void init(HeapRegion* hr, bool clear_links_to_all_list) {

    if (clear_links_to_all_list) {

      set_next(NULL);

      set_prev(NULL);

    }

    _collision_list_next = NULL;

    _occupied = 0;

    _bm.clear();

    // Make sure that the bitmap clearing above has been finished before publishing

    // this PRT to concurrent threads.

    OrderAccess::release_store_ptr(&_hr, hr);

  }

  void add_reference(OopOrNarrowOopStar from) {

    add_reference_work(from, /*parallel*/ true);

  }

  void seq_add_reference(OopOrNarrowOopStar from) {

    add_reference_work(from, /*parallel*/ false);

  }

  void scrub(G1CardLiveData* live_data) {

    live_data->remove_nonlive_cards(hr()->hrm_index(), &_bm);

    recount_occupied();

  }

  void add_card(CardIdx_t from_card_index) {

    add_card_work(from_card_index, /*parallel*/ true);

  }

  void seq_add_card(CardIdx_t from_card_index) {

    add_card_work(from_card_index, /*parallel*/ false);

  }

  // (Destructively) union the bitmap of the current table into the given

  // bitmap (which is assumed to be of the same size.)

  void union_bitmap_into(BitMap* bm) {

    bm->set_union(_bm);

  }

  // Mem size in bytes.

  size_t mem_size() const {

    return sizeof(PerRegionTable) + _bm.size_in_words() * HeapWordSize;

  }

  // Requires "from" to be in "hr()".

  bool contains_reference(OopOrNarrowOopStar from) const {

    assert(hr()->is_in_reserved(from), "Precondition.");

    size_t card_ind = pointer_delta(from, hr()->bottom(),

                                    CardTableModRefBS::card_size);

    return _bm.at(card_ind);

  }

  // Bulk-free the PRTs from prt to last, assumes that they are

  // linked together using their _next field.

  static void bulk_free(PerRegionTable* prt, PerRegionTable* last) {

    while (true) {

      PerRegionTable* fl = _free_list;

      last->set_next(fl);

      PerRegionTable* res = (PerRegionTable*) Atomic::cmpxchg_ptr(prt, &_free_list, fl);

      if (res == fl) {

        return;

      }

    }

    ShouldNotReachHere();

  }

  static void free(PerRegionTable* prt) {

    bulk_free(prt, prt);

  }

  // Returns an initialized PerRegionTable instance.

  static PerRegionTable* alloc(HeapRegion* hr) {

    PerRegionTable* fl = _free_list;

    while (fl != NULL) {

      PerRegionTable* nxt = fl->next();

      PerRegionTable* res =

        (PerRegionTable*)

        Atomic::cmpxchg_ptr(nxt, &_free_list, fl);

      if (res == fl) {

        fl->init(hr, true);

        return fl;

      } else {

        fl = _free_list;

      }

    }

    assert(fl == NULL, "Loop condition.");

    return new PerRegionTable(hr);

  }

  PerRegionTable* next() const { return _next; }

  void set_next(PerRegionTable* next) { _next = next; }

  PerRegionTable* prev() const { return _prev; }

  void set_prev(PerRegionTable* prev) { _prev = prev; }

  // Accessor and Modification routines for the pointer for the

  // singly linked collision list that links the PRTs within the

  // OtherRegionsTable::_fine_grain_regions hash table.

  //

  // It might be useful to also make the collision list doubly linked

  // to avoid iteration over the collisions list during scrubbing/deletion.

  // OTOH there might not be many collisions.

  PerRegionTable* collision_list_next() const {

    return _collision_list_next;

  }

  void set_collision_list_next(PerRegionTable* next) {

    _collision_list_next = next;

  }

  PerRegionTable** collision_list_next_addr() {

    return &_collision_list_next;

  }

  static size_t fl_mem_size() {

    PerRegionTable* cur = _free_list;

    size_t res = 0;

    while (cur != NULL) {

      res += cur->mem_size();

      cur = cur->next();

    }

    return res;

  }

  static void test_fl_mem_size();

};

PerRegionTable* PerRegionTable::_free_list = NULL;

size_t OtherRegionsTable::_max_fine_entries = 0;

size_t OtherRegionsTable::_mod_max_fine_entries_mask = 0;

size_t OtherRegionsTable::_fine_eviction_stride = 0;

size_t OtherRegionsTable::_fine_eviction_sample_size = 0;

OtherRegionsTable::OtherRegionsTable(HeapRegion* hr, Mutex* m) :

  _g1h(G1CollectedHeap::heap()),

  _hr(hr), _m(m),

  _coarse_map(G1CollectedHeap::heap()->max_regions(),

              false /* in-resource-area */),

  _fine_grain_regions(NULL),

  _first_all_fine_prts(NULL), _last_all_fine_prts(NULL),

  _n_fine_entries(0), _n_coarse_entries(0),

  _fine_eviction_start(0),

{

  typedef PerRegionTable* PerRegionTablePtr;

  if (_max_fine_entries == 0) {

    assert(_mod_max_fine_entries_mask == 0, "Both or none.");

    size_t max_entries_log = (size_t)log2_long((jlong)G1RSetRegionEntries);

    _max_fine_entries = (size_t)1 << max_entries_log;

    _mod_max_fine_entries_mask = _max_fine_entries - 1;

    assert(_fine_eviction_sample_size == 0

           && _fine_eviction_stride == 0, "All init at same time.");

    _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log);

    _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size;

  }

  _fine_grain_regions = NEW_C_HEAP_ARRAY3(PerRegionTablePtr, _max_fine_entries,

                        mtGC, CURRENT_PC, AllocFailStrategy::RETURN_NULL);

  if (_fine_grain_regions == NULL) {

    vm_exit_out_of_memory(sizeof(void*)*_max_fine_entries, OOM_MALLOC_ERROR,

                          "Failed to allocate _fine_grain_entries.");

  }

  for (size_t i = 0; i < _max_fine_entries; i++) {

    _fine_grain_regions[i] = NULL;

  }

}

void OtherRegionsTable::link_to_all(PerRegionTable* prt) {

  // We always append to the beginning of the list for convenience;

  // the order of entries in this list does not matter.

  if (_first_all_fine_prts != NULL) {

    assert(_first_all_fine_prts->prev() == NULL, "invariant");

    _first_all_fine_prts->set_prev(prt);

    prt->set_next(_first_all_fine_prts);

  } else {

    // this is the first element we insert. Adjust the "last" pointer

    _last_all_fine_prts = prt;

    assert(prt->next() == NULL, "just checking");

  }

  // the new element is always the first element without a predecessor

  prt->set_prev(NULL);

  _first_all_fine_prts = prt;

  assert(prt->prev() == NULL, "just checking");

  assert(_first_all_fine_prts == prt, "just checking");

  assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) ||

         (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL),

         "just checking");

  assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL,

         "just checking");

  assert(_first_all_fine_prts == NULL || _first_all_fine_prts->prev() == NULL,

         "just checking");

}

void OtherRegionsTable::unlink_from_all(PerRegionTable* prt) {

  if (prt->prev() != NULL) {

    assert(_first_all_fine_prts != prt, "just checking");

    prt->prev()->set_next(prt->next());

    // removing the last element in the list?

    if (_last_all_fine_prts == prt) {

      _last_all_fine_prts = prt->prev();

    }

  } else {

    assert(_first_all_fine_prts == prt, "just checking");

    _first_all_fine_prts = prt->next();

    // list is empty now?

    if (_first_all_fine_prts == NULL) {

      _last_all_fine_prts = NULL;

    }

  }

  if (prt->next() != NULL) {

    prt->next()->set_prev(prt->prev());

  }

  prt->set_next(NULL);

  prt->set_prev(NULL);

  assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) ||

         (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL),

         "just checking");

  assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL,

         "just checking");

  assert(_first_all_fine_prts == NULL || _first_all_fine_prts->prev() == NULL,

         "just checking");

}

void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, uint tid) {

  uint cur_hrm_ind = _hr->hrm_index();

  int from_card = (int)(uintptr_t(from) >> CardTableModRefBS::card_shift);

  if (G1FromCardCache::contains_or_replace(tid, cur_hrm_ind, from_card)) {

    assert(contains_reference(from), "We just found " PTR_FORMAT " in the FromCardCache", p2i(from));

    return;

  }

  // Note that this may be a continued H region.

  HeapRegion* from_hr = _g1h->heap_region_containing(from);

  RegionIdx_t from_hrm_ind = (RegionIdx_t) from_hr->hrm_index();

  // If the region is already coarsened, return.

  if (_coarse_map.at(from_hrm_ind)) {

    assert(contains_reference(from), "We just found " PTR_FORMAT " in the Coarse table", p2i(from));

    return;

  }

  // Otherwise find a per-region table to add it to.

  size_t ind = from_hrm_ind & _mod_max_fine_entries_mask;

  PerRegionTable* prt = find_region_table(ind, from_hr);

  if (prt == NULL) {

    MutexLockerEx x(_m, Mutex::_no_safepoint_check_flag);

    // Confirm that it's really not there...

    prt = find_region_table(ind, from_hr);

    if (prt == NULL) {

      uintptr_t from_hr_bot_card_index =

        uintptr_t(from_hr->bottom())

          >> CardTableModRefBS::card_shift;

      CardIdx_t card_index = from_card - from_hr_bot_card_index;

      assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion,

             "Must be in range.");

      if (G1HRRSUseSparseTable &&

          _sparse_table.add_card(from_hrm_ind, card_index)) {

        assert(contains_reference_locked(from), "We just added " PTR_FORMAT " to the Sparse table", p2i(from));

        return;

      }

      if (_n_fine_entries == _max_fine_entries) {

        prt = delete_region_table();

        // There is no need to clear the links to the 'all' list here:

        // prt will be reused immediately, i.e. remain in the 'all' list.

        prt->init(from_hr, false /* clear_links_to_all_list */);

      } else {

        prt = PerRegionTable::alloc(from_hr);

        link_to_all(prt);

      }

      PerRegionTable* first_prt = _fine_grain_regions[ind];

      prt->set_collision_list_next(first_prt);

      // The assignment into _fine_grain_regions allows the prt to

      // start being used concurrently. In addition to

      // collision_list_next which must be visible (else concurrent

      // parsing of the list, if any, may fail to see other entries),

      // the content of the prt must be visible (else for instance

      // some mark bits may not yet seem cleared or a 'later' update

      // performed by a concurrent thread could be undone when the

      // zeroing becomes visible). This requires store ordering.

      OrderAccess::release_store_ptr((volatile PerRegionTable*)&_fine_grain_regions[ind], prt);

      _n_fine_entries++;

      if (G1HRRSUseSparseTable) {

        // Transfer from sparse to fine-grain.

        SparsePRTEntry *sprt_entry = _sparse_table.get_entry(from_hrm_ind);

        assert(sprt_entry != NULL, "There should have been an entry");

        for (int i = 0; i < SparsePRTEntry::cards_num(); i++) {

          CardIdx_t c = sprt_entry->card(i);

          if (c != SparsePRTEntry::NullEntry) {

            prt->add_card(c);

          }

        }

        // Now we can delete the sparse entry.

        bool res = _sparse_table.delete_entry(from_hrm_ind);

        assert(res, "It should have been there.");

      }

    }

    assert(prt != NULL && prt->hr() == from_hr, "consequence");

  }

  // Note that we can't assert "prt->hr() == from_hr", because of the

  // possibility of concurrent reuse.  But see head comment of

  // OtherRegionsTable for why this is OK.

  assert(prt != NULL, "Inv");

  prt->add_reference(from);

  assert(contains_reference(from), "We just added " PTR_FORMAT " to the PRT", p2i(from));

}

PerRegionTable*

OtherRegionsTable::find_region_table(size_t ind, HeapRegion* hr) const {

  assert(ind < _max_fine_entries, "Preconditions.");

  PerRegionTable* prt = _fine_grain_regions[ind];

  while (prt != NULL && prt->hr() != hr) {

    prt = prt->collision_list_next();

  }

  // Loop postcondition is the method postcondition.

  return prt;

}

jint OtherRegionsTable::_n_coarsenings = 0;

PerRegionTable* OtherRegionsTable::delete_region_table() {

  assert(_m->owned_by_self(), "Precondition");

  assert(_n_fine_entries == _max_fine_entries, "Precondition");

  PerRegionTable* max = NULL;

  jint max_occ = 0;

  PerRegionTable** max_prev = NULL;

  size_t max_ind;

  size_t i = _fine_eviction_start;

  for (size_t k = 0; k < _fine_eviction_sample_size; k++) {

    size_t ii = i;

    // Make sure we get a non-NULL sample.

    while (_fine_grain_regions[ii] == NULL) {

      ii++;

      if (ii == _max_fine_entries) ii = 0;

      guarantee(ii != i, "We must find one.");

    }

    PerRegionTable** prev = &_fine_grain_regions[ii];

    PerRegionTable* cur = *prev;

    while (cur != NULL) {

      jint cur_occ = cur->occupied();

      if (max == NULL || cur_occ > max_occ) {

        max = cur;

        max_prev = prev;

        max_ind = i;

        max_occ = cur_occ;

      }

      prev = cur->collision_list_next_addr();

      cur = cur->collision_list_next();

    }

    i = i + _fine_eviction_stride;

    if (i >= _n_fine_entries) i = i - _n_fine_entries;

  }

  _fine_eviction_start++;

  if (_fine_eviction_start >= _n_fine_entries) {

    _fine_eviction_start -= _n_fine_entries;

  }