/*

 * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "gc_implementation/g1/concurrentG1Refine.hpp"

#include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp"

#include "gc_implementation/g1/g1CollectedHeap.inline.hpp"

#include "gc_implementation/g1/heapRegionRemSet.hpp"

#include "gc_implementation/g1/heapRegionSeq.inline.hpp"

#include "memory/allocation.hpp"

#include "memory/space.inline.hpp"

#include "utilities/bitMap.inline.hpp"

#include "utilities/globalDefinitions.hpp"

// OtherRegionsTable

class PerRegionTable: public CHeapObj {

  friend class OtherRegionsTable;

  friend class HeapRegionRemSetIterator;

  HeapRegion*     _hr;

  BitMap          _bm;

  jint            _occupied;

  static PerRegionTable* _free_list;

#ifdef _MSC_VER

  // For some reason even though the classes are marked as friend they are unable

  // to access CardsPerRegion when private/protected. Only the windows c++ compiler

  // says this Sun CC and linux gcc don't have a problem with access when private

  public:

#endif // _MSC_VER

protected:

  // We need access in order to union things into the base table.

  BitMap* bm() { return &_bm; }

  void recount_occupied() {

    _occupied = (jint) bm()->count_one_bits();

  }

  PerRegionTable(HeapRegion* hr) :

    _hr(hr),

    _occupied(0),

    _bm(HeapRegion::CardsPerRegion, false /* in-resource-area */)

  {}

  void add_card_work(CardIdx_t from_card, bool par) {

    if (!_bm.at(from_card)) {

      if (par) {

        if (_bm.par_at_put(from_card, 1)) {

          Atomic::inc(&_occupied);

        }

      } else {

        _bm.at_put(from_card, 1);

        _occupied++;

      }

    }

  }

  void add_reference_work(OopOrNarrowOopStar from, bool par) {

    // Must make this robust in case "from" is not in "_hr", because of

    // concurrency.

    HeapRegion* loc_hr = hr();

    // If the test below fails, then this table was reused concurrently

    // with this operation.  This is OK, since the old table was coarsened,

    // and adding a bit to the new table is never incorrect.

    // If the table used to belong to a continues humongous region and is

    // now reused for the corresponding start humongous region, we need to

    // make sure that we detect this. Thus, we call is_in_reserved_raw()

    // instead of just is_in_reserved() here.

    if (loc_hr->is_in_reserved_raw(from)) {

      size_t hw_offset = pointer_delta((HeapWord*)from, loc_hr->bottom());

      CardIdx_t from_card = (CardIdx_t)

          hw_offset >> (CardTableModRefBS::card_shift - LogHeapWordSize);

      assert(0 <= from_card && (size_t)from_card < HeapRegion::CardsPerRegion,

             "Must be in range.");

      add_card_work(from_card, par);

    }

  }

public:

  HeapRegion* hr() const { return _hr; }

  jint occupied() const {

    // Overkill, but if we ever need it...

    // guarantee(_occupied == _bm.count_one_bits(), "Check");

    return _occupied;

  }

  void init(HeapRegion* hr) {

    _hr = hr;

    _occupied = 0;

    _bm.clear();

  }

  void add_reference(OopOrNarrowOopStar from) {

    add_reference_work(from, /*parallel*/ true);

  }

  void seq_add_reference(OopOrNarrowOopStar from) {

    add_reference_work(from, /*parallel*/ false);

  }

  void scrub(CardTableModRefBS* ctbs, BitMap* card_bm) {

    HeapWord* hr_bot = hr()->bottom();

    size_t hr_first_card_index = ctbs->index_for(hr_bot);

    bm()->set_intersection_at_offset(*card_bm, hr_first_card_index);

    recount_occupied();

  }

  void add_card(CardIdx_t from_card_index) {

    add_card_work(from_card_index, /*parallel*/ true);

  }

  void seq_add_card(CardIdx_t from_card_index) {

    add_card_work(from_card_index, /*parallel*/ false);

  }

  // (Destructively) union the bitmap of the current table into the given

  // bitmap (which is assumed to be of the same size.)

  void union_bitmap_into(BitMap* bm) {

    bm->set_union(_bm);

  }

  // Mem size in bytes.

  size_t mem_size() const {

    return sizeof(this) + _bm.size_in_words() * HeapWordSize;

  }

  // Requires "from" to be in "hr()".

  bool contains_reference(OopOrNarrowOopStar from) const {

    assert(hr()->is_in_reserved(from), "Precondition.");

    size_t card_ind = pointer_delta(from, hr()->bottom(),

                                    CardTableModRefBS::card_size);

    return _bm.at(card_ind);

  }

    while (true) {

      prt->set_next(fl);

        Atomic::cmpxchg_ptr(prt, &_free_list, fl);

      if (res == fl) return;

    }

    ShouldNotReachHere();

  }

    while (fl != NULL) {

        Atomic::cmpxchg_ptr(nxt, &_free_list, fl);

      if (res == fl) {

        fl->init(hr);

        return fl;

      } else {

        fl = _free_list;

      }

    }

    assert(fl == NULL, "Loop condition.");

  }

  static size_t fl_mem_size() {

    size_t res = 0;

    while (cur != NULL) {

      cur = cur->next();

    }

    return res;

  }

};

size_t OtherRegionsTable::_max_fine_entries = 0;

size_t OtherRegionsTable::_mod_max_fine_entries_mask = 0;

size_t OtherRegionsTable::_fine_eviction_stride = 0;

size_t OtherRegionsTable::_fine_eviction_sample_size = 0;

OtherRegionsTable::OtherRegionsTable(HeapRegion* hr) :

  _g1h(G1CollectedHeap::heap()),

  _m(Mutex::leaf, "An OtherRegionsTable lock", true),

  _hr(hr),

  _coarse_map(G1CollectedHeap::heap()->max_regions(),

              false /* in-resource-area */),

  _fine_grain_regions(NULL),

  _n_fine_entries(0), _n_coarse_entries(0),

  _fine_eviction_start(0),

  _sparse_table(hr)

{

  if (_max_fine_entries == 0) {

    assert(_mod_max_fine_entries_mask == 0, "Both or none.");

    size_t max_entries_log = (size_t)log2_long((jlong)G1RSetRegionEntries);

    _max_fine_entries = (size_t)(1 << max_entries_log);

    _mod_max_fine_entries_mask = _max_fine_entries - 1;

    assert(_fine_eviction_sample_size == 0

           && _fine_eviction_stride == 0, "All init at same time.");

    _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log);

    _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size;

  }

    vm_exit_out_of_memory(sizeof(void*)*_max_fine_entries,

                          "Failed to allocate _fine_grain_entries.");

  for (size_t i = 0; i < _max_fine_entries; i++) {

    _fine_grain_regions[i] = NULL;

  }

}

size_t OtherRegionsTable::_from_card_cache_max_regions = 0;

size_t OtherRegionsTable::_from_card_cache_mem_size = 0;

void OtherRegionsTable::init_from_card_cache(size_t max_regions) {

  _from_card_cache_max_regions = max_regions;

  int n_par_rs = HeapRegionRemSet::num_par_rem_sets();

  _from_card_cache = NEW_C_HEAP_ARRAY(int*, n_par_rs);

  for (int i = 0; i < n_par_rs; i++) {

    _from_card_cache[i] = NEW_C_HEAP_ARRAY(int, max_regions);

    for (size_t j = 0; j < max_regions; j++) {

      _from_card_cache[i][j] = -1;  // An invalid value.

    }

  }

  _from_card_cache_mem_size = n_par_rs * max_regions * sizeof(int);

}

void OtherRegionsTable::shrink_from_card_cache(size_t new_n_regs) {

  for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) {

    assert(new_n_regs <= _from_card_cache_max_regions, "Must be within max.");

    for (size_t j = new_n_regs; j < _from_card_cache_max_regions; j++) {

      _from_card_cache[i][j] = -1;  // An invalid value.

    }

  }

}

#ifndef PRODUCT

void OtherRegionsTable::print_from_card_cache() {

  for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) {

    for (size_t j = 0; j < _from_card_cache_max_regions; j++) {

      gclog_or_tty->print_cr("_from_card_cache[%d][%d] = %d.",

                    i, j, _from_card_cache[i][j]);

    }

  }

}

#endif

void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) {

  size_t cur_hrs_ind = (size_t) hr()->hrs_index();

  int from_card = (int)(uintptr_t(from) >> CardTableModRefBS::card_shift);

  if (from_card == _from_card_cache[tid][cur_hrs_ind]) {

    assert(contains_reference(from), "We just added it!");

    return;

  } else {

    _from_card_cache[tid][cur_hrs_ind] = from_card;

  }

  // Note that this may be a continued H region.

  HeapRegion* from_hr = _g1h->heap_region_containing_raw(from);

  RegionIdx_t from_hrs_ind = (RegionIdx_t) from_hr->hrs_index();

  // If the region is already coarsened, return.

  if (_coarse_map.at(from_hrs_ind)) {

    assert(contains_reference(from), "We just added it!");

    return;

  }

  // Otherwise find a per-region table to add it to.

  size_t ind = from_hrs_ind & _mod_max_fine_entries_mask;

  if (prt == NULL) {

    MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag);

    // Confirm that it's really not there...

    prt = find_region_table(ind, from_hr);

    if (prt == NULL) {

      uintptr_t from_hr_bot_card_index =

        uintptr_t(from_hr->bottom())

          >> CardTableModRefBS::card_shift;

      CardIdx_t card_index = from_card - from_hr_bot_card_index;

      assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion,

             "Must be in range.");

      if (G1HRRSUseSparseTable &&

          _sparse_table.add_card(from_hrs_ind, card_index)) {

        if (G1RecordHRRSOops) {

          HeapRegionRemSet::record(hr(), from);

        }

        assert(contains_reference_locked(from), "We just added it!");

        return;

      } else {

      }

      if (_n_fine_entries == _max_fine_entries) {

        prt = delete_region_table();

      } else {

      }

      prt->init(from_hr);

      prt->set_next(first_prt);  // XXX Maybe move to init?

      _fine_grain_regions[ind] = prt;

      _n_fine_entries++;

      if (G1HRRSUseSparseTable) {

        // Transfer from sparse to fine-grain.

        SparsePRTEntry *sprt_entry = _sparse_table.get_entry(from_hrs_ind);

        assert(sprt_entry != NULL, "There should have been an entry");

        for (int i = 0; i < SparsePRTEntry::cards_num(); i++) {

          CardIdx_t c = sprt_entry->card(i);

          if (c != SparsePRTEntry::NullEntry) {

            prt->add_card(c);

          }

        }

        // Now we can delete the sparse entry.

        bool res = _sparse_table.delete_entry(from_hrs_ind);

        assert(res, "It should have been there.");

      }

    }

    assert(prt != NULL && prt->hr() == from_hr, "consequence");

  }

  // Note that we can't assert "prt->hr() == from_hr", because of the

  // possibility of concurrent reuse.  But see head comment of

  // OtherRegionsTable for why this is OK.

  assert(prt != NULL, "Inv");

  if (G1RecordHRRSOops) {

    HeapRegionRemSet::record(hr(), from);

  }

  assert(contains_reference(from), "We just added it!");

}

OtherRegionsTable::find_region_table(size_t ind, HeapRegion* hr) const {

  assert(0 <= ind && ind < _max_fine_entries, "Preconditions.");

  while (prt != NULL && prt->hr() != hr) {

    prt = prt->next();

  }

  // Loop postcondition is the method postcondition.

  return prt;

}

jint OtherRegionsTable::_n_coarsenings = 0;

  assert(_m.owned_by_self(), "Precondition");

  assert(_n_fine_entries == _max_fine_entries, "Precondition");

  jint max_occ = 0;

  size_t max_ind;

  size_t i = _fine_eviction_start;

  for (size_t k = 0; k < _fine_eviction_sample_size; k++) {

    size_t ii = i;

    // Make sure we get a non-NULL sample.

    while (_fine_grain_regions[ii] == NULL) {

      ii++;

      if (ii == _max_fine_entries) ii = 0;

      guarantee(ii != i, "We must find one.");

    }

    while (cur != NULL) {

      jint cur_occ = cur->occupied();

      if (max == NULL || cur_occ > max_occ) {

        max = cur;

        max_prev = prev;

        max_ind = i;

        max_occ = cur_occ;

      }

      prev = cur->next_addr();

      cur = cur->next();

    }

    i = i + _fine_eviction_stride;

    if (i >= _n_fine_entries) i = i - _n_fine_entries;

  }

  _fine_eviction_start++;

    _fine_eviction_start -= _n_fine_entries;

  }

  guarantee(max != NULL, "Since _n_fine_entries > 0");

  // Set the corresponding coarse bit.

  size_t max_hrs_index = (size_t) max->hr()->hrs_index();

  if (!_coarse_map.at(max_hrs_index)) {

    _coarse_map.at_put(max_hrs_index, true);

    _n_coarse_entries++;

  }

  // Unsplice.

  *max_prev = max->next();

  Atomic::inc(&_n_coarsenings);

  _n_fine_entries--;

  return max;

}