/*

 * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "gc_implementation/g1/concurrentG1Refine.hpp"

#include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp"

#include "gc_implementation/g1/g1CollectedHeap.inline.hpp"

#include "gc_implementation/g1/heapRegionRemSet.hpp"

#include "gc_implementation/g1/heapRegionSeq.inline.hpp"

#include "memory/allocation.hpp"

#include "memory/space.inline.hpp"

#include "oops/oop.inline.hpp"

#include "utilities/bitMap.inline.hpp"

#include "utilities/globalDefinitions.hpp"

class PerRegionTable: public CHeapObj<mtGC> {

  friend class OtherRegionsTable;

  friend class HeapRegionRemSetIterator;

  HeapRegion*     _hr;

  BitMap          _bm;

  jint            _occupied;

  PerRegionTable* _next;

protected:

  // We need access in order to union things into the base table.

  BitMap* bm() { return &_bm; }

  void recount_occupied() {

    _occupied = (jint) bm()->count_one_bits();

  }

  PerRegionTable(HeapRegion* hr) :

    _hr(hr),

    _occupied(0),

  {}

  void add_card_work(CardIdx_t from_card, bool par) {

    if (!_bm.at(from_card)) {

      if (par) {

        if (_bm.par_at_put(from_card, 1)) {

          Atomic::inc(&_occupied);

        }

      } else {

        _bm.at_put(from_card, 1);

        _occupied++;

      }

    }

  }

  void add_reference_work(OopOrNarrowOopStar from, bool par) {

    // Must make this robust in case "from" is not in "_hr", because of

    // concurrency.

    if (G1TraceHeapRegionRememberedSet) {

      gclog_or_tty->print_cr("    PRT::Add_reference_work(" PTR_FORMAT "->" PTR_FORMAT").",

                             from,

                             UseCompressedOops

                             ? oopDesc::load_decode_heap_oop((narrowOop*)from)

                             : oopDesc::load_decode_heap_oop((oop*)from));

    }

    HeapRegion* loc_hr = hr();

    // If the test below fails, then this table was reused concurrently

    // with this operation.  This is OK, since the old table was coarsened,

    // and adding a bit to the new table is never incorrect.

    // If the table used to belong to a continues humongous region and is

    // now reused for the corresponding start humongous region, we need to

    // make sure that we detect this. Thus, we call is_in_reserved_raw()

    // instead of just is_in_reserved() here.

    if (loc_hr->is_in_reserved_raw(from)) {

      size_t hw_offset = pointer_delta((HeapWord*)from, loc_hr->bottom());

      CardIdx_t from_card = (CardIdx_t)

          hw_offset >> (CardTableModRefBS::card_shift - LogHeapWordSize);

      assert(0 <= from_card && (size_t)from_card < HeapRegion::CardsPerRegion,

             "Must be in range.");

      add_card_work(from_card, par);

    }

  }

public:

  HeapRegion* hr() const { return _hr; }

  jint occupied() const {

    // Overkill, but if we ever need it...

    // guarantee(_occupied == _bm.count_one_bits(), "Check");

    return _occupied;

  }

    _hr = hr;

    _occupied = 0;

    _bm.clear();

  }

  void add_reference(OopOrNarrowOopStar from) {

    add_reference_work(from, /*parallel*/ true);

  }

  void seq_add_reference(OopOrNarrowOopStar from) {

    add_reference_work(from, /*parallel*/ false);

  }

  void scrub(CardTableModRefBS* ctbs, BitMap* card_bm) {

    HeapWord* hr_bot = hr()->bottom();

    size_t hr_first_card_index = ctbs->index_for(hr_bot);

    bm()->set_intersection_at_offset(*card_bm, hr_first_card_index);

    recount_occupied();

  }

  void add_card(CardIdx_t from_card_index) {

    add_card_work(from_card_index, /*parallel*/ true);

  }

  void seq_add_card(CardIdx_t from_card_index) {

    add_card_work(from_card_index, /*parallel*/ false);

  }

  // (Destructively) union the bitmap of the current table into the given

  // bitmap (which is assumed to be of the same size.)

  void union_bitmap_into(BitMap* bm) {

    bm->set_union(_bm);

  }

  // Mem size in bytes.

  size_t mem_size() const {

    return sizeof(this) + _bm.size_in_words() * HeapWordSize;

  }

  // Requires "from" to be in "hr()".

  bool contains_reference(OopOrNarrowOopStar from) const {

    assert(hr()->is_in_reserved(from), "Precondition.");

    size_t card_ind = pointer_delta(from, hr()->bottom(),

                                    CardTableModRefBS::card_size);

    return _bm.at(card_ind);

  }

    while (true) {

      PerRegionTable* fl = _free_list;

    }

    ShouldNotReachHere();

  }

  static PerRegionTable* alloc(HeapRegion* hr) {

    PerRegionTable* fl = _free_list;

    while (fl != NULL) {

      PerRegionTable* nxt = fl->next();

      PerRegionTable* res =

        (PerRegionTable*)

        Atomic::cmpxchg_ptr(nxt, &_free_list, fl);

      if (res == fl) {

        return fl;

      } else {

        fl = _free_list;

      }

    }

    assert(fl == NULL, "Loop condition.");

    return new PerRegionTable(hr);

  }

  static size_t fl_mem_size() {

    PerRegionTable* cur = _free_list;

    size_t res = 0;

    while (cur != NULL) {

      res += sizeof(PerRegionTable);

      cur = cur->next();

    }

    return res;

  }

};

PerRegionTable* PerRegionTable::_free_list = NULL;

size_t OtherRegionsTable::_max_fine_entries = 0;

size_t OtherRegionsTable::_mod_max_fine_entries_mask = 0;

size_t OtherRegionsTable::_fine_eviction_stride = 0;

size_t OtherRegionsTable::_fine_eviction_sample_size = 0;

OtherRegionsTable::OtherRegionsTable(HeapRegion* hr) :

  _g1h(G1CollectedHeap::heap()),

  _m(Mutex::leaf, "An OtherRegionsTable lock", true),

  _hr(hr),

  _coarse_map(G1CollectedHeap::heap()->max_regions(),

              false /* in-resource-area */),

  _fine_grain_regions(NULL),

  _n_fine_entries(0), _n_coarse_entries(0),

  _fine_eviction_start(0),

  _sparse_table(hr)

{

  typedef PerRegionTable* PerRegionTablePtr;

  if (_max_fine_entries == 0) {

    assert(_mod_max_fine_entries_mask == 0, "Both or none.");

    size_t max_entries_log = (size_t)log2_long((jlong)G1RSetRegionEntries);

    _max_fine_entries = (size_t)(1 << max_entries_log);

    _mod_max_fine_entries_mask = _max_fine_entries - 1;

    assert(_fine_eviction_sample_size == 0

           && _fine_eviction_stride == 0, "All init at same time.");

    _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log);

    _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size;

  }

  _fine_grain_regions = new PerRegionTablePtr[_max_fine_entries];

  if (_fine_grain_regions == NULL) {

    vm_exit_out_of_memory(sizeof(void*)*_max_fine_entries,

                          "Failed to allocate _fine_grain_entries.");

  }

  for (size_t i = 0; i < _max_fine_entries; i++) {

    _fine_grain_regions[i] = NULL;

  }

}

int**  OtherRegionsTable::_from_card_cache = NULL;

size_t OtherRegionsTable::_from_card_cache_max_regions = 0;

size_t OtherRegionsTable::_from_card_cache_mem_size = 0;

void OtherRegionsTable::init_from_card_cache(size_t max_regions) {

  _from_card_cache_max_regions = max_regions;

  int n_par_rs = HeapRegionRemSet::num_par_rem_sets();

  _from_card_cache = NEW_C_HEAP_ARRAY(int*, n_par_rs, mtGC);

  for (int i = 0; i < n_par_rs; i++) {

    _from_card_cache[i] = NEW_C_HEAP_ARRAY(int, max_regions, mtGC);

    for (size_t j = 0; j < max_regions; j++) {

      _from_card_cache[i][j] = -1;  // An invalid value.

    }

  }

  _from_card_cache_mem_size = n_par_rs * max_regions * sizeof(int);

}

void OtherRegionsTable::shrink_from_card_cache(size_t new_n_regs) {

  for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) {

    assert(new_n_regs <= _from_card_cache_max_regions, "Must be within max.");

    for (size_t j = new_n_regs; j < _from_card_cache_max_regions; j++) {

      _from_card_cache[i][j] = -1;  // An invalid value.

    }

  }

}

#ifndef PRODUCT

void OtherRegionsTable::print_from_card_cache() {

  for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) {

    for (size_t j = 0; j < _from_card_cache_max_regions; j++) {

      gclog_or_tty->print_cr("_from_card_cache[%d][%d] = %d.",

                    i, j, _from_card_cache[i][j]);

    }

  }

}

#endif

void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) {

  size_t cur_hrs_ind = (size_t) hr()->hrs_index();

  if (G1TraceHeapRegionRememberedSet) {

    gclog_or_tty->print_cr("ORT::add_reference_work(" PTR_FORMAT "->" PTR_FORMAT ").",

                                                    from,

                                                    UseCompressedOops

                                                    ? oopDesc::load_decode_heap_oop((narrowOop*)from)

                                                    : oopDesc::load_decode_heap_oop((oop*)from));

  }

  int from_card = (int)(uintptr_t(from) >> CardTableModRefBS::card_shift);

  if (G1TraceHeapRegionRememberedSet) {

    gclog_or_tty->print_cr("Table for [" PTR_FORMAT "...): card %d (cache = %d)",

                  hr()->bottom(), from_card,

                  _from_card_cache[tid][cur_hrs_ind]);

  }

  if (from_card == _from_card_cache[tid][cur_hrs_ind]) {

    if (G1TraceHeapRegionRememberedSet) {

      gclog_or_tty->print_cr("  from-card cache hit.");

    }

    assert(contains_reference(from), "We just added it!");

    return;

  } else {

    _from_card_cache[tid][cur_hrs_ind] = from_card;

  }

  // Note that this may be a continued H region.

  HeapRegion* from_hr = _g1h->heap_region_containing_raw(from);

  RegionIdx_t from_hrs_ind = (RegionIdx_t) from_hr->hrs_index();

  // If the region is already coarsened, return.

  if (_coarse_map.at(from_hrs_ind)) {

    if (G1TraceHeapRegionRememberedSet) {

      gclog_or_tty->print_cr("  coarse map hit.");

    }

    assert(contains_reference(from), "We just added it!");

    return;

  }

  // Otherwise find a per-region table to add it to.

  size_t ind = from_hrs_ind & _mod_max_fine_entries_mask;

  PerRegionTable* prt = find_region_table(ind, from_hr);

  if (prt == NULL) {

    MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag);

    // Confirm that it's really not there...

    prt = find_region_table(ind, from_hr);

    if (prt == NULL) {

      uintptr_t from_hr_bot_card_index =

        uintptr_t(from_hr->bottom())

          >> CardTableModRefBS::card_shift;

      CardIdx_t card_index = from_card - from_hr_bot_card_index;

      assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion,

             "Must be in range.");

      if (G1HRRSUseSparseTable &&

          _sparse_table.add_card(from_hrs_ind, card_index)) {

        if (G1RecordHRRSOops) {

          HeapRegionRemSet::record(hr(), from);

          if (G1TraceHeapRegionRememberedSet) {

            gclog_or_tty->print("   Added card " PTR_FORMAT " to region "

                                "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n",

                                align_size_down(uintptr_t(from),

                                                CardTableModRefBS::card_size),

                                hr()->bottom(), from);

          }

        }

        if (G1TraceHeapRegionRememberedSet) {

          gclog_or_tty->print_cr("   added card to sparse table.");

        }

        assert(contains_reference_locked(from), "We just added it!");

        return;

      } else {

        if (G1TraceHeapRegionRememberedSet) {

          gclog_or_tty->print_cr("   [tid %d] sparse table entry "

                        "overflow(f: %d, t: %d)",

                        tid, from_hrs_ind, cur_hrs_ind);

        }

      }

      if (_n_fine_entries == _max_fine_entries) {

        prt = delete_region_table();

      } else {

        prt = PerRegionTable::alloc(from_hr);

      }

      PerRegionTable* first_prt = _fine_grain_regions[ind];

      _fine_grain_regions[ind] = prt;

      _n_fine_entries++;

      if (G1HRRSUseSparseTable) {

        // Transfer from sparse to fine-grain.

        SparsePRTEntry *sprt_entry = _sparse_table.get_entry(from_hrs_ind);

        assert(sprt_entry != NULL, "There should have been an entry");

        for (int i = 0; i < SparsePRTEntry::cards_num(); i++) {

          CardIdx_t c = sprt_entry->card(i);

          if (c != SparsePRTEntry::NullEntry) {

            prt->add_card(c);

          }

        }

        // Now we can delete the sparse entry.

        bool res = _sparse_table.delete_entry(from_hrs_ind);

        assert(res, "It should have been there.");