/*

 * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,

 * CA 95054 USA or visit www.sun.com if you need additional information or

 * have any questions.

 *

 */

#include "incls/_precompiled.incl"

#include "incls/_g1CollectedHeap.cpp.incl"

// turn it on so that the contents of the young list (scan-only /

// to-be-collected) are printed at "strategic" points before / during

// / after the collection --- this is useful for debugging

#define SCAN_ONLY_VERBOSE 0

// CURRENT STATUS

// This file is under construction.  Search for "FIXME".

// INVARIANTS/NOTES

//

// All allocation activity covered by the G1CollectedHeap interface is

//   serialized by acquiring the HeapLock.  This happens in

//   mem_allocate_work, which all such allocation functions call.

//   (Note that this does not apply to TLAB allocation, which is not part

//   of this interface: it is done by clients of this interface.)

// Local to this file.

class RefineCardTableEntryClosure: public CardTableEntryClosure {

  SuspendibleThreadSet* _sts;

  G1RemSet* _g1rs;

  ConcurrentG1Refine* _cg1r;

  bool _concurrent;

public:

  RefineCardTableEntryClosure(SuspendibleThreadSet* sts,

                              G1RemSet* g1rs,

                              ConcurrentG1Refine* cg1r) :

    _sts(sts), _g1rs(g1rs), _cg1r(cg1r), _concurrent(true)

  {}

  bool do_card_ptr(jbyte* card_ptr, int worker_i) {

    _g1rs->concurrentRefineOneCard(card_ptr, worker_i);

    if (_concurrent && _sts->should_yield()) {

      // Caller will actually yield.

      return false;

    }

    // Otherwise, we finished successfully; return true.

    return true;

  }

  void set_concurrent(bool b) { _concurrent = b; }

};

class ClearLoggedCardTableEntryClosure: public CardTableEntryClosure {

  int _calls;

  G1CollectedHeap* _g1h;

  CardTableModRefBS* _ctbs;

  int _histo[256];

public:

  ClearLoggedCardTableEntryClosure() :

    _calls(0)

  {

    _g1h = G1CollectedHeap::heap();

    _ctbs = (CardTableModRefBS*)_g1h->barrier_set();

    for (int i = 0; i < 256; i++) _histo[i] = 0;

  }

  bool do_card_ptr(jbyte* card_ptr, int worker_i) {

    if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {

      _calls++;

      unsigned char* ujb = (unsigned char*)card_ptr;

      int ind = (int)(*ujb);

      _histo[ind]++;

      *card_ptr = -1;

    }

    return true;

  }

  int calls() { return _calls; }

  void print_histo() {

    gclog_or_tty->print_cr("Card table value histogram:");

    for (int i = 0; i < 256; i++) {

      if (_histo[i] != 0) {

        gclog_or_tty->print_cr("  %d: %d", i, _histo[i]);

      }

    }

  }

};

class RedirtyLoggedCardTableEntryClosure: public CardTableEntryClosure {

  int _calls;

  G1CollectedHeap* _g1h;

  CardTableModRefBS* _ctbs;

public:

  RedirtyLoggedCardTableEntryClosure() :

    _calls(0)

  {

    _g1h = G1CollectedHeap::heap();

    _ctbs = (CardTableModRefBS*)_g1h->barrier_set();

  }

  bool do_card_ptr(jbyte* card_ptr, int worker_i) {

    if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {

      _calls++;

      *card_ptr = 0;

    }

    return true;

  }

  int calls() { return _calls; }

};

class RedirtyLoggedCardTableEntryFastClosure : public CardTableEntryClosure {

public:

  bool do_card_ptr(jbyte* card_ptr, int worker_i) {

    *card_ptr = CardTableModRefBS::dirty_card_val();

    return true;

  }

};

YoungList::YoungList(G1CollectedHeap* g1h)

  : _g1h(g1h), _head(NULL),

    _scan_only_head(NULL), _scan_only_tail(NULL), _curr_scan_only(NULL),

    _length(0), _scan_only_length(0),

    _last_sampled_rs_lengths(0),

    _survivor_head(NULL), _survivor_tail(NULL), _survivor_length(0)

{

  guarantee( check_list_empty(false), "just making sure..." );

}

void YoungList::push_region(HeapRegion *hr) {

  assert(!hr->is_young(), "should not already be young");

  assert(hr->get_next_young_region() == NULL, "cause it should!");

  hr->set_next_young_region(_head);

  _head = hr;

  hr->set_young();

  double yg_surv_rate = _g1h->g1_policy()->predict_yg_surv_rate((int)_length);

  ++_length;

}

void YoungList::add_survivor_region(HeapRegion* hr) {

  assert(hr->is_survivor(), "should be flagged as survivor region");

  assert(hr->get_next_young_region() == NULL, "cause it should!");

  hr->set_next_young_region(_survivor_head);

  if (_survivor_head == NULL) {

    _survivor_tail = hr;

  }

  _survivor_head = hr;

  ++_survivor_length;

}

HeapRegion* YoungList::pop_region() {

  while (_head != NULL) {

    assert( length() > 0, "list should not be empty" );

    HeapRegion* ret = _head;

    _head = ret->get_next_young_region();

    ret->set_next_young_region(NULL);

    --_length;

    assert(ret->is_young(), "region should be very young");

    // Replace 'Survivor' region type with 'Young'. So the region will

    // be treated as a young region and will not be 'confused' with

    // newly created survivor regions.

    if (ret->is_survivor()) {

      ret->set_young();

    }

    if (!ret->is_scan_only()) {

      return ret;

    }

    // scan-only, we'll add it to the scan-only list

    if (_scan_only_tail == NULL) {

      guarantee( _scan_only_head == NULL, "invariant" );

      _scan_only_head = ret;

      _curr_scan_only = ret;

    } else {

      guarantee( _scan_only_head != NULL, "invariant" );

      _scan_only_tail->set_next_young_region(ret);

    }

    guarantee( ret->get_next_young_region() == NULL, "invariant" );

    _scan_only_tail = ret;

    // no need to be tagged as scan-only any more

    ret->set_young();

    ++_scan_only_length;

  }

  assert( length() == 0, "list should be empty" );

  return NULL;

}

void YoungList::empty_list(HeapRegion* list) {

  while (list != NULL) {

    HeapRegion* next = list->get_next_young_region();

    list->set_next_young_region(NULL);

    list->uninstall_surv_rate_group();

    list->set_not_young();

    list = next;

  }

}

void YoungList::empty_list() {

  assert(check_list_well_formed(), "young list should be well formed");

  empty_list(_head);

  _head = NULL;

  _length = 0;

  empty_list(_scan_only_head);

  _scan_only_head = NULL;

  _scan_only_tail = NULL;

  _scan_only_length = 0;

  _curr_scan_only = NULL;

  empty_list(_survivor_head);

  _survivor_head = NULL;

  _survivor_tail = NULL;

  _survivor_length = 0;

  _last_sampled_rs_lengths = 0;

  assert(check_list_empty(false), "just making sure...");

}

bool YoungList::check_list_well_formed() {

  bool ret = true;

  size_t length = 0;

  HeapRegion* curr = _head;

  HeapRegion* last = NULL;

  while (curr != NULL) {

    if (!curr->is_young() || curr->is_scan_only()) {

      gclog_or_tty->print_cr("### YOUNG REGION "PTR_FORMAT"-"PTR_FORMAT" "

                             "incorrectly tagged (%d, %d)",

                             curr->bottom(), curr->end(),

                             curr->is_young(), curr->is_scan_only());

      ret = false;

    }

    ++length;

    last = curr;

    curr = curr->get_next_young_region();

  }

  ret = ret && (length == _length);

  if (!ret) {

    gclog_or_tty->print_cr("### YOUNG LIST seems not well formed!");

    gclog_or_tty->print_cr("###   list has %d entries, _length is %d",

                           length, _length);

  }

  bool scan_only_ret = true;

  length = 0;

  curr = _scan_only_head;

  last = NULL;

  while (curr != NULL) {

    if (!curr->is_young() || curr->is_scan_only()) {

      gclog_or_tty->print_cr("### SCAN-ONLY REGION "PTR_FORMAT"-"PTR_FORMAT" "

                             "incorrectly tagged (%d, %d)",

                             curr->bottom(), curr->end(),

                             curr->is_young(), curr->is_scan_only());

      scan_only_ret = false;

    }

    ++length;

    last = curr;

    curr = curr->get_next_young_region();

  }

  scan_only_ret = scan_only_ret && (length == _scan_only_length);

  if ( (last != _scan_only_tail) ||

       (_scan_only_head == NULL && _scan_only_tail != NULL) ||

       (_scan_only_head != NULL && _scan_only_tail == NULL) ) {

     gclog_or_tty->print_cr("## _scan_only_tail is set incorrectly");

     scan_only_ret = false;

  }

  if (_curr_scan_only != NULL && _curr_scan_only != _scan_only_head) {

    gclog_or_tty->print_cr("### _curr_scan_only is set incorrectly");

    scan_only_ret = false;

   }

  if (!scan_only_ret) {

    gclog_or_tty->print_cr("### SCAN-ONLY LIST seems not well formed!");

    gclog_or_tty->print_cr("###   list has %d entries, _scan_only_length is %d",

                  length, _scan_only_length);

  }

  return ret && scan_only_ret;

}

bool YoungList::check_list_empty(bool ignore_scan_only_list,

                                 bool check_sample) {

  bool ret = true;

  if (_length != 0) {

    gclog_or_tty->print_cr("### YOUNG LIST should have 0 length, not %d",

                  _length);

    ret = false;

  }

  if (check_sample && _last_sampled_rs_lengths != 0) {

    gclog_or_tty->print_cr("### YOUNG LIST has non-zero last sampled RS lengths");

    ret = false;

  }

  if (_head != NULL) {

    gclog_or_tty->print_cr("### YOUNG LIST does not have a NULL head");

    ret = false;

  }

  if (!ret) {

    gclog_or_tty->print_cr("### YOUNG LIST does not seem empty");

  }

  if (ignore_scan_only_list)

    return ret;

  bool scan_only_ret = true;

  if (_scan_only_length != 0) {

    gclog_or_tty->print_cr("### SCAN-ONLY LIST should have 0 length, not %d",

                  _scan_only_length);

    scan_only_ret = false;

  }

  if (_scan_only_head != NULL) {

    gclog_or_tty->print_cr("### SCAN-ONLY LIST does not have a NULL head");

     scan_only_ret = false;

  }

  if (_scan_only_tail != NULL) {

    gclog_or_tty->print_cr("### SCAN-ONLY LIST does not have a NULL tail");

    scan_only_ret = false;

  }

  if (!scan_only_ret) {

    gclog_or_tty->print_cr("### SCAN-ONLY LIST does not seem empty");

  }

  return ret && scan_only_ret;

}

void

YoungList::rs_length_sampling_init() {

  _sampled_rs_lengths = 0;

  _curr               = _head;

}

bool

YoungList::rs_length_sampling_more() {

  return _curr != NULL;

}

void

YoungList::rs_length_sampling_next() {

  assert( _curr != NULL, "invariant" );

  _sampled_rs_lengths += _curr->rem_set()->occupied();

  _curr = _curr->get_next_young_region();

  if (_curr == NULL) {

    _last_sampled_rs_lengths = _sampled_rs_lengths;

    // gclog_or_tty->print_cr("last sampled RS lengths = %d", _last_sampled_rs_lengths);

  }

}

void

YoungList::reset_auxilary_lists() {

  // We could have just "moved" the scan-only list to the young list.

  // However, the scan-only list is ordered according to the region

  // age in descending order, so, by moving one entry at a time, we

  // ensure that it is recreated in ascending order.

  guarantee( is_empty(), "young list should be empty" );

  assert(check_list_well_formed(), "young list should be well formed");

  // Add survivor regions to SurvRateGroup.

  _g1h->g1_policy()->note_start_adding_survivor_regions();

  _g1h->g1_policy()->finished_recalculating_age_indexes(true /* is_survivors */);

  for (HeapRegion* curr = _survivor_head;

       curr != NULL;

       curr = curr->get_next_young_region()) {

    _g1h->g1_policy()->set_region_survivors(curr);

  }

  _g1h->g1_policy()->note_stop_adding_survivor_regions();

  if (_survivor_head != NULL) {

    _head           = _survivor_head;

    _length         = _survivor_length + _scan_only_length;

    _survivor_tail->set_next_young_region(_scan_only_head);

  } else {

    _head           = _scan_only_head;

    _length         = _scan_only_length;

  }

  for (HeapRegion* curr = _scan_only_head;

       curr != NULL;

       curr = curr->get_next_young_region()) {

    curr->recalculate_age_in_surv_rate_group();

  }

  _scan_only_head   = NULL;

  _scan_only_tail   = NULL;

  _scan_only_length = 0;

  _curr_scan_only   = NULL;

  _survivor_head    = NULL;

  _survivor_tail   = NULL;

  _survivor_length  = 0;

  _g1h->g1_policy()->finished_recalculating_age_indexes(false /* is_survivors */);

  assert(check_list_well_formed(), "young list should be well formed");

}

void YoungList::print() {

  HeapRegion* lists[] = {_head,   _scan_only_head, _survivor_head};

  const char* names[] = {"YOUNG", "SCAN-ONLY",     "SURVIVOR"};

  for (unsigned int list = 0; list < ARRAY_SIZE(lists); ++list) {

    gclog_or_tty->print_cr("%s LIST CONTENTS", names[list]);

    HeapRegion *curr = lists[list];

    if (curr == NULL)

      gclog_or_tty->print_cr("  empty");

    while (curr != NULL) {

      gclog_or_tty->print_cr("  [%08x-%08x], t: %08x, P: %08x, N: %08x, C: %08x, "

                             "age: %4d, y: %d, s-o: %d, surv: %d",

                             curr->bottom(), curr->end(),

                             curr->top(),

                             curr->prev_top_at_mark_start(),

                             curr->next_top_at_mark_start(),

                             curr->top_at_conc_mark_count(),

                             curr->age_in_surv_rate_group_cond(),

                             curr->is_young(),

                             curr->is_scan_only(),

                             curr->is_survivor());

      curr = curr->get_next_young_region();

    }

  }

  gclog_or_tty->print_cr("");

}

void G1CollectedHeap::stop_conc_gc_threads() {

  _cg1r->cg1rThread()->stop();

  _czft->stop();

  _cmThread->stop();

}

void G1CollectedHeap::check_ct_logs_at_safepoint() {

  DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();

  CardTableModRefBS* ct_bs = (CardTableModRefBS*)barrier_set();

  // Count the dirty cards at the start.

  CountNonCleanMemRegionClosure count1(this);

  ct_bs->mod_card_iterate(&count1);

  int orig_count = count1.n();

  // First clear the logged cards.

  ClearLoggedCardTableEntryClosure clear;

  dcqs.set_closure(&clear);

  dcqs.apply_closure_to_all_completed_buffers();

  dcqs.iterate_closure_all_threads(false);

  clear.print_histo();

  // Now ensure that there's no dirty cards.

  CountNonCleanMemRegionClosure count2(this);

  ct_bs->mod_card_iterate(&count2);

  if (count2.n() != 0) {

    gclog_or_tty->print_cr("Card table has %d entries; %d originally",

                           count2.n(), orig_count);

  }

  guarantee(count2.n() == 0, "Card table should be clean.");

  RedirtyLoggedCardTableEntryClosure redirty;

  JavaThread::dirty_card_queue_set().set_closure(&redirty);

  dcqs.apply_closure_to_all_completed_buffers();

  dcqs.iterate_closure_all_threads(false);

  gclog_or_tty->print_cr("Log entries = %d, dirty cards = %d.",

                         clear.calls(), orig_count);

  guarantee(redirty.calls() == clear.calls(),

            "Or else mechanism is broken.");

  CountNonCleanMemRegionClosure count3(this);

  ct_bs->mod_card_iterate(&count3);

  if (count3.n() != orig_count) {

    gclog_or_tty->print_cr("Should have restored them all: orig = %d, final = %d.",

                           orig_count, count3.n());

    guarantee(count3.n() >= orig_count, "Should have restored them all.");

  }

  JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl);

}

// Private class members.

G1CollectedHeap* G1CollectedHeap::_g1h;

// Private methods.

// Finds a HeapRegion that can be used to allocate a given size of block.

HeapRegion* G1CollectedHeap::newAllocRegion_work(size_t word_size,

                                                 bool do_expand,

                                                 bool zero_filled) {

  ConcurrentZFThread::note_region_alloc();

  HeapRegion* res = alloc_free_region_from_lists(zero_filled);

  if (res == NULL && do_expand) {

    expand(word_size * HeapWordSize);

    res = alloc_free_region_from_lists(zero_filled);

    assert(res == NULL ||

           (!res->isHumongous() &&

            (!zero_filled ||

             res->zero_fill_state() == HeapRegion::Allocated)),

           "Alloc Regions must be zero filled (and non-H)");

  }

  if (res != NULL && res->is_empty()) _free_regions--;

  assert(res == NULL ||

         (!res->isHumongous() &&