/*

 * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "code/icBuffer.hpp"

#include "gc_implementation/g1/bufferingOopClosure.hpp"

#include "gc_implementation/g1/concurrentG1Refine.hpp"

#include "gc_implementation/g1/concurrentG1RefineThread.hpp"

#include "gc_implementation/g1/concurrentMarkThread.inline.hpp"

#include "gc_implementation/g1/concurrentZFThread.hpp"

#include "gc_implementation/g1/g1CollectedHeap.inline.hpp"

#include "gc_implementation/g1/g1CollectorPolicy.hpp"

#include "gc_implementation/g1/g1MarkSweep.hpp"

#include "gc_implementation/g1/g1OopClosures.inline.hpp"

#include "gc_implementation/g1/g1RemSet.inline.hpp"

#include "gc_implementation/g1/heapRegionRemSet.hpp"

#include "gc_implementation/g1/heapRegionSeq.inline.hpp"

#include "gc_implementation/g1/vm_operations_g1.hpp"

#include "gc_implementation/shared/isGCActiveMark.hpp"

#include "memory/gcLocker.inline.hpp"

#include "memory/genOopClosures.inline.hpp"

#include "memory/generationSpec.hpp"

#include "oops/oop.inline.hpp"

#include "oops/oop.pcgc.inline.hpp"

#include "runtime/aprofiler.hpp"

#include "runtime/vmThread.hpp"

size_t G1CollectedHeap::_humongous_object_threshold_in_words = 0;

// turn it on so that the contents of the young list (scan-only /

// to-be-collected) are printed at "strategic" points before / during

// / after the collection --- this is useful for debugging

#define YOUNG_LIST_VERBOSE 0

// CURRENT STATUS

// This file is under construction.  Search for "FIXME".

// INVARIANTS/NOTES

//

// All allocation activity covered by the G1CollectedHeap interface is

// serialized by acquiring the HeapLock.  This happens in mem_allocate

// and allocate_new_tlab, which are the "entry" points to the

// allocation code from the rest of the JVM.  (Note that this does not

// apply to TLAB allocation, which is not part of this interface: it

// is done by clients of this interface.)

// Local to this file.

class RefineCardTableEntryClosure: public CardTableEntryClosure {

  SuspendibleThreadSet* _sts;

  G1RemSet* _g1rs;

  ConcurrentG1Refine* _cg1r;

  bool _concurrent;

public:

  RefineCardTableEntryClosure(SuspendibleThreadSet* sts,

                              G1RemSet* g1rs,

                              ConcurrentG1Refine* cg1r) :

    _sts(sts), _g1rs(g1rs), _cg1r(cg1r), _concurrent(true)

  {}

  bool do_card_ptr(jbyte* card_ptr, int worker_i) {

    bool oops_into_cset = _g1rs->concurrentRefineOneCard(card_ptr, worker_i, false);

    // This path is executed by the concurrent refine or mutator threads,

    // concurrently, and so we do not care if card_ptr contains references

    // that point into the collection set.

    assert(!oops_into_cset, "should be");

    if (_concurrent && _sts->should_yield()) {

      // Caller will actually yield.

      return false;

    }

    // Otherwise, we finished successfully; return true.

    return true;

  }

  void set_concurrent(bool b) { _concurrent = b; }

};

class ClearLoggedCardTableEntryClosure: public CardTableEntryClosure {

  int _calls;

  G1CollectedHeap* _g1h;

  CardTableModRefBS* _ctbs;

  int _histo[256];

public:

  ClearLoggedCardTableEntryClosure() :

    _calls(0)

  {

    _g1h = G1CollectedHeap::heap();

    _ctbs = (CardTableModRefBS*)_g1h->barrier_set();

    for (int i = 0; i < 256; i++) _histo[i] = 0;

  }

  bool do_card_ptr(jbyte* card_ptr, int worker_i) {

    if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {

      _calls++;

      unsigned char* ujb = (unsigned char*)card_ptr;

      int ind = (int)(*ujb);

      _histo[ind]++;

      *card_ptr = -1;

    }

    return true;

  }

  int calls() { return _calls; }

  void print_histo() {

    gclog_or_tty->print_cr("Card table value histogram:");

    for (int i = 0; i < 256; i++) {

      if (_histo[i] != 0) {

        gclog_or_tty->print_cr("  %d: %d", i, _histo[i]);

      }

    }

  }

};

class RedirtyLoggedCardTableEntryClosure: public CardTableEntryClosure {

  int _calls;

  G1CollectedHeap* _g1h;

  CardTableModRefBS* _ctbs;

public:

  RedirtyLoggedCardTableEntryClosure() :

    _calls(0)

  {

    _g1h = G1CollectedHeap::heap();

    _ctbs = (CardTableModRefBS*)_g1h->barrier_set();

  }

  bool do_card_ptr(jbyte* card_ptr, int worker_i) {

    if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {

      _calls++;

      *card_ptr = 0;

    }

    return true;

  }

  int calls() { return _calls; }

};

class RedirtyLoggedCardTableEntryFastClosure : public CardTableEntryClosure {

public:

  bool do_card_ptr(jbyte* card_ptr, int worker_i) {

    *card_ptr = CardTableModRefBS::dirty_card_val();

    return true;

  }

};

YoungList::YoungList(G1CollectedHeap* g1h)

  : _g1h(g1h), _head(NULL),

    _length(0),

    _last_sampled_rs_lengths(0),

    _survivor_head(NULL), _survivor_tail(NULL), _survivor_length(0)

{

  guarantee( check_list_empty(false), "just making sure..." );

}

void YoungList::push_region(HeapRegion *hr) {

  assert(!hr->is_young(), "should not already be young");

  assert(hr->get_next_young_region() == NULL, "cause it should!");

  hr->set_next_young_region(_head);

  _head = hr;

  hr->set_young();

  double yg_surv_rate = _g1h->g1_policy()->predict_yg_surv_rate((int)_length);

  ++_length;

}

void YoungList::add_survivor_region(HeapRegion* hr) {

  assert(hr->is_survivor(), "should be flagged as survivor region");

  assert(hr->get_next_young_region() == NULL, "cause it should!");

  hr->set_next_young_region(_survivor_head);

  if (_survivor_head == NULL) {

    _survivor_tail = hr;

  }

  _survivor_head = hr;

  ++_survivor_length;

}

void YoungList::empty_list(HeapRegion* list) {

  while (list != NULL) {

    HeapRegion* next = list->get_next_young_region();

    list->set_next_young_region(NULL);

    list->uninstall_surv_rate_group();

    list->set_not_young();

    list = next;

  }

}

void YoungList::empty_list() {

  assert(check_list_well_formed(), "young list should be well formed");

  empty_list(_head);

  _head = NULL;

  _length = 0;

  empty_list(_survivor_head);

  _survivor_head = NULL;

  _survivor_tail = NULL;

  _survivor_length = 0;

  _last_sampled_rs_lengths = 0;

  assert(check_list_empty(false), "just making sure...");

}

bool YoungList::check_list_well_formed() {

  bool ret = true;

  size_t length = 0;

  HeapRegion* curr = _head;

  HeapRegion* last = NULL;

  while (curr != NULL) {

    if (!curr->is_young()) {

      gclog_or_tty->print_cr("### YOUNG REGION "PTR_FORMAT"-"PTR_FORMAT" "

                             "incorrectly tagged (y: %d, surv: %d)",

                             curr->bottom(), curr->end(),

                             curr->is_young(), curr->is_survivor());

      ret = false;

    }

    ++length;

    last = curr;

    curr = curr->get_next_young_region();

  }

  ret = ret && (length == _length);

  if (!ret) {

    gclog_or_tty->print_cr("### YOUNG LIST seems not well formed!");

    gclog_or_tty->print_cr("###   list has %d entries, _length is %d",

                           length, _length);

  }

  return ret;

}

bool YoungList::check_list_empty(bool check_sample) {

  bool ret = true;

  if (_length != 0) {

    gclog_or_tty->print_cr("### YOUNG LIST should have 0 length, not %d",

                  _length);

    ret = false;

  }

  if (check_sample && _last_sampled_rs_lengths != 0) {

    gclog_or_tty->print_cr("### YOUNG LIST has non-zero last sampled RS lengths");

    ret = false;

  }

  if (_head != NULL) {

    gclog_or_tty->print_cr("### YOUNG LIST does not have a NULL head");

    ret = false;

  }

  if (!ret) {

    gclog_or_tty->print_cr("### YOUNG LIST does not seem empty");

  }

  return ret;

}

void

YoungList::rs_length_sampling_init() {

  _sampled_rs_lengths = 0;

  _curr               = _head;

}

bool

YoungList::rs_length_sampling_more() {

  return _curr != NULL;

}

void

YoungList::rs_length_sampling_next() {

  assert( _curr != NULL, "invariant" );

  size_t rs_length = _curr->rem_set()->occupied();

  _sampled_rs_lengths += rs_length;

  // The current region may not yet have been added to the

  // incremental collection set (it gets added when it is

  // retired as the current allocation region).

  if (_curr->in_collection_set()) {

    // Update the collection set policy information for this region

    _g1h->g1_policy()->update_incremental_cset_info(_curr, rs_length);

  }

  _curr = _curr->get_next_young_region();

  if (_curr == NULL) {

    _last_sampled_rs_lengths = _sampled_rs_lengths;

    // gclog_or_tty->print_cr("last sampled RS lengths = %d", _last_sampled_rs_lengths);

  }

}

void

YoungList::reset_auxilary_lists() {

  guarantee( is_empty(), "young list should be empty" );

  assert(check_list_well_formed(), "young list should be well formed");

  // Add survivor regions to SurvRateGroup.

  _g1h->g1_policy()->note_start_adding_survivor_regions();

  _g1h->g1_policy()->finished_recalculating_age_indexes(true /* is_survivors */);

  for (HeapRegion* curr = _survivor_head;

       curr != NULL;

       curr = curr->get_next_young_region()) {

    _g1h->g1_policy()->set_region_survivors(curr);

    // The region is a non-empty survivor so let's add it to

    // the incremental collection set for the next evacuation

    // pause.

    _g1h->g1_policy()->add_region_to_incremental_cset_rhs(curr);

  }

  _g1h->g1_policy()->note_stop_adding_survivor_regions();

  _head   = _survivor_head;

  _length = _survivor_length;

  if (_survivor_head != NULL) {

    assert(_survivor_tail != NULL, "cause it shouldn't be");

    assert(_survivor_length > 0, "invariant");

    _survivor_tail->set_next_young_region(NULL);

  }

  // Don't clear the survivor list handles until the start of

  // the next evacuation pause - we need it in order to re-tag

  // the survivor regions from this evacuation pause as 'young'

  // at the start of the next.

  _g1h->g1_policy()->finished_recalculating_age_indexes(false /* is_survivors */);

  assert(check_list_well_formed(), "young list should be well formed");

}

void YoungList::print() {

  HeapRegion* lists[] = {_head,   _survivor_head};

  const char* names[] = {"YOUNG", "SURVIVOR"};

  for (unsigned int list = 0; list < ARRAY_SIZE(lists); ++list) {

    gclog_or_tty->print_cr("%s LIST CONTENTS", names[list]);

    HeapRegion *curr = lists[list];

    if (curr == NULL)

      gclog_or_tty->print_cr("  empty");

    while (curr != NULL) {

      gclog_or_tty->print_cr("  [%08x-%08x], t: %08x, P: %08x, N: %08x, C: %08x, "

                             "age: %4d, y: %d, surv: %d",

                             curr->bottom(), curr->end(),

                             curr->top(),

                             curr->prev_top_at_mark_start(),

                             curr->next_top_at_mark_start(),

                             curr->top_at_conc_mark_count(),

                             curr->age_in_surv_rate_group_cond(),

                             curr->is_young(),

                             curr->is_survivor());

      curr = curr->get_next_young_region();

    }

  }

  gclog_or_tty->print_cr("");

}

void G1CollectedHeap::push_dirty_cards_region(HeapRegion* hr)

{

  // Claim the right to put the region on the dirty cards region list

  // by installing a self pointer.

  HeapRegion* next = hr->get_next_dirty_cards_region();

  if (next == NULL) {

    HeapRegion* res = (HeapRegion*)

      Atomic::cmpxchg_ptr(hr, hr->next_dirty_cards_region_addr(),

                          NULL);

    if (res == NULL) {

      HeapRegion* head;

      do {

        // Put the region to the dirty cards region list.

        head = _dirty_cards_region_list;

        next = (HeapRegion*)

          Atomic::cmpxchg_ptr(hr, &_dirty_cards_region_list, head);

        if (next == head) {

          assert(hr->get_next_dirty_cards_region() == hr,

                 "hr->get_next_dirty_cards_region() != hr");

          if (next == NULL) {

            // The last region in the list points to itself.

            hr->set_next_dirty_cards_region(hr);

          } else {

            hr->set_next_dirty_cards_region(next);

          }

        }

      } while (next != head);

    }

  }

}

HeapRegion* G1CollectedHeap::pop_dirty_cards_region()

{

  HeapRegion* head;

  HeapRegion* hr;

  do {

    head = _dirty_cards_region_list;

    if (head == NULL) {

      return NULL;

    }

    HeapRegion* new_head = head->get_next_dirty_cards_region();

    if (head == new_head) {

      // The last region.

      new_head = NULL;

    }

    hr = (HeapRegion*)Atomic::cmpxchg_ptr(new_head, &_dirty_cards_region_list,

                                          head);

  } while (hr != head);

  assert(hr != NULL, "invariant");

  hr->set_next_dirty_cards_region(NULL);

  return hr;

}

void G1CollectedHeap::stop_conc_gc_threads() {

  _cg1r->stop();

  _czft->stop();

  _cmThread->stop();

}

void G1CollectedHeap::check_ct_logs_at_safepoint() {

  DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();

  CardTableModRefBS* ct_bs = (CardTableModRefBS*)barrier_set();

  // Count the dirty cards at the start.

  CountNonCleanMemRegionClosure count1(this);

  ct_bs->mod_card_iterate(&count1);

  int orig_count = count1.n();

  // First clear the logged cards.

  ClearLoggedCardTableEntryClosure clear;

  dcqs.set_closure(&clear);

  dcqs.apply_closure_to_all_completed_buffers();

  dcqs.iterate_closure_all_threads(false);

  clear.print_histo();

  // Now ensure that there's no dirty cards.

  CountNonCleanMemRegionClosure count2(this);

  ct_bs->mod_card_iterate(&count2);

  if (count2.n() != 0) {

    gclog_or_tty->print_cr("Card table has %d entries; %d originally",

                           count2.n(), orig_count);

  }

  guarantee(count2.n() == 0, "Card table should be clean.");

  RedirtyLoggedCardTableEntryClosure redirty;

  JavaThread::dirty_card_queue_set().set_closure(&redirty);

  dcqs.apply_closure_to_all_completed_buffers();

  dcqs.iterate_closure_all_threads(false);

  gclog_or_tty->print_cr("Log entries = %d, dirty cards = %d.",

                         clear.calls(), orig_count);

  guarantee(redirty.calls() == clear.calls(),

            "Or else mechanism is broken.");

  CountNonCleanMemRegionClosure count3(this);

  ct_bs->mod_card_iterate(&count3);

  if (count3.n() != orig_count) {

    gclog_or_tty->print_cr("Should have restored them all: orig = %d, final = %d.",

                           orig_count, count3.n());

    guarantee(count3.n() >= orig_count, "Should have restored them all.");

  }

  JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl);

}

// Private class members.

G1CollectedHeap* G1CollectedHeap::_g1h;

// Private methods.

// Finds a HeapRegion that can be used to allocate a given size of block.

HeapRegion* G1CollectedHeap::newAllocRegion_work(size_t word_size,

                                                 bool do_expand,

                                                 bool zero_filled) {

  ConcurrentZFThread::note_region_alloc();

  HeapRegion* res = alloc_free_region_from_lists(zero_filled);

  if (res == NULL && do_expand) {

    expand(word_size * HeapWordSize);

    res = alloc_free_region_from_lists(zero_filled);

    assert(res == NULL ||

           (!res->isHumongous() &&

            (!zero_filled ||

             res->zero_fill_state() == HeapRegion::Allocated)),

           "Alloc Regions must be zero filled (and non-H)");

  }

  if (res != NULL) {

    if (res->is_empty()) {

      _free_regions--;

    }

    assert(!res->isHumongous() &&

           (!zero_filled || res->zero_fill_state() == HeapRegion::Allocated),

           err_msg("Non-young alloc Regions must be zero filled (and non-H):"

                   " res->isHumongous()=%d, zero_filled=%d, res->zero_fill_state()=%d",

                   res->isHumongous(), zero_filled, res->zero_fill_state()));

    assert(!res->is_on_unclean_list(),

           "Alloc Regions must not be on the unclean list");

    if (G1PrintHeapRegions) {

      gclog_or_tty->print_cr("new alloc region %d:["PTR_FORMAT", "PTR_FORMAT"], "