/*

 * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "gc_implementation/g1/concurrentG1Refine.hpp"

#include "gc_implementation/g1/concurrentMark.hpp"

#include "gc_implementation/g1/concurrentMarkThread.inline.hpp"

#include "gc_implementation/g1/g1CollectedHeap.inline.hpp"

#include "gc_implementation/g1/g1CollectorPolicy.hpp"

#include "gc_implementation/g1/g1ErgoVerbose.hpp"

#include "gc_implementation/g1/heapRegionRemSet.hpp"

#include "gc_implementation/shared/gcPolicyCounters.hpp"

#include "runtime/arguments.hpp"

#include "runtime/java.hpp"

#include "runtime/mutexLocker.hpp"

#include "utilities/debug.hpp"

// Different defaults for different number of GC threads

// They were chosen by running GCOld and SPECjbb on debris with different

//   numbers of GC threads and choosing them based on the results

// all the same

static double rs_length_diff_defaults[] = {

  0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0

};

static double cost_per_card_ms_defaults[] = {

  0.01, 0.005, 0.005, 0.003, 0.003, 0.002, 0.002, 0.0015

};

// all the same

static double young_cards_per_entry_ratio_defaults[] = {

  1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0

};

static double cost_per_entry_ms_defaults[] = {

  0.015, 0.01, 0.01, 0.008, 0.008, 0.0055, 0.0055, 0.005

};

static double cost_per_byte_ms_defaults[] = {

  0.00006, 0.00003, 0.00003, 0.000015, 0.000015, 0.00001, 0.00001, 0.000009

};

// these should be pretty consistent

static double constant_other_time_ms_defaults[] = {

  5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0

};

static double young_other_cost_per_region_ms_defaults[] = {

  0.3, 0.2, 0.2, 0.15, 0.15, 0.12, 0.12, 0.1

};

static double non_young_other_cost_per_region_ms_defaults[] = {

  1.0, 0.7, 0.7, 0.5, 0.5, 0.42, 0.42, 0.30

};

// Help class for avoiding interleaved logging

class LineBuffer: public StackObj {

private:

  static const int BUFFER_LEN = 1024;

  static const int INDENT_CHARS = 3;

  char _buffer[BUFFER_LEN];

  int _indent_level;

  int _cur;

  void vappend(const char* format, va_list ap) {

    int res = vsnprintf(&_buffer[_cur], BUFFER_LEN - _cur, format, ap);

    if (res != -1) {

      _cur += res;

    } else {

      DEBUG_ONLY(warning("buffer too small in LineBuffer");)

      _buffer[BUFFER_LEN -1] = 0;

      _cur = BUFFER_LEN; // vsnprintf above should not add to _buffer if we are called again

    }

  }

public:

  explicit LineBuffer(int indent_level): _indent_level(indent_level), _cur(0) {

    for (; (_cur < BUFFER_LEN && _cur < (_indent_level * INDENT_CHARS)); _cur++) {

      _buffer[_cur] = ' ';

    }

  }

#ifndef PRODUCT

  ~LineBuffer() {

    assert(_cur == _indent_level * INDENT_CHARS, "pending data in buffer - append_and_print_cr() not called?");

  }

#endif

  void append(const char* format, ...) {

    va_list ap;

    va_start(ap, format);

    vappend(format, ap);

    va_end(ap);

  }

  void append_and_print_cr(const char* format, ...) {

    va_list ap;

    va_start(ap, format);

    vappend(format, ap);

    va_end(ap);

    gclog_or_tty->print_cr("%s", _buffer);

    _cur = _indent_level * INDENT_CHARS;

  }

};

G1CollectorPolicy::G1CollectorPolicy() :

  _parallel_gc_threads(G1CollectedHeap::use_parallel_gc_threads()

                        ? ParallelGCThreads : 1),

  _recent_gc_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)),

  _all_pause_times_ms(new NumberSeq()),

  _stop_world_start(0.0),

  _all_stop_world_times_ms(new NumberSeq()),

  _all_yield_times_ms(new NumberSeq()),

  _summary(new Summary()),

  _cur_clear_ct_time_ms(0.0),

  _mark_closure_time_ms(0.0),

  _cur_ref_proc_time_ms(0.0),

  _cur_ref_enq_time_ms(0.0),

#ifndef PRODUCT

  _min_clear_cc_time_ms(-1.0),

  _max_clear_cc_time_ms(-1.0),

  _cur_clear_cc_time_ms(0.0),

  _cum_clear_cc_time_ms(0.0),

  _num_cc_clears(0L),

#endif

  _aux_num(10),

  _all_aux_times_ms(new NumberSeq[_aux_num]),

  _cur_aux_start_times_ms(new double[_aux_num]),

  _cur_aux_times_ms(new double[_aux_num]),

  _cur_aux_times_set(new bool[_aux_num]),

  _concurrent_mark_remark_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)),

  _concurrent_mark_cleanup_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)),

  _alloc_rate_ms_seq(new TruncatedSeq(TruncatedSeqLength)),

  _prev_collection_pause_end_ms(0.0),

  _pending_card_diff_seq(new TruncatedSeq(TruncatedSeqLength)),

  _rs_length_diff_seq(new TruncatedSeq(TruncatedSeqLength)),

  _cost_per_card_ms_seq(new TruncatedSeq(TruncatedSeqLength)),

  _young_cards_per_entry_ratio_seq(new TruncatedSeq(TruncatedSeqLength)),

  _mixed_cards_per_entry_ratio_seq(new TruncatedSeq(TruncatedSeqLength)),

  _cost_per_entry_ms_seq(new TruncatedSeq(TruncatedSeqLength)),

  _mixed_cost_per_entry_ms_seq(new TruncatedSeq(TruncatedSeqLength)),

  _cost_per_byte_ms_seq(new TruncatedSeq(TruncatedSeqLength)),

  _cost_per_byte_ms_during_cm_seq(new TruncatedSeq(TruncatedSeqLength)),

  _constant_other_time_ms_seq(new TruncatedSeq(TruncatedSeqLength)),

  _young_other_cost_per_region_ms_seq(new TruncatedSeq(TruncatedSeqLength)),

  _non_young_other_cost_per_region_ms_seq(

                                         new TruncatedSeq(TruncatedSeqLength)),

  _pending_cards_seq(new TruncatedSeq(TruncatedSeqLength)),

  _rs_lengths_seq(new TruncatedSeq(TruncatedSeqLength)),

  _pause_time_target_ms((double) MaxGCPauseMillis),

  _gcs_are_young(true),

  _young_pause_num(0),

  _mixed_pause_num(0),

  _during_marking(false),

  _in_marking_window(false),

  _in_marking_window_im(false),

  _known_garbage_ratio(0.0),

  _known_garbage_bytes(0),

  _young_gc_eff_seq(new TruncatedSeq(TruncatedSeqLength)),

  _recent_prev_end_times_for_all_gcs_sec(

                                new TruncatedSeq(NumPrevPausesForHeuristics)),

  _recent_avg_pause_time_ratio(0.0),

  _all_full_gc_times_ms(new NumberSeq()),

  _initiate_conc_mark_if_possible(false),

  _during_initial_mark_pause(false),

  _should_revert_to_young_gcs(false),

  _last_young_gc(false),

  _last_gc_was_young(false),

  _eden_bytes_before_gc(0),

  _survivor_bytes_before_gc(0),

  _capacity_before_gc(0),

  _eden_cset_region_length(0),

  _survivor_cset_region_length(0),

  _old_cset_region_length(0),

  _collection_set(NULL),

  _collection_set_bytes_used_before(0),

  // Incremental CSet attributes

  _inc_cset_build_state(Inactive),

  _inc_cset_head(NULL),

  _inc_cset_tail(NULL),

  _inc_cset_bytes_used_before(0),

  _inc_cset_max_finger(NULL),

  _inc_cset_recorded_rs_lengths(0),

  _inc_cset_recorded_rs_lengths_diffs(0),

  _inc_cset_predicted_elapsed_time_ms(0.0),

  _inc_cset_predicted_elapsed_time_ms_diffs(0.0),

#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away

#pragma warning( disable:4355 ) // 'this' : used in base member initializer list

#endif // _MSC_VER

  _short_lived_surv_rate_group(new SurvRateGroup(this, "Short Lived",

                                                 G1YoungSurvRateNumRegionsSummary)),

  _survivor_surv_rate_group(new SurvRateGroup(this, "Survivor",

                                              G1YoungSurvRateNumRegionsSummary)),

  // add here any more surv rate groups

  _recorded_survivor_regions(0),

  _recorded_survivor_head(NULL),

  _recorded_survivor_tail(NULL),

  _survivors_age_table(true),

  _gc_overhead_perc(0.0) {

  // Set up the region size and associated fields. Given that the

  // policy is created before the heap, we have to set this up here,

  // so it's done as soon as possible.

  HeapRegion::setup_heap_region_size(Arguments::min_heap_size());

  HeapRegionRemSet::setup_remset_size();

  G1ErgoVerbose::initialize();

  if (PrintAdaptiveSizePolicy) {

    // Currently, we only use a single switch for all the heuristics.

    G1ErgoVerbose::set_enabled(true);

    // Given that we don't currently have a verboseness level

    // parameter, we'll hardcode this to high. This can be easily

    // changed in the future.

    G1ErgoVerbose::set_level(ErgoHigh);

  } else {

    G1ErgoVerbose::set_enabled(false);

  }

  // Verify PLAB sizes

  const size_t region_size = HeapRegion::GrainWords;

  if (YoungPLABSize > region_size || OldPLABSize > region_size) {

    char buffer[128];

    jio_snprintf(buffer, sizeof(buffer), "%sPLABSize should be at most "SIZE_FORMAT,

                 OldPLABSize > region_size ? "Old" : "Young", region_size);

    vm_exit_during_initialization(buffer);

  }

  _recent_prev_end_times_for_all_gcs_sec->add(os::elapsedTime());

  _prev_collection_pause_end_ms = os::elapsedTime() * 1000.0;

  _par_last_gc_worker_start_times_ms = new double[_parallel_gc_threads];

  _par_last_ext_root_scan_times_ms = new double[_parallel_gc_threads];

  _par_last_satb_filtering_times_ms = new double[_parallel_gc_threads];

  _par_last_update_rs_times_ms = new double[_parallel_gc_threads];

  _par_last_update_rs_processed_buffers = new double[_parallel_gc_threads];

  _par_last_scan_rs_times_ms = new double[_parallel_gc_threads];

  _par_last_obj_copy_times_ms = new double[_parallel_gc_threads];

  _par_last_termination_times_ms = new double[_parallel_gc_threads];

  _par_last_termination_attempts = new double[_parallel_gc_threads];

  _par_last_gc_worker_end_times_ms = new double[_parallel_gc_threads];

  _par_last_gc_worker_times_ms = new double[_parallel_gc_threads];

  _par_last_gc_worker_other_times_ms = new double[_parallel_gc_threads];

  // start conservatively

  _expensive_region_limit_ms = 0.5 * (double) MaxGCPauseMillis;

  int index;

  if (ParallelGCThreads == 0)

    index = 0;

  else if (ParallelGCThreads > 8)

    index = 7;

  else

    index = ParallelGCThreads - 1;

  _pending_card_diff_seq->add(0.0);

  _rs_length_diff_seq->add(rs_length_diff_defaults[index]);

  _cost_per_card_ms_seq->add(cost_per_card_ms_defaults[index]);

  _young_cards_per_entry_ratio_seq->add(

                                  young_cards_per_entry_ratio_defaults[index]);

  _cost_per_entry_ms_seq->add(cost_per_entry_ms_defaults[index]);

  _cost_per_byte_ms_seq->add(cost_per_byte_ms_defaults[index]);

  _constant_other_time_ms_seq->add(constant_other_time_ms_defaults[index]);

  _young_other_cost_per_region_ms_seq->add(

                               young_other_cost_per_region_ms_defaults[index]);

  _non_young_other_cost_per_region_ms_seq->add(

                           non_young_other_cost_per_region_ms_defaults[index]);

  // Below, we might need to calculate the pause time target based on

  // the pause interval. When we do so we are going to give G1 maximum

  // flexibility and allow it to do pauses when it needs to. So, we'll

  // arrange that the pause interval to be pause time target + 1 to

  // ensure that a) the pause time target is maximized with respect to

  // the pause interval and b) we maintain the invariant that pause

  // time target < pause interval. If the user does not want this

  // maximum flexibility, they will have to set the pause interval

  // explicitly.

  // First make sure that, if either parameter is set, its value is

  // reasonable.

  if (!FLAG_IS_DEFAULT(MaxGCPauseMillis)) {

    if (MaxGCPauseMillis < 1) {

      vm_exit_during_initialization("MaxGCPauseMillis should be "

                                    "greater than 0");

    }

  }

  if (!FLAG_IS_DEFAULT(GCPauseIntervalMillis)) {

    if (GCPauseIntervalMillis < 1) {

      vm_exit_during_initialization("GCPauseIntervalMillis should be "

                                    "greater than 0");

    }

  }

  // Then, if the pause time target parameter was not set, set it to

  // the default value.

  if (FLAG_IS_DEFAULT(MaxGCPauseMillis)) {

    if (FLAG_IS_DEFAULT(GCPauseIntervalMillis)) {

      // The default pause time target in G1 is 200ms

      FLAG_SET_DEFAULT(MaxGCPauseMillis, 200);

    } else {

      // We do not allow the pause interval to be set without the

      // pause time target

      vm_exit_during_initialization("GCPauseIntervalMillis cannot be set "

                                    "without setting MaxGCPauseMillis");

    }

  }

  // Then, if the interval parameter was not set, set it according to

  // the pause time target (this will also deal with the case when the

  // pause time target is the default value).

  if (FLAG_IS_DEFAULT(GCPauseIntervalMillis)) {

    FLAG_SET_DEFAULT(GCPauseIntervalMillis, MaxGCPauseMillis + 1);

  }

  // Finally, make sure that the two parameters are consistent.

  if (MaxGCPauseMillis >= GCPauseIntervalMillis) {

    char buffer[256];

    jio_snprintf(buffer, 256,

                 "MaxGCPauseMillis (%u) should be less than "

                 "GCPauseIntervalMillis (%u)",

                 MaxGCPauseMillis, GCPauseIntervalMillis);

    vm_exit_during_initialization(buffer);

  }

  double max_gc_time = (double) MaxGCPauseMillis / 1000.0;

  double time_slice  = (double) GCPauseIntervalMillis / 1000.0;

  _mmu_tracker = new G1MMUTrackerQueue(time_slice, max_gc_time);

  _sigma = (double) G1ConfidencePercent / 100.0;

  // start conservatively (around 50ms is about right)

  _concurrent_mark_remark_times_ms->add(0.05);

  _concurrent_mark_cleanup_times_ms->add(0.20);

  _tenuring_threshold = MaxTenuringThreshold;

  // _max_survivor_regions will be calculated by

  // update_young_list_target_length() during initialization.

  _max_survivor_regions = 0;

  assert(GCTimeRatio > 0,

         "we should have set it to a default value set_g1_gc_flags() "

         "if a user set it to 0");

  _gc_overhead_perc = 100.0 * (1.0 / (1.0 + GCTimeRatio));

  uintx reserve_perc = G1ReservePercent;

  // Put an artificial ceiling on this so that it's not set to a silly value.

  if (reserve_perc > 50) {

    reserve_perc = 50;

    warning("G1ReservePercent is set to a value that is too large, "

            "it's been updated to %u", reserve_perc);

  }

  _reserve_factor = (double) reserve_perc / 100.0;

  // This will be set when the heap is expanded

  // for the first time during initialization.

  _reserve_regions = 0;

  initialize_all();

  _collectionSetChooser = new CollectionSetChooser();

  _young_gen_sizer = new G1YoungGenSizer(); // Must be after call to initialize_flags

}

void G1CollectorPolicy::initialize_flags() {

  set_min_alignment(HeapRegion::GrainBytes);

  set_max_alignment(GenRemSet::max_alignment_constraint(rem_set_name()));

  if (SurvivorRatio < 1) {

    vm_exit_during_initialization("Invalid survivor ratio specified");

  }

  CollectorPolicy::initialize_flags();

}

G1YoungGenSizer::G1YoungGenSizer() : _sizer_kind(SizerDefaults), _adaptive_size(true) {

  assert(G1DefaultMinNewGenPercent <= G1DefaultMaxNewGenPercent, "Min larger than max");

  assert(G1DefaultMinNewGenPercent > 0 && G1DefaultMinNewGenPercent < 100, "Min out of bounds");

  assert(G1DefaultMaxNewGenPercent > 0 && G1DefaultMaxNewGenPercent < 100, "Max out of bounds");

  if (FLAG_IS_CMDLINE(NewRatio)) {

    if (FLAG_IS_CMDLINE(NewSize) || FLAG_IS_CMDLINE(MaxNewSize)) {

      warning("-XX:NewSize and -XX:MaxNewSize override -XX:NewRatio");

    } else {

      _sizer_kind = SizerNewRatio;

      _adaptive_size = false;

      return;

    }

  }

  if (FLAG_IS_CMDLINE(NewSize)) {

     _min_desired_young_length = MAX2((size_t) 1, NewSize / HeapRegion::GrainBytes);

    if (FLAG_IS_CMDLINE(MaxNewSize)) {

      _max_desired_young_length = MAX2((size_t) 1, MaxNewSize / HeapRegion::GrainBytes);

      _sizer_kind = SizerMaxAndNewSize;

      _adaptive_size = _min_desired_young_length == _max_desired_young_length;

    } else {

      _sizer_kind = SizerNewSizeOnly;

    }

  } else if (FLAG_IS_CMDLINE(MaxNewSize)) {

    _max_desired_young_length = MAX2((size_t) 1, MaxNewSize / HeapRegion::GrainBytes);

    _sizer_kind = SizerMaxNewSizeOnly;

  }

}

size_t G1YoungGenSizer::calculate_default_min_length(size_t new_number_of_heap_regions) {

  size_t default_value = (new_number_of_heap_regions * G1DefaultMinNewGenPercent) / 100;

  return MAX2((size_t)1, default_value);

}

size_t G1YoungGenSizer::calculate_default_max_length(size_t new_number_of_heap_regions) {

  size_t default_value = (new_number_of_heap_regions * G1DefaultMaxNewGenPercent) / 100;

  return MAX2((size_t)1, default_value);

}

void G1YoungGenSizer::heap_size_changed(size_t new_number_of_heap_regions) {

  assert(new_number_of_heap_regions > 0, "Heap must be initialized");

  switch (_sizer_kind) {

    case SizerDefaults:

      _min_desired_young_length = calculate_default_min_length(new_number_of_heap_regions);

      _max_desired_young_length = calculate_default_max_length(new_number_of_heap_regions);

      break;

    case SizerNewSizeOnly:

      _max_desired_young_length = calculate_default_max_length(new_number_of_heap_regions);

      _max_desired_young_length = MAX2(_min_desired_young_length, _max_desired_young_length);

      break;

    case SizerMaxNewSizeOnly:

      _min_desired_young_length = calculate_default_min_length(new_number_of_heap_regions);

      _min_desired_young_length = MIN2(_min_desired_young_length, _max_desired_young_length);

      break;

    case SizerMaxAndNewSize:

      // Do nothing. Values set on the command line, don't update them at runtime.

      break;

    case SizerNewRatio:

      _min_desired_young_length = new_number_of_heap_regions / (NewRatio + 1);

      _max_desired_young_length = _min_desired_young_length;

      break;

    default:

      ShouldNotReachHere();

  }

  assert(_min_desired_young_length <= _max_desired_young_length, "Invalid min/max young gen size values");

}

void G1CollectorPolicy::init() {

  // Set aside an initial future to_space.

  _g1 = G1CollectedHeap::heap();

  assert(Heap_lock->owned_by_self(), "Locking discipline.");

  initialize_gc_policy_counters();

  if (adaptive_young_list_length()) {