/*

 * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "gc_implementation/parallelScavenge/adjoiningGenerations.hpp"

#include "gc_implementation/parallelScavenge/adjoiningVirtualSpaces.hpp"

#include "gc_implementation/parallelScavenge/cardTableExtension.hpp"

#include "gc_implementation/parallelScavenge/gcTaskManager.hpp"

#include "gc_implementation/parallelScavenge/generationSizer.hpp"

#include "gc_implementation/parallelScavenge/parallelScavengeHeap.inline.hpp"

#include "gc_implementation/parallelScavenge/psAdaptiveSizePolicy.hpp"

#include "gc_implementation/parallelScavenge/psMarkSweep.hpp"

#include "gc_implementation/parallelScavenge/psParallelCompact.hpp"

#include "gc_implementation/parallelScavenge/psPromotionManager.hpp"

#include "gc_implementation/parallelScavenge/psScavenge.hpp"

#include "gc_implementation/parallelScavenge/vmPSOperations.hpp"

#include "memory/gcLocker.inline.hpp"

#include "oops/oop.inline.hpp"

#include "runtime/handles.inline.hpp"

#include "runtime/java.hpp"

#include "runtime/vmThread.hpp"

#include "utilities/vmError.hpp"

PSYoungGen*  ParallelScavengeHeap::_young_gen = NULL;

PSOldGen*    ParallelScavengeHeap::_old_gen = NULL;

PSPermGen*   ParallelScavengeHeap::_perm_gen = NULL;

PSAdaptiveSizePolicy* ParallelScavengeHeap::_size_policy = NULL;

PSGCAdaptivePolicyCounters* ParallelScavengeHeap::_gc_policy_counters = NULL;

ParallelScavengeHeap* ParallelScavengeHeap::_psh = NULL;

GCTaskManager* ParallelScavengeHeap::_gc_task_manager = NULL;

static void trace_gen_sizes(const char* const str,

                            size_t pg_min, size_t pg_max,

                            size_t og_min, size_t og_max,

                            size_t yg_min, size_t yg_max)

{

  if (TracePageSizes) {

    tty->print_cr("%s:  " SIZE_FORMAT "," SIZE_FORMAT " "

                  SIZE_FORMAT "," SIZE_FORMAT " "

                  SIZE_FORMAT "," SIZE_FORMAT " "

                  SIZE_FORMAT,

                  str, pg_min / K, pg_max / K,

                  og_min / K, og_max / K,

                  yg_min / K, yg_max / K,

                  (pg_max + og_max + yg_max) / K);

  }

}

jint ParallelScavengeHeap::initialize() {

  CollectedHeap::pre_initialize();

  // Cannot be initialized until after the flags are parsed

  // GenerationSizer flag_parser;

  _collector_policy = new GenerationSizer();

  size_t yg_min_size = _collector_policy->min_young_gen_size();

  size_t yg_max_size = _collector_policy->max_young_gen_size();

  size_t og_min_size = _collector_policy->min_old_gen_size();

  size_t og_max_size = _collector_policy->max_old_gen_size();

  // Why isn't there a min_perm_gen_size()?

  size_t pg_min_size = _collector_policy->perm_gen_size();

  size_t pg_max_size = _collector_policy->max_perm_gen_size();

  trace_gen_sizes("ps heap raw",

                  pg_min_size, pg_max_size,

                  og_min_size, og_max_size,

                  yg_min_size, yg_max_size);

  // The ReservedSpace ctor used below requires that the page size for the perm

  // gen is <= the page size for the rest of the heap (young + old gens).

  const size_t og_page_sz = os::page_size_for_region(yg_min_size + og_min_size,

                                                     yg_max_size + og_max_size,

                                                     8);

  const size_t pg_page_sz = MIN2(os::page_size_for_region(pg_min_size,

                                                          pg_max_size, 16),

                                 og_page_sz);

  const size_t pg_align = set_alignment(_perm_gen_alignment,  pg_page_sz);

  const size_t og_align = set_alignment(_old_gen_alignment,   og_page_sz);

  const size_t yg_align = set_alignment(_young_gen_alignment, og_page_sz);

  // Update sizes to reflect the selected page size(s).

  //

  // NEEDS_CLEANUP.  The default TwoGenerationCollectorPolicy uses NewRatio; it

  // should check UseAdaptiveSizePolicy.  Changes from generationSizer could

  // move to the common code.

  yg_min_size = align_size_up(yg_min_size, yg_align);

  yg_max_size = align_size_up(yg_max_size, yg_align);

  size_t yg_cur_size =

    align_size_up(_collector_policy->young_gen_size(), yg_align);

  yg_cur_size = MAX2(yg_cur_size, yg_min_size);

  og_min_size = align_size_up(og_min_size, og_align);

  // Align old gen size down to preserve specified heap size.

  assert(og_align == yg_align, "sanity");

  og_max_size = align_size_down(og_max_size, og_align);

  og_max_size = MAX2(og_max_size, og_min_size);

  size_t og_cur_size =

    align_size_down(_collector_policy->old_gen_size(), og_align);

  og_cur_size = MAX2(og_cur_size, og_min_size);

  pg_min_size = align_size_up(pg_min_size, pg_align);

  pg_max_size = align_size_up(pg_max_size, pg_align);

  size_t pg_cur_size = pg_min_size;

  trace_gen_sizes("ps heap rnd",

                  pg_min_size, pg_max_size,

                  og_min_size, og_max_size,

                  yg_min_size, yg_max_size);

  const size_t total_reserved = pg_max_size + og_max_size + yg_max_size;

  char* addr = Universe::preferred_heap_base(total_reserved, Universe::UnscaledNarrowOop);

  // The main part of the heap (old gen + young gen) can often use a larger page

  // size than is needed or wanted for the perm gen.  Use the "compound

  // alignment" ReservedSpace ctor to avoid having to use the same page size for

  // all gens.

  ReservedHeapSpace heap_rs(pg_max_size, pg_align, og_max_size + yg_max_size,

                            og_align, addr);

  if (UseCompressedOops) {

    if (addr != NULL && !heap_rs.is_reserved()) {

      // Failed to reserve at specified address - the requested memory

      // region is taken already, for example, by 'java' launcher.

      // Try again to reserver heap higher.

      addr = Universe::preferred_heap_base(total_reserved, Universe::ZeroBasedNarrowOop);

      ReservedHeapSpace heap_rs0(pg_max_size, pg_align, og_max_size + yg_max_size,

                                 og_align, addr);

      if (addr != NULL && !heap_rs0.is_reserved()) {

        // Failed to reserve at specified address again - give up.

        addr = Universe::preferred_heap_base(total_reserved, Universe::HeapBasedNarrowOop);

        assert(addr == NULL, "");

        ReservedHeapSpace heap_rs1(pg_max_size, pg_align, og_max_size + yg_max_size,

                                   og_align, addr);

        heap_rs = heap_rs1;

      } else {

        heap_rs = heap_rs0;

      }

    }

  }

  os::trace_page_sizes("ps perm", pg_min_size, pg_max_size, pg_page_sz,

                       heap_rs.base(), pg_max_size);

  os::trace_page_sizes("ps main", og_min_size + yg_min_size,

                       og_max_size + yg_max_size, og_page_sz,

                       heap_rs.base() + pg_max_size,

                       heap_rs.size() - pg_max_size);

  if (!heap_rs.is_reserved()) {

    vm_shutdown_during_initialization(

      "Could not reserve enough space for object heap");

    return JNI_ENOMEM;

  }

  _reserved = MemRegion((HeapWord*)heap_rs.base(),

                        (HeapWord*)(heap_rs.base() + heap_rs.size()));

  CardTableExtension* const barrier_set = new CardTableExtension(_reserved, 3);

  _barrier_set = barrier_set;

  oopDesc::set_bs(_barrier_set);

  if (_barrier_set == NULL) {

    vm_shutdown_during_initialization(

      "Could not reserve enough space for barrier set");

    return JNI_ENOMEM;

  }

  // Initial young gen size is 4 Mb

  //

  // XXX - what about flag_parser.young_gen_size()?

  const size_t init_young_size = align_size_up(4 * M, yg_align);

  yg_cur_size = MAX2(MIN2(init_young_size, yg_max_size), yg_cur_size);

  // Split the reserved space into perm gen and the main heap (everything else).

  // The main heap uses a different alignment.

  ReservedSpace perm_rs = heap_rs.first_part(pg_max_size);

  ReservedSpace main_rs = heap_rs.last_part(pg_max_size, og_align);

  // Make up the generations

  // Calculate the maximum size that a generation can grow.  This

  // includes growth into the other generation.  Note that the

  // parameter _max_gen_size is kept as the maximum

  // size of the generation as the boundaries currently stand.

  // _max_gen_size is still used as that value.

  double max_gc_pause_sec = ((double) MaxGCPauseMillis)/1000.0;

  double max_gc_minor_pause_sec = ((double) MaxGCMinorPauseMillis)/1000.0;

  _gens = new AdjoiningGenerations(main_rs,

                                   og_cur_size,

                                   og_min_size,

                                   og_max_size,

                                   yg_cur_size,

                                   yg_min_size,

                                   yg_max_size,

                                   yg_align);

  _old_gen = _gens->old_gen();

  _young_gen = _gens->young_gen();

  const size_t eden_capacity = _young_gen->eden_space()->capacity_in_bytes();

  const size_t old_capacity = _old_gen->capacity_in_bytes();

  const size_t initial_promo_size = MIN2(eden_capacity, old_capacity);

  _size_policy =

    new PSAdaptiveSizePolicy(eden_capacity,

                             initial_promo_size,

                             young_gen()->to_space()->capacity_in_bytes(),

                             intra_heap_alignment(),

                             max_gc_pause_sec,

                             max_gc_minor_pause_sec,

                             GCTimeRatio

                             );

  _perm_gen = new PSPermGen(perm_rs,

                            pg_align,

                            pg_cur_size,

                            pg_cur_size,

                            pg_max_size,

                            "perm", 2);

  assert(!UseAdaptiveGCBoundary ||

    (old_gen()->virtual_space()->high_boundary() ==

     young_gen()->virtual_space()->low_boundary()),

    "Boundaries must meet");

  // initialize the policy counters - 2 collectors, 3 generations

  _gc_policy_counters =

    new PSGCAdaptivePolicyCounters("ParScav:MSC", 2, 3, _size_policy);

  _psh = this;

  // Set up the GCTaskManager

  _gc_task_manager = GCTaskManager::create(ParallelGCThreads);

  if (UseParallelOldGC && !PSParallelCompact::initialize()) {

    return JNI_ENOMEM;

  }

  return JNI_OK;

}

void ParallelScavengeHeap::post_initialize() {

  // Need to init the tenuring threshold

  PSScavenge::initialize();

  if (UseParallelOldGC) {

    PSParallelCompact::post_initialize();

  } else {

    PSMarkSweep::initialize();

  }

  PSPromotionManager::initialize();

}

void ParallelScavengeHeap::update_counters() {

  young_gen()->update_counters();

  old_gen()->update_counters();

  perm_gen()->update_counters();

}

size_t ParallelScavengeHeap::capacity() const {

  size_t value = young_gen()->capacity_in_bytes() + old_gen()->capacity_in_bytes();

  return value;

}

size_t ParallelScavengeHeap::used() const {

  size_t value = young_gen()->used_in_bytes() + old_gen()->used_in_bytes();

  return value;

}

bool ParallelScavengeHeap::is_maximal_no_gc() const {

  return old_gen()->is_maximal_no_gc() && young_gen()->is_maximal_no_gc();

}

size_t ParallelScavengeHeap::permanent_capacity() const {

  return perm_gen()->capacity_in_bytes();

}

size_t ParallelScavengeHeap::permanent_used() const {

  return perm_gen()->used_in_bytes();

}

size_t ParallelScavengeHeap::max_capacity() const {

  size_t estimated = reserved_region().byte_size();

  estimated -= perm_gen()->reserved().byte_size();

  if (UseAdaptiveSizePolicy) {

    estimated -= _size_policy->max_survivor_size(young_gen()->max_size());

  } else {

    estimated -= young_gen()->to_space()->capacity_in_bytes();

  }

  return MAX2(estimated, capacity());

}

bool ParallelScavengeHeap::is_in(const void* p) const {

  if (young_gen()->is_in(p)) {

    return true;

  }

  if (old_gen()->is_in(p)) {

    return true;

  }

  if (perm_gen()->is_in(p)) {

    return true;

  }

  return false;

}

bool ParallelScavengeHeap::is_in_reserved(const void* p) const {

  if (young_gen()->is_in_reserved(p)) {

    return true;

  }

  if (old_gen()->is_in_reserved(p)) {

    return true;

  }

  if (perm_gen()->is_in_reserved(p)) {

    return true;

  }

  return false;

}

bool ParallelScavengeHeap::is_scavengable(const void* addr) {

  return is_in_young((oop)addr);

}

#ifdef ASSERT

// Don't implement this by using is_in_young().  This method is used

// in some cases to check that is_in_young() is correct.

bool ParallelScavengeHeap::is_in_partial_collection(const void *p) {

  assert(is_in_reserved(p) || p == NULL,

    "Does not work if address is non-null and outside of the heap");

  // The order of the generations is perm (low addr), old, young (high addr)

  return p >= old_gen()->reserved().end();

}

#endif

// There are two levels of allocation policy here.

//

// When an allocation request fails, the requesting thread must invoke a VM

// operation, transfer control to the VM thread, and await the results of a

// garbage collection. That is quite expensive, and we should avoid doing it

// multiple times if possible.

//

// To accomplish this, we have a basic allocation policy, and also a

// failed allocation policy.

//

// The basic allocation policy controls how you allocate memory without

// attempting garbage collection. It is okay to grab locks and

// expand the heap, if that can be done without coming to a safepoint.

// It is likely that the basic allocation policy will not be very

// aggressive.

//

// The failed allocation policy is invoked from the VM thread after

// the basic allocation policy is unable to satisfy a mem_allocate

// request. This policy needs to cover the entire range of collection,

// heap expansion, and out-of-memory conditions. It should make every

// attempt to allocate the requested memory.

// Basic allocation policy. Should never be called at a safepoint, or

// from the VM thread.

//

// This method must handle cases where many mem_allocate requests fail

// simultaneously. When that happens, only one VM operation will succeed,

// and the rest will not be executed. For that reason, this method loops

// during failed allocation attempts. If the java heap becomes exhausted,

// we rely on the size_policy object to force a bail out.

HeapWord* ParallelScavengeHeap::mem_allocate(

                                     size_t size,

                                     bool* gc_overhead_limit_was_exceeded) {

  assert(!SafepointSynchronize::is_at_safepoint(), "should not be at safepoint");

  assert(Thread::current() != (Thread*)VMThread::vm_thread(), "should not be in vm thread");

  assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock");

  // In general gc_overhead_limit_was_exceeded should be false so

  // set it so here and reset it to true only if the gc time

  // limit is being exceeded as checked below.

  *gc_overhead_limit_was_exceeded = false;

  uint loop_count = 0;

  uint gc_count = 0;

  while (result == NULL) {

    // We don't want to have multiple collections for a single filled generation.

    // To prevent this, each thread tracks the total_collections() value, and if

    // the count has changed, does not do a new collection.

    //

    // The collection count must be read only while holding the heap lock. VM

    // operations also hold the heap lock during collections. There is a lock

    // contention case where thread A blocks waiting on the Heap_lock, while

    // thread B is holding it doing a collection. When thread A gets the lock,

    // the collection count has already changed. To prevent duplicate collections,

    // The policy MUST attempt allocations during the same period it reads the

    // total_collections() value!

    {

      MutexLocker ml(Heap_lock);

      gc_count = Universe::heap()->total_collections();

      // (1) If the requested object is too large to easily fit in the

      //     young_gen, or

      // (2) If GC is locked out via GCLocker, young gen is full and

      //     the need for a GC already signalled to GCLocker (done

      //     at a safepoint),

      // ... then, rather than force a safepoint and (a potentially futile)

      // collection (attempt) for each allocation, try allocation directly

      // in old_gen. For case (2) above, we may in the future allow

      // TLAB allocation directly in the old gen.

      if (result != NULL) {

        return result;

      }

        if (result != NULL) {

          return result;

        }

      }

      if (GC_locker::is_active_and_needs_gc()) {

        // If this thread is not in a jni critical section, we stall

        // the requestor until the critical section has cleared and

        // GC allowed. When the critical section clears, a GC is

        // initiated by the last thread exiting the critical section; so

        // we retry the allocation sequence from the beginning of the loop,

        // rather than causing more, now probably unnecessary, GC attempts.

        JavaThread* jthr = JavaThread::current();

        if (!jthr->in_critical()) {

          MutexUnlocker mul(Heap_lock);

          GC_locker::stall_until_clear();

          continue;

        } else {

          if (CheckJNICalls) {

            fatal("Possible deadlock due to allocating while"

                  " in jni critical section");

          }

          return NULL;

        }

      }

    }

    if (result == NULL) {

      // Generate a VM operation

      VMThread::execute(&op);

      // Did the VM operation execute? If so, return the result directly.

      // This prevents us from looping until time out on requests that can

      // not be satisfied.

      if (op.prologue_succeeded()) {

        assert(Universe::heap()->is_in_or_null(op.result()),

          "result not in heap");

        // If GC was locked out during VM operation then retry allocation

        // and/or stall as necessary.

        if (op.gc_locked()) {

          assert(op.result() == NULL, "must be NULL if gc_locked() is true");

          continue;  // retry and/or stall as necessary

        }

        // Exit the loop if the gc time limit has been exceeded.

        // The allocation must have failed above ("result" guarding

        // this path is NULL) and the most recent collection has exceeded the

        // gc overhead limit (although enough may have been collected to

        // satisfy the allocation).  Exit the loop so that an out-of-memory

        // will be thrown (return a NULL ignoring the contents of

        // op.result()),

        // but clear gc_overhead_limit_exceeded so that the next collection

        // starts with a clean slate (i.e., forgets about previous overhead

        // excesses).  Fill op.result() with a filler object so that the

        // heap remains parsable.

        const bool limit_exceeded = size_policy()->gc_overhead_limit_exceeded();

        const bool softrefs_clear = collector_policy()->all_soft_refs_clear();

        assert(!limit_exceeded || softrefs_clear, "Should have been cleared");

        if (limit_exceeded && softrefs_clear) {

          *gc_overhead_limit_was_exceeded = true;

          size_policy()->set_gc_overhead_limit_exceeded(false);

          if (PrintGCDetails && Verbose) {

            gclog_or_tty->print_cr("ParallelScavengeHeap::mem_allocate: "

              "return NULL because gc_overhead_limit_exceeded is set");

          }

          if (op.result() != NULL) {

            CollectedHeap::fill_with_object(op.result(), size);

          }

          return NULL;

        }

        return op.result();

      }

    }

    // The policy object will prevent us from looping forever. If the

    // time spent in gc crosses a threshold, we will bail out.

    loop_count++;

    if ((result == NULL) && (QueuedAllocationWarningCount > 0) &&

        (loop_count % QueuedAllocationWarningCount == 0)) {