/*

 * Copyright (c) 2002, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "aot/aotLoader.hpp"

#include "classfile/classLoaderDataGraph.hpp"

#include "classfile/stringTable.hpp"

#include "code/codeCache.hpp"

#include "gc/parallel/parallelScavengeHeap.hpp"

#include "gc/parallel/psAdaptiveSizePolicy.hpp"

#include "gc/parallel/psClosure.inline.hpp"

#include "gc/parallel/psCompactionManager.hpp"

#include "gc/parallel/psMarkSweepProxy.hpp"

#include "gc/parallel/psParallelCompact.inline.hpp"

#include "gc/parallel/psPromotionManager.inline.hpp"

#include "gc/parallel/psRootType.hpp"

#include "gc/parallel/psScavenge.inline.hpp"

#include "gc/shared/gcCause.hpp"

#include "gc/shared/gcHeapSummary.hpp"

#include "gc/shared/gcId.hpp"

#include "gc/shared/gcLocker.hpp"

#include "gc/shared/gcTimer.hpp"

#include "gc/shared/gcTrace.hpp"

#include "gc/shared/gcTraceTime.inline.hpp"

#include "gc/shared/isGCActiveMark.hpp"

#include "gc/shared/referencePolicy.hpp"

#include "gc/shared/referenceProcessor.hpp"

#include "gc/shared/referenceProcessorPhaseTimes.hpp"

#include "gc/shared/scavengableNMethods.hpp"

#include "gc/shared/spaceDecorator.hpp"

#include "gc/shared/weakProcessor.hpp"

#include "gc/shared/workerPolicy.hpp"

#include "gc/shared/workgroup.hpp"

#include "memory/resourceArea.hpp"

#include "memory/universe.hpp"

#include "logging/log.hpp"

#include "oops/access.inline.hpp"

#include "oops/compressedOops.inline.hpp"

#include "oops/oop.inline.hpp"

#include "runtime/biasedLocking.hpp"

#include "runtime/handles.inline.hpp"

#include "runtime/threadCritical.hpp"

#include "runtime/vmThread.hpp"

#include "runtime/vmOperations.hpp"

#include "services/management.hpp"

#include "services/memoryService.hpp"

#include "utilities/stack.inline.hpp"

HeapWord*                     PSScavenge::_to_space_top_before_gc = NULL;

int                           PSScavenge::_consecutive_skipped_scavenges = 0;

SpanSubjectToDiscoveryClosure PSScavenge::_span_based_discoverer;

ReferenceProcessor*           PSScavenge::_ref_processor = NULL;

PSCardTable*                  PSScavenge::_card_table = NULL;

bool                          PSScavenge::_survivor_overflow = false;

uint                          PSScavenge::_tenuring_threshold = 0;

HeapWord*                     PSScavenge::_young_generation_boundary = NULL;

uintptr_t                     PSScavenge::_young_generation_boundary_compressed = 0;

elapsedTimer                  PSScavenge::_accumulated_time;

STWGCTimer                    PSScavenge::_gc_timer;

ParallelScavengeTracer        PSScavenge::_gc_tracer;

CollectorCounters*            PSScavenge::_counters = NULL;

static void scavenge_roots_work(ParallelRootType::Value root_type, uint worker_id) {

  assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc");

  PSPromotionManager* pm = PSPromotionManager::gc_thread_promotion_manager(worker_id);

  PSScavengeRootsClosure roots_closure(pm);

  PSPromoteRootsClosure  roots_to_old_closure(pm);

  switch (root_type) {

    case ParallelRootType::universe:

      Universe::oops_do(&roots_closure);

      break;

    case ParallelRootType::jni_handles:

      JNIHandles::oops_do(&roots_closure);

      break;

    case ParallelRootType::object_synchronizer:

      ObjectSynchronizer::oops_do(&roots_closure);

      break;

    case ParallelRootType::system_dictionary:

      SystemDictionary::oops_do(&roots_closure);

      break;

    case ParallelRootType::class_loader_data:

      {

        PSScavengeCLDClosure cld_closure(pm);

        ClassLoaderDataGraph::cld_do(&cld_closure);

      }

      break;

    case ParallelRootType::management:

      Management::oops_do(&roots_closure);

      break;

    case ParallelRootType::jvmti:

      JvmtiExport::oops_do(&roots_closure);

      break;

    case ParallelRootType::code_cache:

      {

        MarkingCodeBlobClosure code_closure(&roots_to_old_closure, CodeBlobToOopClosure::FixRelocations);

        ScavengableNMethods::nmethods_do(&code_closure);

        AOTLoader::oops_do(&roots_closure);

      }

      break;

    case ParallelRootType::sentinel:

    DEBUG_ONLY(default:) // DEBUG_ONLY hack will create compile error on release builds (-Wswitch) and runtime check on debug builds

      fatal("Bad enumeration value: %u", root_type);

      break;

  }

  // Do the real work

  pm->drain_stacks(false);

}

static void steal_work(ParallelTaskTerminator& terminator, uint worker_id) {

  assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc");

  PSPromotionManager* pm =

    PSPromotionManager::gc_thread_promotion_manager(worker_id);

  pm->drain_stacks(true);

  guarantee(pm->stacks_empty(),

            "stacks should be empty at this point");

  while (true) {

    StarTask p;

    if (PSPromotionManager::steal_depth(worker_id, p)) {

      TASKQUEUE_STATS_ONLY(pm->record_steal(p));

      pm->process_popped_location_depth(p);

      pm->drain_stacks_depth(true);

    } else {

      if (terminator.offer_termination()) {

        break;

      }

    }

  }

  guarantee(pm->stacks_empty(), "stacks should be empty at this point");

}

// Define before use

class PSIsAliveClosure: public BoolObjectClosure {

public:

  bool do_object_b(oop p) {

    return (!PSScavenge::is_obj_in_young(p)) || p->is_forwarded();

  }

};

PSIsAliveClosure PSScavenge::_is_alive_closure;

class PSKeepAliveClosure: public OopClosure {

protected:

  MutableSpace* _to_space;

  PSPromotionManager* _promotion_manager;

public:

  PSKeepAliveClosure(PSPromotionManager* pm) : _promotion_manager(pm) {

    ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();

    _to_space = heap->young_gen()->to_space();

    assert(_promotion_manager != NULL, "Sanity");

  }

  template <class T> void do_oop_work(T* p) {

    assert (oopDesc::is_oop(RawAccess<IS_NOT_NULL>::oop_load(p)),

            "expected an oop while scanning weak refs");

    // Weak refs may be visited more than once.

    if (PSScavenge::should_scavenge(p, _to_space)) {

      _promotion_manager->copy_and_push_safe_barrier<T, /*promote_immediately=*/false>(p);

    }

  }

  virtual void do_oop(oop* p)       { PSKeepAliveClosure::do_oop_work(p); }

  virtual void do_oop(narrowOop* p) { PSKeepAliveClosure::do_oop_work(p); }

};

class PSEvacuateFollowersClosure: public VoidClosure {

 private:

  PSPromotionManager* _promotion_manager;

 public:

  PSEvacuateFollowersClosure(PSPromotionManager* pm) : _promotion_manager(pm) {}

  virtual void do_void() {

    assert(_promotion_manager != NULL, "Sanity");

    _promotion_manager->drain_stacks(true);

    guarantee(_promotion_manager->stacks_empty(),

              "stacks should be empty at this point");

  }

};

class PSRefProcTaskExecutor: public AbstractRefProcTaskExecutor {

  virtual void execute(ProcessTask& process_task, uint ergo_workers);

};

class PSRefProcTask : public AbstractGangTask {

  typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;

  TaskTerminator _terminator;

  ProcessTask& _task;

  uint _active_workers;

public:

  PSRefProcTask(ProcessTask& task, uint active_workers)

    : AbstractGangTask("PSRefProcTask"),

      _terminator(active_workers, PSPromotionManager::stack_array_depth()),

      _task(task),

      _active_workers(active_workers) {

  }

  virtual void work(uint worker_id) {

    PSPromotionManager* promotion_manager =

      PSPromotionManager::gc_thread_promotion_manager(worker_id);

    assert(promotion_manager != NULL, "sanity check");

    PSKeepAliveClosure keep_alive(promotion_manager);

    PSEvacuateFollowersClosure evac_followers(promotion_manager);

    PSIsAliveClosure is_alive;

    _task.work(worker_id, is_alive, keep_alive, evac_followers);

    if (_task.marks_oops_alive() && _active_workers > 1) {

      steal_work(*_terminator.terminator(), worker_id);

    }

  }

};

void PSRefProcTaskExecutor::execute(ProcessTask& process_task, uint ergo_workers) {

  PSRefProcTask task(process_task, ergo_workers);

  ParallelScavengeHeap::heap()->workers().run_task(&task);

}

// This method contains all heap specific policy for invoking scavenge.

// PSScavenge::invoke_no_policy() will do nothing but attempt to

// scavenge. It will not clean up after failed promotions, bail out if

// we've exceeded policy time limits, or any other special behavior.

// All such policy should be placed here.

//

// Note that this method should only be called from the vm_thread while

// at a safepoint!

bool PSScavenge::invoke() {

  assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");

  assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread");

  assert(!ParallelScavengeHeap::heap()->is_gc_active(), "not reentrant");

  ParallelScavengeHeap* const heap = ParallelScavengeHeap::heap();

  PSAdaptiveSizePolicy* policy = heap->size_policy();

  IsGCActiveMark mark;

  const bool scavenge_done = PSScavenge::invoke_no_policy();

  const bool need_full_gc = !scavenge_done ||

    policy->should_full_GC(heap->old_gen()->free_in_bytes());

  bool full_gc_done = false;

  if (UsePerfData) {

    PSGCAdaptivePolicyCounters* const counters = heap->gc_policy_counters();

    const int ffs_val = need_full_gc ? full_follows_scavenge : not_skipped;

    counters->update_full_follows_scavenge(ffs_val);

  }

  if (need_full_gc) {

    GCCauseSetter gccs(heap, GCCause::_adaptive_size_policy);

    SoftRefPolicy* srp = heap->soft_ref_policy();

    const bool clear_all_softrefs = srp->should_clear_all_soft_refs();

    if (UseParallelOldGC) {

      full_gc_done = PSParallelCompact::invoke_no_policy(clear_all_softrefs);

    } else {

      full_gc_done = PSMarkSweepProxy::invoke_no_policy(clear_all_softrefs);

    }

  }

  return full_gc_done;

}

class PSThreadRootsTaskClosure : public ThreadClosure {

  uint _worker_id;

public:

  PSThreadRootsTaskClosure(uint worker_id) : _worker_id(worker_id) { }

  virtual void do_thread(Thread* thread) {

    assert(ParallelScavengeHeap::heap()->is_gc_active(), "called outside gc");

    PSPromotionManager* pm = PSPromotionManager::gc_thread_promotion_manager(_worker_id);

    PSScavengeRootsClosure roots_closure(pm);

    MarkingCodeBlobClosure roots_in_blobs(&roots_closure, CodeBlobToOopClosure::FixRelocations);

    thread->oops_do(&roots_closure, &roots_in_blobs);

    // Do the real work

    pm->drain_stacks(false);

  }

};

class ScavengeRootsTask : public AbstractGangTask {

  StrongRootsScope _strong_roots_scope; // needed for Threads::possibly_parallel_threads_do

  SequentialSubTasksDone _subtasks;

  PSOldGen* _old_gen;

  HeapWord* _gen_top;

  uint _active_workers;

  bool _is_empty;

  TaskTerminator _terminator;

public:

  ScavengeRootsTask(PSOldGen* old_gen,

                    HeapWord* gen_top,

                    uint active_workers,

                    bool is_empty) :

      AbstractGangTask("ScavengeRootsTask"),

      _strong_roots_scope(active_workers),

      _subtasks(),

      _old_gen(old_gen),

      _gen_top(gen_top),

      _active_workers(active_workers),

      _is_empty(is_empty),

      _terminator(active_workers, PSPromotionManager::vm_thread_promotion_manager()->stack_array_depth()) {

    _subtasks.set_n_threads(active_workers);

    _subtasks.set_n_tasks(ParallelRootType::sentinel);

  }

  virtual void work(uint worker_id) {

    ResourceMark rm;

    if (!_is_empty) {

      // There are only old-to-young pointers if there are objects

      // in the old gen.

      assert(_old_gen != NULL, "Sanity");

      // There are no old-to-young pointers if the old gen is empty.

      assert(!_old_gen->object_space()->is_empty(), "Should not be called is there is no work");

      assert(_old_gen->object_space()->contains(_gen_top) || _gen_top == _old_gen->object_space()->top(), "Sanity");

      assert(worker_id < ParallelGCThreads, "Sanity");

      {

        PSPromotionManager* pm = PSPromotionManager::gc_thread_promotion_manager(worker_id);

        PSCardTable* card_table = ParallelScavengeHeap::heap()->card_table();

        card_table->scavenge_contents_parallel(_old_gen->start_array(),

                                               _old_gen->object_space(),

                                               _gen_top,

                                               pm,

                                               worker_id,

                                               _active_workers);

        // Do the real work

        pm->drain_stacks(false);

      }

    }

    for (uint root_type = 0; _subtasks.try_claim_task(root_type); /* empty */ ) {

      scavenge_roots_work(static_cast<ParallelRootType::Value>(root_type), worker_id);

    }

    _subtasks.all_tasks_completed();

    PSThreadRootsTaskClosure closure(worker_id);

    Threads::possibly_parallel_threads_do(true /*parallel */, &closure);

    // If active_workers can exceed 1, add a steal_work().

    // PSPromotionManager::drain_stacks_depth() does not fully drain its

    // stacks and expects a steal_work() to complete the draining if

    // ParallelGCThreads is > 1.

    if (_active_workers > 1) {

      steal_work(*_terminator.terminator() , worker_id);

    }

  }

};

// This method contains no policy. You should probably

// be calling invoke() instead.

bool PSScavenge::invoke_no_policy() {

  assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");

  assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread");

  _gc_timer.register_gc_start();

  TimeStamp scavenge_entry;

  TimeStamp scavenge_midpoint;

  TimeStamp scavenge_exit;

  scavenge_entry.update();

  if (GCLocker::check_active_before_gc()) {

    return false;

  }

  ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();

  GCCause::Cause gc_cause = heap->gc_cause();

  // Check for potential problems.

  if (!should_attempt_scavenge()) {

    return false;

  }

  GCIdMark gc_id_mark;

  _gc_tracer.report_gc_start(heap->gc_cause(), _gc_timer.gc_start());

  bool promotion_failure_occurred = false;

  PSYoungGen* young_gen = heap->young_gen();

  PSOldGen* old_gen = heap->old_gen();

  PSAdaptiveSizePolicy* size_policy = heap->size_policy();

  heap->increment_total_collections();

  if (AdaptiveSizePolicy::should_update_eden_stats(gc_cause)) {

    // Gather the feedback data for eden occupancy.

    young_gen->eden_space()->accumulate_statistics();

  }

  heap->print_heap_before_gc();

  heap->trace_heap_before_gc(&_gc_tracer);

  assert(!NeverTenure || _tenuring_threshold == markWord::max_age + 1, "Sanity");

  assert(!AlwaysTenure || _tenuring_threshold == 0, "Sanity");

  // Fill in TLABs

  heap->ensure_parsability(true);  // retire TLABs

  if (VerifyBeforeGC && heap->total_collections() >= VerifyGCStartAt) {

    HandleMark hm;  // Discard invalid handles created during verification

    Universe::verify("Before GC");

  }

  {

    ResourceMark rm;

    HandleMark hm;

    GCTraceCPUTime tcpu;

    GCTraceTime(Info, gc) tm("Pause Young", NULL, gc_cause, true);

    TraceCollectorStats tcs(counters());

    TraceMemoryManagerStats tms(heap->young_gc_manager(), gc_cause);

    if (log_is_enabled(Debug, gc, heap, exit)) {

      accumulated_time()->start();

    }

    // Let the size policy know we're starting

    size_policy->minor_collection_begin();

    // Verify the object start arrays.

    if (VerifyObjectStartArray &&

        VerifyBeforeGC) {

      old_gen->verify_object_start_array();

    }

    // Verify no unmarked old->young roots

    if (VerifyRememberedSets) {

      heap->card_table()->verify_all_young_refs_imprecise();

    }

    assert(young_gen->to_space()->is_empty(),

           "Attempt to scavenge with live objects in to_space");

    young_gen->to_space()->clear(SpaceDecorator::Mangle);

    save_to_space_top_before_gc();

#if COMPILER2_OR_JVMCI

    DerivedPointerTable::clear();

#endif

    reference_processor()->enable_discovery();

    reference_processor()->setup_policy(false);

    const PreGenGCValues pre_gc_values = heap->get_pre_gc_values();

    // Reset our survivor overflow.

    set_survivor_overflow(false);

    // We need to save the old top values before

    // creating the promotion_manager. We pass the top

    // values to the card_table, to prevent it from

    // straying into the promotion labs.

    HeapWord* old_top = old_gen->object_space()->top();

    const uint active_workers =

      WorkerPolicy::calc_active_workers(ParallelScavengeHeap::heap()->workers().total_workers(),