/*

 * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#ifndef SHARE_GC_G1_G1COLLECTEDHEAP_HPP

#define SHARE_GC_G1_G1COLLECTEDHEAP_HPP

#include "gc/g1/g1BarrierSet.hpp"

#include "gc/g1/g1BiasedArray.hpp"

#include "gc/g1/g1CardTable.hpp"

#include "gc/g1/g1CollectionSet.hpp"

#include "gc/g1/g1CollectorState.hpp"

#include "gc/g1/g1ConcurrentMark.hpp"

#include "gc/g1/g1EdenRegions.hpp"

#include "gc/g1/g1EvacFailure.hpp"

#include "gc/g1/g1EvacStats.hpp"

#include "gc/g1/g1GCPhaseTimes.hpp"

#include "gc/g1/g1HeapTransition.hpp"

#include "gc/g1/g1HeapVerifier.hpp"

#include "gc/g1/g1HRPrinter.hpp"

#include "gc/g1/g1InCSetState.hpp"

#include "gc/g1/g1MonitoringSupport.hpp"

#include "gc/g1/g1SurvivorRegions.hpp"

#include "gc/g1/g1YCTypes.hpp"

#include "gc/g1/heapRegionManager.hpp"

#include "gc/g1/heapRegionSet.hpp"

#include "gc/g1/heterogeneousHeapRegionManager.hpp"

#include "gc/shared/barrierSet.hpp"

#include "gc/shared/collectedHeap.hpp"

#include "gc/shared/gcHeapSummary.hpp"

#include "gc/shared/plab.hpp"

#include "gc/shared/preservedMarks.hpp"

#include "gc/shared/softRefPolicy.hpp"

#include "memory/memRegion.hpp"

#include "utilities/stack.hpp"

// A "G1CollectedHeap" is an implementation of a java heap for HotSpot.

// It uses the "Garbage First" heap organization and algorithm, which

// may combine concurrent marking with parallel, incremental compaction of

// heap subsets that will yield large amounts of garbage.

// Forward declarations

class HeapRegion;

class GenerationSpec;

class G1ParScanThreadState;

class G1ParScanThreadStateSet;

class G1ParScanThreadState;

class MemoryPool;

class MemoryManager;

class ObjectClosure;

class SpaceClosure;

class CompactibleSpaceClosure;

class Space;

class G1CollectionSet;

class G1CollectorPolicy;

class G1Policy;

class G1HotCardCache;

class G1RemSet;

class G1YoungRemSetSamplingThread;

class HeapRegionRemSetIterator;

class G1ConcurrentMark;

class G1ConcurrentMarkThread;

class G1ConcurrentRefine;

class GenerationCounters;

class STWGCTimer;

class G1NewTracer;

class EvacuationFailedInfo;

class nmethod;

class WorkGang;

class G1Allocator;

class G1ArchiveAllocator;

class G1FullGCScope;

class G1HeapVerifier;

class G1HeapSizingPolicy;

class G1HeapSummary;

class G1EvacSummary;

typedef OverflowTaskQueue<StarTask, mtGC>         RefToScanQueue;

typedef GenericTaskQueueSet<RefToScanQueue, mtGC> RefToScanQueueSet;

typedef int RegionIdx_t;   // needs to hold [ 0..max_regions() )

typedef int CardIdx_t;     // needs to hold [ 0..CardsPerRegion )

// The G1 STW is alive closure.

// An instance is embedded into the G1CH and used as the

// (optional) _is_alive_non_header closure in the STW

// reference processor. It is also extensively used during

// reference processing during STW evacuation pauses.

class G1STWIsAliveClosure : public BoolObjectClosure {

  G1CollectedHeap* _g1h;

public:

  G1STWIsAliveClosure(G1CollectedHeap* g1h) : _g1h(g1h) {}

  bool do_object_b(oop p);

};

class G1STWSubjectToDiscoveryClosure : public BoolObjectClosure {

  G1CollectedHeap* _g1h;

public:

  G1STWSubjectToDiscoveryClosure(G1CollectedHeap* g1h) : _g1h(g1h) {}

  bool do_object_b(oop p);

};

class G1RegionMappingChangedListener : public G1MappingChangedListener {

 private:

  void reset_from_card_cache(uint start_idx, size_t num_regions);

 public:

  virtual void on_commit(uint start_idx, size_t num_regions, bool zero_filled);

};

class G1CollectedHeap : public CollectedHeap {

  friend class G1FreeCollectionSetTask;

  friend class VM_CollectForMetadataAllocation;

  friend class VM_G1CollectForAllocation;

  friend class VM_G1CollectFull;

  friend class VMStructs;

  friend class MutatorAllocRegion;

  friend class G1FullCollector;

  friend class G1GCAllocRegion;

  friend class G1HeapVerifier;

  // Closures used in implementation.

  friend class G1ParScanThreadState;

  friend class G1ParScanThreadStateSet;

  friend class G1ParTask;

  friend class G1PLABAllocator;

  friend class G1PrepareCompactClosure;

  // Other related classes.

  friend class HeapRegionClaimer;

  // Testing classes.

  friend class G1CheckCSetFastTableClosure;

private:

  G1YoungRemSetSamplingThread* _young_gen_sampling_thread;

  WorkGang* _workers;

  G1CollectorPolicy* _collector_policy;

  G1CardTable* _card_table;

  SoftRefPolicy      _soft_ref_policy;

  static size_t _humongous_object_threshold_in_words;

  // These sets keep track of old, archive and humongous regions respectively.

  HeapRegionSet _old_set;

  HeapRegionSet _archive_set;

  HeapRegionSet _humongous_set;

  void eagerly_reclaim_humongous_regions();

  // Start a new incremental collection set for the next pause.

  void start_new_collection_set();

  // The block offset table for the G1 heap.

  G1BlockOffsetTable* _bot;

  // Tears down the region sets / lists so that they are empty and the

  // regions on the heap do not belong to a region set / list. The

  // only exception is the humongous set which we leave unaltered. If

  // free_list_only is true, it will only tear down the master free

  // list. It is called before a Full GC (free_list_only == false) or

  // before heap shrinking (free_list_only == true).

  void tear_down_region_sets(bool free_list_only);

  // Rebuilds the region sets / lists so that they are repopulated to

  // reflect the contents of the heap. The only exception is the

  // humongous set which was not torn down in the first place. If

  // free_list_only is true, it will only rebuild the master free

  // list. It is called after a Full GC (free_list_only == false) or

  // after heap shrinking (free_list_only == true).

  void rebuild_region_sets(bool free_list_only);

  // Callback for region mapping changed events.

  G1RegionMappingChangedListener _listener;

  // The sequence of all heap regions in the heap.

  HeapRegionManager* _hrm;

  // Manages all allocations with regions except humongous object allocations.

  G1Allocator* _allocator;

  // Manages all heap verification.

  G1HeapVerifier* _verifier;

  // Outside of GC pauses, the number of bytes used in all regions other

  // than the current allocation region(s).

  size_t _summary_bytes_used;

  void increase_used(size_t bytes);

  void decrease_used(size_t bytes);

  void set_used(size_t bytes);

  // Class that handles archive allocation ranges.

  G1ArchiveAllocator* _archive_allocator;

  // GC allocation statistics policy for survivors.

  G1EvacStats _survivor_evac_stats;

  // GC allocation statistics policy for tenured objects.

  G1EvacStats _old_evac_stats;

  // It specifies whether we should attempt to expand the heap after a

  // region allocation failure. If heap expansion fails we set this to

  // false so that we don't re-attempt the heap expansion (it's likely

  // that subsequent expansion attempts will also fail if one fails).

  // Currently, it is only consulted during GC and it's reset at the

  // start of each GC.

  bool _expand_heap_after_alloc_failure;

  // Helper for monitoring and management support.

  G1MonitoringSupport* _g1mm;

  // Records whether the region at the given index is (still) a

  // candidate for eager reclaim.  Only valid for humongous start

  // regions; other regions have unspecified values.  Humongous start

  // regions are initialized at start of collection pause, with

  // candidates removed from the set as they are found reachable from

  // roots or the young generation.

  class HumongousReclaimCandidates : public G1BiasedMappedArray<bool> {

   protected:

    bool default_value() const { return false; }

   public:

    void clear() { G1BiasedMappedArray<bool>::clear(); }

    void set_candidate(uint region, bool value) {

      set_by_index(region, value);

    }

    bool is_candidate(uint region) {

      return get_by_index(region);

    }

  };

  HumongousReclaimCandidates _humongous_reclaim_candidates;

  // Stores whether during humongous object registration we found candidate regions.

  // If not, we can skip a few steps.

  bool _has_humongous_reclaim_candidates;

  G1HRPrinter _hr_printer;

  // It decides whether an explicit GC should start a concurrent cycle

  // instead of doing a STW GC. Currently, a concurrent cycle is

  // explicitly started if:

  // (a) cause == _gc_locker and +GCLockerInvokesConcurrent, or

  // (b) cause == _g1_humongous_allocation

  // (c) cause == _java_lang_system_gc and +ExplicitGCInvokesConcurrent.

  // (d) cause == _dcmd_gc_run and +ExplicitGCInvokesConcurrent.

  // (e) cause == _wb_conc_mark

  bool should_do_concurrent_full_gc(GCCause::Cause cause);

  // Return true if should upgrade to full gc after an incremental one.

  bool should_upgrade_to_full_gc(GCCause::Cause cause);

  // indicates whether we are in young or mixed GC mode

  G1CollectorState _collector_state;

  // Keeps track of how many "old marking cycles" (i.e., Full GCs or

  // concurrent cycles) we have started.

  volatile uint _old_marking_cycles_started;

  // Keeps track of how many "old marking cycles" (i.e., Full GCs or

  // concurrent cycles) we have completed.

  volatile uint _old_marking_cycles_completed;

  // This is a non-product method that is helpful for testing. It is

  // called at the end of a GC and artificially expands the heap by

  // allocating a number of dead regions. This way we can induce very

  // frequent marking cycles and stress the cleanup / concurrent

  // cleanup code more (as all the regions that will be allocated by

  // this method will be found dead by the marking cycle).

  void allocate_dummy_regions() PRODUCT_RETURN;

  // If the HR printer is active, dump the state of the regions in the

  // heap after a compaction.

  void print_hrm_post_compaction();

  // Create a memory mapper for auxiliary data structures of the given size and

  // translation factor.

  static G1RegionToSpaceMapper* create_aux_memory_mapper(const char* description,

                                                         size_t size,

                                                         size_t translation_factor);

  void trace_heap(GCWhen::Type when, const GCTracer* tracer);

  // These are macros so that, if the assert fires, we get the correct

  // line number, file, etc.

#define heap_locking_asserts_params(_extra_message_)                          \

  "%s : Heap_lock locked: %s, at safepoint: %s, is VM thread: %s",            \

  (_extra_message_),                                                          \

  BOOL_TO_STR(Heap_lock->owned_by_self()),                                    \

  BOOL_TO_STR(SafepointSynchronize::is_at_safepoint()),                       \

  BOOL_TO_STR(Thread::current()->is_VM_thread())

#define assert_heap_locked()                                                  \

  do {                                                                        \

    assert(Heap_lock->owned_by_self(),                                        \

           heap_locking_asserts_params("should be holding the Heap_lock"));   \

  } while (0)

#define assert_heap_locked_or_at_safepoint(_should_be_vm_thread_)             \

  do {                                                                        \

    assert(Heap_lock->owned_by_self() ||                                      \

           (SafepointSynchronize::is_at_safepoint() &&                        \

             ((_should_be_vm_thread_) == Thread::current()->is_VM_thread())), \

           heap_locking_asserts_params("should be holding the Heap_lock or "  \

                                        "should be at a safepoint"));         \

  } while (0)

#define assert_heap_locked_and_not_at_safepoint()                             \

  do {                                                                        \

    assert(Heap_lock->owned_by_self() &&                                      \

                                    !SafepointSynchronize::is_at_safepoint(), \

          heap_locking_asserts_params("should be holding the Heap_lock and "  \

                                       "should not be at a safepoint"));      \

  } while (0)

#define assert_heap_not_locked()                                              \

  do {                                                                        \

    assert(!Heap_lock->owned_by_self(),                                       \

        heap_locking_asserts_params("should not be holding the Heap_lock"));  \

  } while (0)

#define assert_heap_not_locked_and_not_at_safepoint()                         \

  do {                                                                        \

    assert(!Heap_lock->owned_by_self() &&                                     \

                                    !SafepointSynchronize::is_at_safepoint(), \

      heap_locking_asserts_params("should not be holding the Heap_lock and "  \

                                   "should not be at a safepoint"));          \

  } while (0)

#define assert_at_safepoint_on_vm_thread()                                    \

  do {                                                                        \

    assert_at_safepoint();                                                    \

    assert(Thread::current_or_null() != NULL, "no current thread");           \

    assert(Thread::current()->is_VM_thread(), "current thread is not VM thread"); \

  } while (0)

  // The young region list.

  G1EdenRegions _eden;

  G1SurvivorRegions _survivor;

  STWGCTimer* _gc_timer_stw;

  G1NewTracer* _gc_tracer_stw;

  // The current policy object for the collector.

  G1Policy* _g1_policy;

  G1HeapSizingPolicy* _heap_sizing_policy;

  G1CollectionSet _collection_set;

  // Try to allocate a single non-humongous HeapRegion sufficient for

  // an allocation of the given word_size. If do_expand is true,

  // attempt to expand the heap if necessary to satisfy the allocation

  // request. 'type' takes the type of region to be allocated. (Use constants

  // Old, Eden, Humongous, Survivor defined in HeapRegionType.)

  HeapRegion* new_region(size_t word_size, HeapRegionType type, bool do_expand);

  // Initialize a contiguous set of free regions of length num_regions

  // and starting at index first so that they appear as a single

  // humongous region.

  HeapWord* humongous_obj_allocate_initialize_regions(uint first,

                                                      uint num_regions,

                                                      size_t word_size);

  // Attempt to allocate a humongous object of the given size. Return

  // NULL if unsuccessful.

  HeapWord* humongous_obj_allocate(size_t word_size);

  // The following two methods, allocate_new_tlab() and

  // mem_allocate(), are the two main entry points from the runtime

  // into the G1's allocation routines. They have the following

  // assumptions:

  //

  // * They should both be called outside safepoints.

  //

  // * They should both be called without holding the Heap_lock.

  //

  // * All allocation requests for new TLABs should go to

  //   allocate_new_tlab().

  //

  // * All non-TLAB allocation requests should go to mem_allocate().

  //

  // * If either call cannot satisfy the allocation request using the

  //   current allocating region, they will try to get a new one. If

  //   this fails, they will attempt to do an evacuation pause and

  //   retry the allocation.

  //

  // * If all allocation attempts fail, even after trying to schedule

  //   an evacuation pause, allocate_new_tlab() will return NULL,

  //   whereas mem_allocate() will attempt a heap expansion and/or

  //   schedule a Full GC.

  //

  // * We do not allow humongous-sized TLABs. So, allocate_new_tlab

  //   should never be called with word_size being humongous. All

  //   humongous allocation requests should go to mem_allocate() which

  //   will satisfy them with a special path.

  virtual HeapWord* allocate_new_tlab(size_t min_size,

                                      size_t requested_size,

                                      size_t* actual_size);

  virtual HeapWord* mem_allocate(size_t word_size,

                                 bool*  gc_overhead_limit_was_exceeded);

  // First-level mutator allocation attempt: try to allocate out of

  // the mutator alloc region without taking the Heap_lock. This

  // should only be used for non-humongous allocations.

  inline HeapWord* attempt_allocation(size_t min_word_size,

                                      size_t desired_word_size,

                                      size_t* actual_word_size);

  // Second-level mutator allocation attempt: take the Heap_lock and

  // retry the allocation attempt, potentially scheduling a GC

  // pause. This should only be used for non-humongous allocations.

  HeapWord* attempt_allocation_slow(size_t word_size);

  // Takes the Heap_lock and attempts a humongous allocation. It can

  // potentially schedule a GC pause.

  HeapWord* attempt_allocation_humongous(size_t word_size);

  // Allocation attempt that should be called during safepoints (e.g.,

  // at the end of a successful GC). expect_null_mutator_alloc_region

  // specifies whether the mutator alloc region is expected to be NULL

  // or not.

  HeapWord* attempt_allocation_at_safepoint(size_t word_size,

                                            bool expect_null_mutator_alloc_region);

  // These methods are the "callbacks" from the G1AllocRegion class.

  // For mutator alloc regions.

  HeapRegion* new_mutator_alloc_region(size_t word_size, bool force);

  void retire_mutator_alloc_region(HeapRegion* alloc_region,

                                   size_t allocated_bytes);

  // For GC alloc regions.

  bool has_more_regions(InCSetState dest);

  HeapRegion* new_gc_alloc_region(size_t word_size, InCSetState dest);

  void retire_gc_alloc_region(HeapRegion* alloc_region,

                              size_t allocated_bytes, InCSetState dest);

  // - if explicit_gc is true, the GC is for a System.gc() etc,

  //   otherwise it's for a failed allocation.

  // - if clear_all_soft_refs is true, all soft references should be

  //   cleared during the GC.

  // - it returns false if it is unable to do the collection due to the

  //   GC locker being active, true otherwise.

  bool do_full_collection(bool explicit_gc,

                          bool clear_all_soft_refs);

  // Callback from VM_G1CollectFull operation, or collect_as_vm_thread.

  virtual void do_full_collection(bool clear_all_soft_refs);

  // Callback from VM_G1CollectForAllocation operation.

  // This function does everything necessary/possible to satisfy a

  // failed allocation request (including collection, expansion, etc.)

  HeapWord* satisfy_failed_allocation(size_t word_size,

                                      bool* succeeded);

  // Internal helpers used during full GC to split it up to

  // increase readability.

  void abort_concurrent_cycle();

  void verify_before_full_collection(bool explicit_gc);

  void prepare_heap_for_full_collection();

  void prepare_heap_for_mutators();