/*

 * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP

#define SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP

#include "gc/shared/markBitMap.hpp"

#include "gc/shared/softRefPolicy.hpp"

#include "gc/shared/collectedHeap.hpp"

#include "gc/shenandoah/shenandoahAsserts.hpp"

#include "gc/shenandoah/shenandoahAllocRequest.hpp"

#include "gc/shenandoah/shenandoahHeapLock.hpp"

#include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"

#include "gc/shenandoah/shenandoahSharedVariables.hpp"

#include "services/memoryManager.hpp"

class ConcurrentGCTimer;

class ReferenceProcessor;

class ShenandoahAllocTracker;

class ShenandoahCollectorPolicy;

class ShenandoahControlThread;

class ShenandoahGCSession;

class ShenandoahHeuristics;

class ShenandoahMarkingContext;

class ShenandoahPhaseTimings;

class ShenandoahHeap;

class ShenandoahHeapRegion;

class ShenandoahHeapRegionClosure;

class ShenandoahCollectionSet;

class ShenandoahFreeSet;

class ShenandoahConcurrentMark;

class ShenandoahMarkCompact;

class ShenandoahMonitoringSupport;

class ShenandoahPacer;

class ShenandoahTraversalGC;

class ShenandoahVerifier;

class ShenandoahWorkGang;

class VMStructs;

class ShenandoahRegionIterator : public StackObj {

private:

  ShenandoahHeap* _heap;

  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));

  volatile size_t _index;

  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);

  // No implicit copying: iterators should be passed by reference to capture the state

  ShenandoahRegionIterator(const ShenandoahRegionIterator& that);

  ShenandoahRegionIterator& operator=(const ShenandoahRegionIterator& o);

public:

  ShenandoahRegionIterator();

  ShenandoahRegionIterator(ShenandoahHeap* heap);

  // Reset iterator to default state

  void reset();

  // Returns next region, or NULL if there are no more regions.

  // This is multi-thread-safe.

  inline ShenandoahHeapRegion* next();

  // This is *not* MT safe. However, in the absence of multithreaded access, it

  // can be used to determine if there is more work to do.

  bool has_next() const;

};

class ShenandoahHeapRegionClosure : public StackObj {

public:

  virtual void heap_region_do(ShenandoahHeapRegion* r) = 0;

  virtual bool is_thread_safe() { return false; }

};

class ShenandoahUpdateRefsClosure: public OopClosure {

private:

  ShenandoahHeap* _heap;

  template <class T>

  inline void do_oop_work(T* p);

public:

  ShenandoahUpdateRefsClosure();

  inline void do_oop(oop* p);

  inline void do_oop(narrowOop* p);

};

#ifdef ASSERT

class ShenandoahAssertToSpaceClosure : public OopClosure {

private:

  template <class T>

  void do_oop_work(T* p);

public:

  void do_oop(narrowOop* p);

  void do_oop(oop* p);

};

#endif

class ShenandoahAlwaysTrueClosure : public BoolObjectClosure {

public:

  bool do_object_b(oop p) { return true; }

};

class ShenandoahForwardedIsAliveClosure: public BoolObjectClosure {

private:

  ShenandoahMarkingContext* const _mark_context;

public:

  ShenandoahForwardedIsAliveClosure();

  bool do_object_b(oop obj);

};

class ShenandoahIsAliveClosure: public BoolObjectClosure {

private:

  ShenandoahMarkingContext* const _mark_context;

public:

  ShenandoahIsAliveClosure();

  bool do_object_b(oop obj);

};

class ShenandoahIsAliveSelector : public StackObj {

private:

  ShenandoahIsAliveClosure _alive_cl;

  ShenandoahForwardedIsAliveClosure _fwd_alive_cl;

public:

  BoolObjectClosure* is_alive_closure();

};

// Shenandoah GC is low-pause concurrent GC that uses Brooks forwarding pointers

// to encode forwarding data. See BrooksPointer for details on forwarding data encoding.

// See ShenandoahControlThread for GC cycle structure.

//

class ShenandoahHeap : public CollectedHeap {

  friend class ShenandoahAsserts;

  friend class VMStructs;

  friend class ShenandoahGCSession;

// ---------- Locks that guard important data structures in Heap

//

private:

  ShenandoahHeapLock _lock;

public:

  ShenandoahHeapLock* lock() {

    return &_lock;

  }

  void assert_heaplock_owned_by_current_thread()     NOT_DEBUG_RETURN;

  void assert_heaplock_not_owned_by_current_thread() NOT_DEBUG_RETURN;

  void assert_heaplock_or_safepoint()                NOT_DEBUG_RETURN;

// ---------- Initialization, termination, identification, printing routines

//

public:

  static ShenandoahHeap* heap();

  static ShenandoahHeap* heap_no_check();

  const char* name()          const { return "Shenandoah"; }

  ShenandoahHeap::Name kind() const { return CollectedHeap::Shenandoah; }

  ShenandoahHeap(ShenandoahCollectorPolicy* policy);

  jint initialize();

  void post_initialize();

  void initialize_heuristics();

  void initialize_serviceability();

  void print_on(outputStream* st)              const;

  void print_extended_on(outputStream *st)     const;

  void print_tracing_info()                    const;

  void print_gc_threads_on(outputStream* st)   const;

  void print_heap_regions_on(outputStream* st) const;

  void stop();

  void prepare_for_verify();

  void verify(VerifyOption vo);

// ---------- Heap counters and metrics

//

private:

           size_t _initial_size;

  DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile size_t));

  volatile size_t _used;

  volatile size_t _committed;

  volatile size_t _bytes_allocated_since_gc_start;

  DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);

public:

  void increase_used(size_t bytes);

  void decrease_used(size_t bytes);

  void set_used(size_t bytes);

  void increase_committed(size_t bytes);

  void decrease_committed(size_t bytes);

  void increase_allocated(size_t bytes);

  size_t bytes_allocated_since_gc_start();

  void reset_bytes_allocated_since_gc_start();

  size_t max_capacity()     const;

  size_t initial_capacity() const;

  size_t capacity()         const;

  size_t used()             const;

  size_t committed()        const;

// ---------- Workers handling

//

private:

  uint _max_workers;

  ShenandoahWorkGang* _workers;

  ShenandoahWorkGang* _safepoint_workers;

public:

  uint max_workers();

  void assert_gc_workers(uint nworker) NOT_DEBUG_RETURN;

  WorkGang* workers() const;

  WorkGang* get_safepoint_workers();

  void gc_threads_do(ThreadClosure* tcl) const;

// ---------- Heap regions handling machinery

//

private:

  MemRegion _heap_region;

  size_t    _num_regions;

  ShenandoahHeapRegion** _regions;

  ShenandoahRegionIterator _update_refs_iterator;

public:

  inline size_t num_regions() const { return _num_regions; }

  inline ShenandoahHeapRegion* const heap_region_containing(const void* addr) const;

  inline size_t heap_region_index_containing(const void* addr) const;

  inline ShenandoahHeapRegion* const get_region(size_t region_idx) const;

  void heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;

  void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;

// ---------- GC state machinery

//

// GC state describes the important parts of collector state, that may be

// used to make barrier selection decisions in the native and generated code.

// Multiple bits can be set at once.

//

// Important invariant: when GC state is zero, the heap is stable, and no barriers

// are required.

//

public:

  enum GCStateBitPos {

    // Heap has forwarded objects: need RB, ACMP, CAS barriers.

    HAS_FORWARDED_BITPOS   = 0,

    // Heap is under marking: needs SATB barriers.

    MARKING_BITPOS    = 1,

    // Heap is under evacuation: needs WB barriers. (Set together with UNSTABLE)

    EVACUATION_BITPOS = 2,

    // Heap is under updating: needs SVRB/SVWB barriers.

    UPDATEREFS_BITPOS = 3,

    // Heap is under traversal collection

    TRAVERSAL_BITPOS  = 4,

  };

  enum GCState {

    STABLE        = 0,

    HAS_FORWARDED = 1 << HAS_FORWARDED_BITPOS,

    MARKING       = 1 << MARKING_BITPOS,

    EVACUATION    = 1 << EVACUATION_BITPOS,

    UPDATEREFS    = 1 << UPDATEREFS_BITPOS,

    TRAVERSAL     = 1 << TRAVERSAL_BITPOS,

  };

private:

  ShenandoahSharedBitmap _gc_state;

  ShenandoahSharedFlag   _degenerated_gc_in_progress;

  ShenandoahSharedFlag   _full_gc_in_progress;

  ShenandoahSharedFlag   _full_gc_move_in_progress;

  ShenandoahSharedFlag   _progress_last_gc;

  void set_gc_state_all_threads(char state);

  void set_gc_state_mask(uint mask, bool value);

public:

  char gc_state() const;

  static address gc_state_addr();

  void set_concurrent_mark_in_progress(bool in_progress);

  void set_evacuation_in_progress(bool in_progress);

  void set_update_refs_in_progress(bool in_progress);

  void set_degenerated_gc_in_progress(bool in_progress);

  void set_full_gc_in_progress(bool in_progress);

  void set_full_gc_move_in_progress(bool in_progress);

  void set_concurrent_traversal_in_progress(bool in_progress);

  void set_has_forwarded_objects(bool cond);

  inline bool is_stable() const;

  inline bool is_idle() const;

  inline bool is_concurrent_mark_in_progress() const;

  inline bool is_update_refs_in_progress() const;

  inline bool is_evacuation_in_progress() const;

  inline bool is_degenerated_gc_in_progress() const;

  inline bool is_full_gc_in_progress() const;

  inline bool is_full_gc_move_in_progress() const;

  inline bool is_concurrent_traversal_in_progress() const;

  inline bool has_forwarded_objects() const;

  inline bool is_gc_in_progress_mask(uint mask) const;

// ---------- GC cancellation and degeneration machinery

//

// Cancelled GC flag is used to notify concurrent phases that they should terminate.

//

public:

  enum ShenandoahDegenPoint {

    _degenerated_unset,

    _degenerated_traversal,

    _degenerated_outside_cycle,

    _degenerated_mark,

    _degenerated_evac,

    _degenerated_updaterefs,

    _DEGENERATED_LIMIT,

  };

  static const char* degen_point_to_string(ShenandoahDegenPoint point) {

    switch (point) {

      case _degenerated_unset:

        return "<UNSET>";

      case _degenerated_traversal:

        return "Traversal";

      case _degenerated_outside_cycle:

        return "Outside of Cycle";

      case _degenerated_mark:

        return "Mark";

      case _degenerated_evac:

        return "Evacuation";

      case _degenerated_updaterefs:

        return "Update Refs";

      default:

        ShouldNotReachHere();

        return "ERROR";

    }

  };

private:

  enum CancelState {

    // Normal state. GC has not been cancelled and is open for cancellation.

    // Worker threads can suspend for safepoint.

    CANCELLABLE,

    // GC has been cancelled. Worker threads can not suspend for

    // safepoint but must finish their work as soon as possible.

    CANCELLED,

    // GC has not been cancelled and must not be cancelled. At least

    // one worker thread checks for pending safepoint and may suspend

    // if a safepoint is pending.

    NOT_CANCELLED

  };

  ShenandoahSharedEnumFlag<CancelState> _cancelled_gc;

  bool try_cancel_gc();

public:

  static address cancelled_gc_addr();

  inline bool cancelled_gc() const;

  inline bool check_cancelled_gc_and_yield(bool sts_active = true);

  inline void clear_cancelled_gc();

  void cancel_gc(GCCause::Cause cause);

// ---------- GC operations entry points

//

public:

  // Entry points to STW GC operations, these cause a related safepoint, that then

  // call the entry method below

  void vmop_entry_init_mark();

  void vmop_entry_final_mark();

  void vmop_entry_final_evac();

  void vmop_entry_init_updaterefs();

  void vmop_entry_final_updaterefs();

  void vmop_entry_init_traversal();

  void vmop_entry_final_traversal();

  void vmop_entry_full(GCCause::Cause cause);

  void vmop_degenerated(ShenandoahDegenPoint point);

  // Entry methods to normally STW GC operations. These set up logging, monitoring

  // and workers for net VM operation

  void entry_init_mark();

  void entry_final_mark();

  void entry_final_evac();

  void entry_init_updaterefs();

  void entry_final_updaterefs();

  void entry_init_traversal();

  void entry_final_traversal();

  void entry_full(GCCause::Cause cause);

  void entry_degenerated(int point);

  // Entry methods to normally concurrent GC operations. These set up logging, monitoring

  // for concurrent operation.

  void entry_reset();

  void entry_mark();

  void entry_preclean();

  void entry_cleanup();

  void entry_evac();

  void entry_updaterefs();

  void entry_traversal();

  void entry_uncommit(double shrink_before);

private:

  // Actual work for the phases

  void op_init_mark();

  void op_final_mark();

  void op_final_evac();

  void op_init_updaterefs();

  void op_final_updaterefs();

  void op_init_traversal();

  void op_final_traversal();

  void op_full(GCCause::Cause cause);

  void op_degenerated(ShenandoahDegenPoint point);

  void op_degenerated_fail();

  void op_degenerated_futile();

  void op_reset();

  void op_mark();

  void op_preclean();

  void op_cleanup();

  void op_conc_evac();

  void op_stw_evac();

  void op_updaterefs();

  void op_traversal();

  void op_uncommit(double shrink_before);

  // Messages for GC trace events, they have to be immortal for

  // passing around the logging/tracing systems

  const char* init_mark_event_message() const;

  const char* final_mark_event_message() const;

  const char* conc_mark_event_message() const;

  const char* degen_event_message(ShenandoahDegenPoint point) const;

// ---------- GC subsystems

//

private:

  ShenandoahControlThread*   _control_thread;

  ShenandoahCollectorPolicy* _shenandoah_policy;

  ShenandoahHeuristics*      _heuristics;

  ShenandoahFreeSet*         _free_set;

  ShenandoahConcurrentMark*  _scm;

  ShenandoahTraversalGC*     _traversal_gc;

  ShenandoahMarkCompact*     _full_gc;

  ShenandoahPacer*           _pacer;

  ShenandoahVerifier*        _verifier;

  ShenandoahAllocTracker*    _alloc_tracker;

  ShenandoahPhaseTimings*    _phase_timings;

  ShenandoahControlThread*   control_thread()          { return _control_thread;    }

  ShenandoahMarkCompact*     full_gc()                 { return _full_gc;           }

public:

  ShenandoahCollectorPolicy* shenandoah_policy() const { return _shenandoah_policy; }

  ShenandoahHeuristics*      heuristics()        const { return _heuristics;        }

  ShenandoahFreeSet*         free_set()          const { return _free_set;          }

  ShenandoahConcurrentMark*  concurrent_mark()         { return _scm;               }

  ShenandoahTraversalGC*     traversal_gc()            { return _traversal_gc;      }

  ShenandoahPhaseTimings*    phase_timings()     const { return _phase_timings;     }

  ShenandoahAllocTracker*    alloc_tracker()     const { return _alloc_tracker;     }

  ShenandoahVerifier*        verifier();

// ---------- VM subsystem bindings

//

private:

  ShenandoahMonitoringSupport* _monitoring_support;

  MemoryPool*                  _memory_pool;

  GCMemoryManager              _stw_memory_manager;

  GCMemoryManager              _cycle_memory_manager;

  ConcurrentGCTimer*           _gc_timer;

  SoftRefPolicy                _soft_ref_policy;

public:

  ShenandoahMonitoringSupport* monitoring_support() { return _monitoring_support;    }

  GCMemoryManager* cycle_memory_manager()           { return &_cycle_memory_manager; }

  GCMemoryManager* stw_memory_manager()             { return &_stw_memory_manager;   }

  SoftRefPolicy* soft_ref_policy()                  { return &_soft_ref_policy;      }

  GrowableArray<GCMemoryManager*> memory_managers();

  GrowableArray<MemoryPool*> memory_pools();

  MemoryUsage memory_usage();

  GCTracer* tracer();

  GCTimer* gc_timer() const;

  CollectorPolicy* collector_policy() const;

// ---------- Reference processing

//

private:

  AlwaysTrueClosure    _subject_to_discovery;

  ReferenceProcessor*  _ref_processor;

  ShenandoahSharedFlag _process_references;

  void ref_processing_init();