1 /*

     2  * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  *

    23  */

    24 

    25 #ifndef SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPGENERATION_HPP

    26 #define SHARE_VM_GC_CMS_CONCURRENTMARKSWEEPGENERATION_HPP

    27 

    28 #include "gc/cms/cmsOopClosures.hpp"

    29 #include "gc/cms/gSpaceCounters.hpp"

    30 #include "gc/cms/yieldingWorkgroup.hpp"

    31 #include "gc/shared/cardGeneration.hpp"

    32 #include "gc/shared/gcHeapSummary.hpp"

    33 #include "gc/shared/gcStats.hpp"

    34 #include "gc/shared/gcWhen.hpp"

    35 #include "gc/shared/generationCounters.hpp"

    36 #include "gc/shared/space.hpp"

    37 #include "gc/shared/taskqueue.hpp"

    38 #include "memory/freeBlockDictionary.hpp"

    39 #include "memory/iterator.hpp"

    40 #include "memory/virtualspace.hpp"

    41 #include "runtime/mutexLocker.hpp"

    42 #include "services/memoryService.hpp"

    43 #include "utilities/bitMap.hpp"

    44 #include "utilities/stack.hpp"

    45 

    46 // ConcurrentMarkSweepGeneration is in support of a concurrent

    47 // mark-sweep old generation in the Detlefs-Printezis--Boehm-Demers-Schenker

    48 // style. We assume, for now, that this generation is always the

    49 // seniormost generation and for simplicity

    50 // in the first implementation, that this generation is a single compactible

    51 // space. Neither of these restrictions appears essential, and will be

    52 // relaxed in the future when more time is available to implement the

    53 // greater generality (and there's a need for it).

    54 //

    55 // Concurrent mode failures are currently handled by

    56 // means of a sliding mark-compact.

    57 

    58 class AdaptiveSizePolicy;

    59 class CMSCollector;

    60 class CMSConcMarkingTask;

    61 class CMSGCAdaptivePolicyCounters;

    62 class CMSTracer;

    63 class ConcurrentGCTimer;

    64 class ConcurrentMarkSweepGeneration;

    65 class ConcurrentMarkSweepPolicy;

    66 class ConcurrentMarkSweepThread;

    67 class CompactibleFreeListSpace;

    68 class FreeChunk;

    69 class ParNewGeneration;

    70 class PromotionInfo;

    71 class ScanMarkedObjectsAgainCarefullyClosure;

    72 class TenuredGeneration;

    73 class SerialOldTracer;

    74 

    75 // A generic CMS bit map. It's the basis for both the CMS marking bit map

    76 // as well as for the mod union table (in each case only a subset of the

    77 // methods are used). This is essentially a wrapper around the BitMap class,

    78 // with one bit per (1<<_shifter) HeapWords. (i.e. for the marking bit map,

    79 // we have _shifter == 0. and for the mod union table we have

    80 // shifter == CardTableModRefBS::card_shift - LogHeapWordSize.)

    81 // XXX 64-bit issues in BitMap?

    82 class CMSBitMap VALUE_OBJ_CLASS_SPEC {

    83   friend class VMStructs;

    84 

    85   HeapWord* _bmStartWord;   // base address of range covered by map

    86   size_t    _bmWordSize;    // map size (in #HeapWords covered)

    87   const int _shifter;       // shifts to convert HeapWord to bit position

    88   VirtualSpace _virtual_space; // underlying the bit map

    89   BitMap    _bm;            // the bit map itself

    90  public:

    91   Mutex* const _lock;       // mutex protecting _bm;

    92 

    93  public:

    94   // constructor

    95   CMSBitMap(int shifter, int mutex_rank, const char* mutex_name);

    96 

    97   // allocates the actual storage for the map

    98   bool allocate(MemRegion mr);

    99   // field getter

   100   Mutex* lock() const { return _lock; }

   101   // locking verifier convenience function

   102   void assert_locked() const PRODUCT_RETURN;

   103 

   104   // inquiries

   105   HeapWord* startWord()   const { return _bmStartWord; }

   106   size_t    sizeInWords() const { return _bmWordSize;  }

   107   size_t    sizeInBits()  const { return _bm.size();   }

   108   // the following is one past the last word in space

   109   HeapWord* endWord()     const { return _bmStartWord + _bmWordSize; }

   110 

   111   // reading marks

   112   bool isMarked(HeapWord* addr) const;

   113   bool par_isMarked(HeapWord* addr) const; // do not lock checks

   114   bool isUnmarked(HeapWord* addr) const;

   115   bool isAllClear() const;

   116 

   117   // writing marks

   118   void mark(HeapWord* addr);

   119   // For marking by parallel GC threads;

   120   // returns true if we did, false if another thread did

   121   bool par_mark(HeapWord* addr);

   122 

   123   void mark_range(MemRegion mr);

   124   void par_mark_range(MemRegion mr);

   125   void mark_large_range(MemRegion mr);

   126   void par_mark_large_range(MemRegion mr);

   127   void par_clear(HeapWord* addr); // For unmarking by parallel GC threads.

   128   void clear_range(MemRegion mr);

   129   void par_clear_range(MemRegion mr);

   130   void clear_large_range(MemRegion mr);

   131   void par_clear_large_range(MemRegion mr);

   132   void clear_all();

   133   void clear_all_incrementally();  // Not yet implemented!!

   134 

   135   NOT_PRODUCT(

   136     // checks the memory region for validity

   137     void region_invariant(MemRegion mr);

   138   )

   139 

   140   // iteration

   141   void iterate(BitMapClosure* cl) {

   142     _bm.iterate(cl);

   143   }

   144   void iterate(BitMapClosure* cl, HeapWord* left, HeapWord* right);

   145   void dirty_range_iterate_clear(MemRegionClosure* cl);

   146   void dirty_range_iterate_clear(MemRegion mr, MemRegionClosure* cl);

   147 

   148   // auxiliary support for iteration

   149   HeapWord* getNextMarkedWordAddress(HeapWord* addr) const;

   150   HeapWord* getNextMarkedWordAddress(HeapWord* start_addr,

   151                                             HeapWord* end_addr) const;

   152   HeapWord* getNextUnmarkedWordAddress(HeapWord* addr) const;

   153   HeapWord* getNextUnmarkedWordAddress(HeapWord* start_addr,

   154                                               HeapWord* end_addr) const;

   155   MemRegion getAndClearMarkedRegion(HeapWord* addr);

   156   MemRegion getAndClearMarkedRegion(HeapWord* start_addr,

   157                                            HeapWord* end_addr);

   158 

   159   // conversion utilities

   160   HeapWord* offsetToHeapWord(size_t offset) const;

   161   size_t    heapWordToOffset(HeapWord* addr) const;

   162   size_t    heapWordDiffToOffsetDiff(size_t diff) const;

   163 

   164   void print_on_error(outputStream* st, const char* prefix) const;

   165 

   166   // debugging

   167   // is this address range covered by the bit-map?

   168   NOT_PRODUCT(

   169     bool covers(MemRegion mr) const;

   170     bool covers(HeapWord* start, size_t size = 0) const;

   171   )

   172   void verifyNoOneBitsInRange(HeapWord* left, HeapWord* right) PRODUCT_RETURN;

   173 };

   174 

   175 // Represents a marking stack used by the CMS collector.

   176 // Ideally this should be GrowableArray<> just like MSC's marking stack(s).

   177 class CMSMarkStack: public CHeapObj<mtGC>  {

   178   friend class CMSCollector;   // To get at expansion stats further below.

   179 

   180   VirtualSpace _virtual_space;  // Space for the stack

   181   oop*   _base;      // Bottom of stack

   182   size_t _index;     // One more than last occupied index

   183   size_t _capacity;  // Max #elements

   184   Mutex  _par_lock;  // An advisory lock used in case of parallel access

   185   NOT_PRODUCT(size_t _max_depth;)  // Max depth plumbed during run

   186 

   187  protected:

   188   size_t _hit_limit;      // We hit max stack size limit

   189   size_t _failed_double;  // We failed expansion before hitting limit

   190 

   191  public:

   192   CMSMarkStack():

   193     _par_lock(Mutex::event, "CMSMarkStack._par_lock", true,

   194               Monitor::_safepoint_check_never),

   195     _hit_limit(0),

   196     _failed_double(0) {}

   197 

   198   bool allocate(size_t size);

   199 

   200   size_t capacity() const { return _capacity; }

   201 

   202   oop pop() {

   203     if (!isEmpty()) {

   204       return _base[--_index] ;

   205     }

   206     return NULL;

   207   }

   208 

   209   bool push(oop ptr) {

   210     if (isFull()) {

   211       return false;

   212     } else {

   213       _base[_index++] = ptr;

   214       NOT_PRODUCT(_max_depth = MAX2(_max_depth, _index));

   215       return true;

   216     }

   217   }

   218 

   219   bool isEmpty() const { return _index == 0; }

   220   bool isFull()  const {

   221     assert(_index <= _capacity, "buffer overflow");

   222     return _index == _capacity;

   223   }

   224 

   225   size_t length() { return _index; }

   226 

   227   // "Parallel versions" of some of the above

   228   oop par_pop() {

   229     // lock and pop

   230     MutexLockerEx x(&_par_lock, Mutex::_no_safepoint_check_flag);

   231     return pop();

   232   }

   233 

   234   bool par_push(oop ptr) {

   235     // lock and push

   236     MutexLockerEx x(&_par_lock, Mutex::_no_safepoint_check_flag);

   237     return push(ptr);

   238   }

   239 

   240   // Forcibly reset the stack, losing all of its contents.

   241   void reset() {

   242     _index = 0;

   243   }

   244 

   245   // Expand the stack, typically in response to an overflow condition.

   246   void expand();

   247 

   248   // Compute the least valued stack element.

   249   oop least_value(HeapWord* low) {

   250      oop least = (oop)low;

   251      for (size_t i = 0; i < _index; i++) {

   252        least = MIN2(least, _base[i]);

   253      }

   254      return least;

   255   }

   256 

   257   // Exposed here to allow stack expansion in || case.

   258   Mutex* par_lock() { return &_par_lock; }

   259 };

   260 

   261 class CardTableRS;

   262 class CMSParGCThreadState;

   263 

   264 class ModUnionClosure: public MemRegionClosure {

   265  protected:

   266   CMSBitMap* _t;

   267  public:

   268   ModUnionClosure(CMSBitMap* t): _t(t) { }

   269   void do_MemRegion(MemRegion mr);

   270 };

   271 

   272 class ModUnionClosurePar: public ModUnionClosure {

   273  public:

   274   ModUnionClosurePar(CMSBitMap* t): ModUnionClosure(t) { }

   275   void do_MemRegion(MemRegion mr);

   276 };

   277 

   278 // Survivor Chunk Array in support of parallelization of

   279 // Survivor Space rescan.

   280 class ChunkArray: public CHeapObj<mtGC> {

   281   size_t _index;

   282   size_t _capacity;

   283   size_t _overflows;

   284   HeapWord** _array;   // storage for array

   285 

   286  public:

   287   ChunkArray() : _index(0), _capacity(0), _overflows(0), _array(NULL) {}

   288   ChunkArray(HeapWord** a, size_t c):

   289     _index(0), _capacity(c), _overflows(0), _array(a) {}

   290 

   291   HeapWord** array() { return _array; }

   292   void set_array(HeapWord** a) { _array = a; }

   293 

   294   size_t capacity() { return _capacity; }

   295   void set_capacity(size_t c) { _capacity = c; }

   296 

   297   size_t end() {

   298     assert(_index <= capacity(),

   299            err_msg("_index (" SIZE_FORMAT ") > _capacity (" SIZE_FORMAT "): out of bounds",

   300                    _index, _capacity));

   301     return _index;

   302   }  // exclusive

   303 

   304   HeapWord* nth(size_t n) {

   305     assert(n < end(), "Out of bounds access");

   306     return _array[n];

   307   }

   308 

   309   void reset() {

   310     _index = 0;

   311     if (_overflows > 0 && PrintCMSStatistics > 1) {

   312       warning("CMS: ChunkArray[" SIZE_FORMAT "] overflowed " SIZE_FORMAT " times",

   313               _capacity, _overflows);

   314     }

   315     _overflows = 0;

   316   }

   317 

   318   void record_sample(HeapWord* p, size_t sz) {

   319     // For now we do not do anything with the size

   320     if (_index < _capacity) {

   321       _array[_index++] = p;

   322     } else {

   323       ++_overflows;

   324       assert(_index == _capacity,

   325              err_msg("_index (" SIZE_FORMAT ") > _capacity (" SIZE_FORMAT

   326                      "): out of bounds at overflow#" SIZE_FORMAT,

   327                      _index, _capacity, _overflows));

   328     }

   329   }

   330 };

   331 

   332 //

   333 // Timing, allocation and promotion statistics for gc scheduling and incremental

   334 // mode pacing.  Most statistics are exponential averages.

   335 //

   336 class CMSStats VALUE_OBJ_CLASS_SPEC {

   337  private:

   338   ConcurrentMarkSweepGeneration* const _cms_gen;   // The cms (old) gen.

   339 

   340   // The following are exponential averages with factor alpha:

   341   //   avg = (100 - alpha) * avg + alpha * cur_sample

   342   //

   343   //   The durations measure:  end_time[n] - start_time[n]

   344   //   The periods measure:    start_time[n] - start_time[n-1]

   345   //

   346   // The cms period and duration include only concurrent collections; time spent

   347   // in foreground cms collections due to System.gc() or because of a failure to

   348   // keep up are not included.

   349   //

   350   // There are 3 alphas to "bootstrap" the statistics.  The _saved_alpha is the

   351   // real value, but is used only after the first period.  A value of 100 is

   352   // used for the first sample so it gets the entire weight.

   353   unsigned int _saved_alpha; // 0-100

   354   unsigned int _gc0_alpha;

   355   unsigned int _cms_alpha;

   356 

   357   double _gc0_duration;

   358   double _gc0_period;

   359   size_t _gc0_promoted;         // bytes promoted per gc0

   360   double _cms_duration;

   361   double _cms_duration_pre_sweep; // time from initiation to start of sweep

   362   double _cms_period;

   363   size_t _cms_allocated;        // bytes of direct allocation per gc0 period

   364 

   365   // Timers.

   366   elapsedTimer _cms_timer;

   367   TimeStamp    _gc0_begin_time;

   368   TimeStamp    _cms_begin_time;

   369   TimeStamp    _cms_end_time;

   370 

   371   // Snapshots of the amount used in the CMS generation.

   372   size_t _cms_used_at_gc0_begin;

   373   size_t _cms_used_at_gc0_end;

   374   size_t _cms_used_at_cms_begin;

   375 

   376   // Used to prevent the duty cycle from being reduced in the middle of a cms

   377   // cycle.

   378   bool _allow_duty_cycle_reduction;

   379 

   380   enum {

   381     _GC0_VALID = 0x1,

   382     _CMS_VALID = 0x2,

   383     _ALL_VALID = _GC0_VALID | _CMS_VALID

   384   };

   385 

   386   unsigned int _valid_bits;

   387 

   388  protected:

   389   // In support of adjusting of cms trigger ratios based on history

   390   // of concurrent mode failure.

   391   double cms_free_adjustment_factor(size_t free) const;

   392   void   adjust_cms_free_adjustment_factor(bool fail, size_t free);

   393 

   394  public:

   395   CMSStats(ConcurrentMarkSweepGeneration* cms_gen,

   396            unsigned int alpha = CMSExpAvgFactor);

   397 

   398   // Whether or not the statistics contain valid data; higher level statistics

   399   // cannot be called until this returns true (they require at least one young

   400   // gen and one cms cycle to have completed).

   401   bool valid() const;

   402 

   403   // Record statistics.

   404   void record_gc0_begin();

   405   void record_gc0_end(size_t cms_gen_bytes_used);

   406   void record_cms_begin();

   407   void record_cms_end();

   408 

   409   // Allow management of the cms timer, which must be stopped/started around

   410   // yield points.

   411   elapsedTimer& cms_timer()     { return _cms_timer; }

   412   void start_cms_timer()        { _cms_timer.start(); }

   413   void stop_cms_timer()         { _cms_timer.stop(); }

   414 

   415   // Basic statistics; units are seconds or bytes.

   416   double gc0_period() const     { return _gc0_period; }

   417   double gc0_duration() const   { return _gc0_duration; }

   418   size_t gc0_promoted() const   { return _gc0_promoted; }

   419   double cms_period() const          { return _cms_period; }

   420   double cms_duration() const        { return _cms_duration; }

   421   size_t cms_allocated() const       { return _cms_allocated; }

   422 

   423   size_t cms_used_at_gc0_end() const { return _cms_used_at_gc0_end;}

   424 

   425   // Seconds since the last background cms cycle began or ended.

   426   double cms_time_since_begin() const;

   427   double cms_time_since_end() const;

   428 

   429   // Higher level statistics--caller must check that valid() returns true before

   430   // calling.

   431 

   432   // Returns bytes promoted per second of wall clock time.

   433   double promotion_rate() const;

   434 

   435   // Returns bytes directly allocated per second of wall clock time.

   436   double cms_allocation_rate() const;

   437 

   438   // Rate at which space in the cms generation is being consumed (sum of the

   439   // above two).

   440   double cms_consumption_rate() const;

   441 

   442   // Returns an estimate of the number of seconds until the cms generation will

   443   // fill up, assuming no collection work is done.

   444   double time_until_cms_gen_full() const;

   445 

   446   // Returns an estimate of the number of seconds remaining until

   447   // the cms generation collection should start.

   448   double time_until_cms_start() const;

   449 

   450   // End of higher level statistics.

   451 

   452   // Debugging.

   453   void print_on(outputStream* st) const PRODUCT_RETURN;

   454   void print() const { print_on(gclog_or_tty); }

   455 };

   456 

   457 // A closure related to weak references processing which

   458 // we embed in the CMSCollector, since we need to pass

   459 // it to the reference processor for secondary filtering

   460 // of references based on reachability of referent;

   461 // see role of _is_alive_non_header closure in the

   462 // ReferenceProcessor class.

   463 // For objects in the CMS generation, this closure checks

   464 // if the object is "live" (reachable). Used in weak

   465 // reference processing.

   466 class CMSIsAliveClosure: public BoolObjectClosure {

   467   const MemRegion  _span;

   468   const CMSBitMap* _bit_map;

   469 

   470   friend class CMSCollector;

   471  public:

   472   CMSIsAliveClosure(MemRegion span,

   473                     CMSBitMap* bit_map):

   474     _span(span),

   475     _bit_map(bit_map) {

   476     assert(!span.is_empty(), "Empty span could spell trouble");

   477   }

   478 

   479   bool do_object_b(oop obj);

   480 };

   481 

   482 

   483 // Implements AbstractRefProcTaskExecutor for CMS.

   484 class CMSRefProcTaskExecutor: public AbstractRefProcTaskExecutor {

   485 public:

   486 

   487   CMSRefProcTaskExecutor(CMSCollector& collector)

   488     : _collector(collector)

   489   { }

   490 

   491   // Executes a task using worker threads.

   492   virtual void execute(ProcessTask& task);

   493   virtual void execute(EnqueueTask& task);

   494 private:

   495   CMSCollector& _collector;

   496 };

   497 

   498 

   499 class CMSCollector: public CHeapObj<mtGC> {

   500   friend class VMStructs;

   501   friend class ConcurrentMarkSweepThread;

   502   friend class ConcurrentMarkSweepGeneration;

   503   friend class CompactibleFreeListSpace;

   504   friend class CMSParMarkTask;

   505   friend class CMSParInitialMarkTask;

   506   friend class CMSParRemarkTask;

   507   friend class CMSConcMarkingTask;

   508   friend class CMSRefProcTaskProxy;

   509   friend class CMSRefProcTaskExecutor;

   510   friend class ScanMarkedObjectsAgainCarefullyClosure;  // for sampling eden

   511   friend class SurvivorSpacePrecleanClosure;            // --- ditto -------

   512   friend class PushOrMarkClosure;             // to access _restart_addr

   513   friend class Par_PushOrMarkClosure;             // to access _restart_addr

   514   friend class MarkFromRootsClosure;          //  -- ditto --

   515                                               // ... and for clearing cards

   516   friend class Par_MarkFromRootsClosure;      //  to access _restart_addr

   517                                               // ... and for clearing cards

   518   friend class Par_ConcMarkingClosure;        //  to access _restart_addr etc.

   519   friend class MarkFromRootsVerifyClosure;    // to access _restart_addr

   520   friend class PushAndMarkVerifyClosure;      //  -- ditto --

   521   friend class MarkRefsIntoAndScanClosure;    // to access _overflow_list

   522   friend class PushAndMarkClosure;            //  -- ditto --

   523   friend class Par_PushAndMarkClosure;        //  -- ditto --

   524   friend class CMSKeepAliveClosure;           //  -- ditto --

   525   friend class CMSDrainMarkingStackClosure;   //  -- ditto --

   526   friend class CMSInnerParMarkAndPushClosure; //  -- ditto --

   527   NOT_PRODUCT(friend class ScanMarkedObjectsAgainClosure;) //  assertion on _overflow_list

   528   friend class ReleaseForegroundGC;  // to access _foregroundGCShouldWait

   529   friend class VM_CMS_Operation;

   530   friend class VM_CMS_Initial_Mark;

   531   friend class VM_CMS_Final_Remark;

   532   friend class TraceCMSMemoryManagerStats;

   533 

   534  private:

   535   jlong _time_of_last_gc;

   536   void update_time_of_last_gc(jlong now) {

   537     _time_of_last_gc = now;

   538   }

   539 

   540   OopTaskQueueSet* _task_queues;

   541 

   542   // Overflow list of grey objects, threaded through mark-word

   543   // Manipulated with CAS in the parallel/multi-threaded case.

   544   oop _overflow_list;

   545   // The following array-pair keeps track of mark words

   546   // displaced for accommodating overflow list above.

   547   // This code will likely be revisited under RFE#4922830.

   548   Stack<oop, mtGC>     _preserved_oop_stack;

   549   Stack<markOop, mtGC> _preserved_mark_stack;

   550 

   551   int*             _hash_seed;

   552 

   553   // In support of multi-threaded concurrent phases

   554   YieldingFlexibleWorkGang* _conc_workers;

   555 

   556   // Performance Counters

   557   CollectorCounters* _gc_counters;

   558 

   559   // Initialization Errors

   560   bool _completed_initialization;

   561 

   562   // In support of ExplicitGCInvokesConcurrent

   563   static bool _full_gc_requested;

   564   static GCCause::Cause _full_gc_cause;

   565   unsigned int _collection_count_start;

   566 

   567   // Should we unload classes this concurrent cycle?

   568   bool _should_unload_classes;

   569   unsigned int  _concurrent_cycles_since_last_unload;

   570   unsigned int concurrent_cycles_since_last_unload() const {

   571     return _concurrent_cycles_since_last_unload;

   572   }

   573   // Did we (allow) unload classes in the previous concurrent cycle?

   574   bool unloaded_classes_last_cycle() const {

   575     return concurrent_cycles_since_last_unload() == 0;

   576   }

   577   // Root scanning options for perm gen

   578   int _roots_scanning_options;

   579   int roots_scanning_options() const      { return _roots_scanning_options; }

   580   void add_root_scanning_option(int o)    { _roots_scanning_options |= o;   }

   581   void remove_root_scanning_option(int o) { _roots_scanning_options &= ~o;  }

   582 

   583   // Verification support

   584   CMSBitMap     _verification_mark_bm;

   585   void verify_after_remark_work_1();

   586   void verify_after_remark_work_2();

   587 

   588   // True if any verification flag is on.

   589   bool _verifying;

   590   bool verifying() const { return _verifying; }

   591   void set_verifying(bool v) { _verifying = v; }

   592 

   593   // Collector policy

   594   ConcurrentMarkSweepPolicy* _collector_policy;

   595   ConcurrentMarkSweepPolicy* collector_policy() { return _collector_policy; }

   596 

   597   void set_did_compact(bool v);

   598 

   599   // XXX Move these to CMSStats ??? FIX ME !!!

   600   elapsedTimer _inter_sweep_timer;   // Time between sweeps

   601   elapsedTimer _intra_sweep_timer;   // Time _in_ sweeps

   602   // Padded decaying average estimates of the above

   603   AdaptivePaddedAverage _inter_sweep_estimate;

   604   AdaptivePaddedAverage _intra_sweep_estimate;

   605 

   606   CMSTracer* _gc_tracer_cm;

   607   ConcurrentGCTimer* _gc_timer_cm;

   608 

   609   bool _cms_start_registered;

   610 

   611   GCHeapSummary _last_heap_summary;

   612   MetaspaceSummary _last_metaspace_summary;

   613 

   614   void register_gc_start(GCCause::Cause cause);

   615   void register_gc_end();

   616   void save_heap_summary();

   617   void report_heap_summary(GCWhen::Type when);

   618 

   619  protected:

   620   ConcurrentMarkSweepGeneration* _cmsGen;  // Old gen (CMS)

   621   MemRegion                      _span;    // Span covering above two

   622   CardTableRS*                   _ct;      // Card table

   623 

   624   // CMS marking support structures

   625   CMSBitMap     _markBitMap;

   626   CMSBitMap     _modUnionTable;

   627   CMSMarkStack  _markStack;

   628 

   629   HeapWord*     _restart_addr; // In support of marking stack overflow

   630   void          lower_restart_addr(HeapWord* low);

   631 

   632   // Counters in support of marking stack / work queue overflow handling:

   633   // a non-zero value indicates certain types of overflow events during

   634   // the current CMS cycle and could lead to stack resizing efforts at

   635   // an opportune future time.

   636   size_t        _ser_pmc_preclean_ovflw;

   637   size_t        _ser_pmc_remark_ovflw;

   638   size_t        _par_pmc_remark_ovflw;

   639   size_t        _ser_kac_preclean_ovflw;

   640   size_t        _ser_kac_ovflw;

   641   size_t        _par_kac_ovflw;

   642   NOT_PRODUCT(ssize_t _num_par_pushes;)

   643 

   644   // ("Weak") Reference processing support.

   645   ReferenceProcessor*            _ref_processor;

   646   CMSIsAliveClosure              _is_alive_closure;

   647   // Keep this textually after _markBitMap and _span; c'tor dependency.

   648 

   649   ConcurrentMarkSweepThread*     _cmsThread;   // The thread doing the work

   650   ModUnionClosurePar _modUnionClosurePar;

   651 

   652   // CMS abstract state machine

   653   // initial_state: Idling

   654   // next_state(Idling)            = {Marking}

   655   // next_state(Marking)           = {Precleaning, Sweeping}

   656   // next_state(Precleaning)       = {AbortablePreclean, FinalMarking}

   657   // next_state(AbortablePreclean) = {FinalMarking}

   658   // next_state(FinalMarking)      = {Sweeping}

   659   // next_state(Sweeping)          = {Resizing}

   660   // next_state(Resizing)          = {Resetting}

   661   // next_state(Resetting)         = {Idling}

   662   // The numeric values below are chosen so that:

   663   // . _collectorState <= Idling ==  post-sweep && pre-mark

   664   // . _collectorState in (Idling, Sweeping) == {initial,final}marking ||

   665   //                                            precleaning || abortablePrecleanb

   666  public:

   667   enum CollectorState {

   668     Resizing            = 0,

   669     Resetting           = 1,

   670     Idling              = 2,

   671     InitialMarking      = 3,

   672     Marking             = 4,

   673     Precleaning         = 5,

   674     AbortablePreclean   = 6,

   675     FinalMarking        = 7,

   676     Sweeping            = 8

   677   };

   678  protected:

   679   static CollectorState _collectorState;

   680 

   681   // State related to prologue/epilogue invocation for my generations

   682   bool _between_prologue_and_epilogue;

   683 

   684   // Signaling/State related to coordination between fore- and background GC

   685   // Note: When the baton has been passed from background GC to foreground GC,

   686   // _foregroundGCIsActive is true and _foregroundGCShouldWait is false.

   687   static bool _foregroundGCIsActive;    // true iff foreground collector is active or

   688                                  // wants to go active

   689   static bool _foregroundGCShouldWait;  // true iff background GC is active and has not

   690                                  // yet passed the baton to the foreground GC

   691 

   692   // Support for CMSScheduleRemark (abortable preclean)

   693   bool _abort_preclean;

   694   bool _start_sampling;

   695 

   696   int    _numYields;

   697   size_t _numDirtyCards;

   698   size_t _sweep_count;

   699 

   700   // Occupancy used for bootstrapping stats

   701   double _bootstrap_occupancy;

   702 

   703   // Timer

   704   elapsedTimer _timer;

   705 

   706   // Timing, allocation and promotion statistics, used for scheduling.

   707   CMSStats      _stats;

   708 

   709   enum CMS_op_type {

   710     CMS_op_checkpointRootsInitial,

   711     CMS_op_checkpointRootsFinal

   712   };

   713 

   714   void do_CMS_operation(CMS_op_type op, GCCause::Cause gc_cause);

   715   bool stop_world_and_do(CMS_op_type op);

   716 

   717   OopTaskQueueSet* task_queues() { return _task_queues; }

   718   int*             hash_seed(int i) { return &_hash_seed[i]; }

   719   YieldingFlexibleWorkGang* conc_workers() { return _conc_workers; }

   720 

   721   // Support for parallelizing Eden rescan in CMS remark phase

   722   void sample_eden(); // ... sample Eden space top

   723 

   724  private:

   725   // Support for parallelizing young gen rescan in CMS remark phase

   726   ParNewGeneration* _young_gen;  // the younger gen

   727 

   728   HeapWord** _top_addr;    // ... Top of Eden

   729   HeapWord** _end_addr;    // ... End of Eden

   730   Mutex*     _eden_chunk_lock;

   731   HeapWord** _eden_chunk_array; // ... Eden partitioning array

   732   size_t     _eden_chunk_index; // ... top (exclusive) of array

   733   size_t     _eden_chunk_capacity;  // ... max entries in array

   734 

   735   // Support for parallelizing survivor space rescan

   736   HeapWord** _survivor_chunk_array;

   737   size_t     _survivor_chunk_index;

   738   size_t     _survivor_chunk_capacity;

   739   size_t*    _cursor;

   740   ChunkArray* _survivor_plab_array;

   741 

   742   // A bounded minimum size of PLABs, should not return too small values since

   743   // this will affect the size of the data structures used for parallel young gen rescan

   744   size_t plab_sample_minimum_size();

   745 

   746   // Support for marking stack overflow handling

   747   bool take_from_overflow_list(size_t num, CMSMarkStack* to_stack);

   748   bool par_take_from_overflow_list(size_t num,

   749                                    OopTaskQueue* to_work_q,

   750                                    int no_of_gc_threads);

   751   void push_on_overflow_list(oop p);

   752   void par_push_on_overflow_list(oop p);

   753   // The following is, obviously, not, in general, "MT-stable"

   754   bool overflow_list_is_empty() const;

   755 

   756   void preserve_mark_if_necessary(oop p);

   757   void par_preserve_mark_if_necessary(oop p);

   758   void preserve_mark_work(oop p, markOop m);

   759   void restore_preserved_marks_if_any();

   760   NOT_PRODUCT(bool no_preserved_marks() const;)

   761   // In support of testing overflow code

   762   NOT_PRODUCT(int _overflow_counter;)

   763   NOT_PRODUCT(bool simulate_overflow();)       // Sequential

   764   NOT_PRODUCT(bool par_simulate_overflow();)   // MT version

   765 

   766   // CMS work methods

   767   void checkpointRootsInitialWork(); // Initial checkpoint work

   768 

   769   // A return value of false indicates failure due to stack overflow

   770   bool markFromRootsWork();  // Concurrent marking work

   771 

   772  public:   // FIX ME!!! only for testing

   773   bool do_marking_st();      // Single-threaded marking

   774   bool do_marking_mt();      // Multi-threaded  marking

   775 

   776  private:

   777 

   778   // Concurrent precleaning work

   779   size_t preclean_mod_union_table(ConcurrentMarkSweepGeneration* gen,

   780                                   ScanMarkedObjectsAgainCarefullyClosure* cl);

   781   size_t preclean_card_table(ConcurrentMarkSweepGeneration* gen,

   782                              ScanMarkedObjectsAgainCarefullyClosure* cl);

   783   // Does precleaning work, returning a quantity indicative of

   784   // the amount of "useful work" done.

   785   size_t preclean_work(bool clean_refs, bool clean_survivors);

   786   void preclean_klasses(MarkRefsIntoAndScanClosure* cl, Mutex* freelistLock);

   787   void abortable_preclean(); // Preclean while looking for possible abort

   788   void initialize_sequential_subtasks_for_young_gen_rescan(int i);

   789   // Helper function for above; merge-sorts the per-thread plab samples

   790   void merge_survivor_plab_arrays(ContiguousSpace* surv, int no_of_gc_threads);

   791   // Resets (i.e. clears) the per-thread plab sample vectors

   792   void reset_survivor_plab_arrays();

   793 

   794   // Final (second) checkpoint work

   795   void checkpointRootsFinalWork();

   796   // Work routine for parallel version of remark

   797   void do_remark_parallel();

   798   // Work routine for non-parallel version of remark

   799   void do_remark_non_parallel();

   800   // Reference processing work routine (during second checkpoint)

   801   void refProcessingWork();

   802 

   803   // Concurrent sweeping work

   804   void sweepWork(ConcurrentMarkSweepGeneration* gen);

   805 

   806   // (Concurrent) resetting of support data structures

   807   void reset(bool concurrent);

   808 

   809   // Clear _expansion_cause fields of constituent generations

   810   void clear_expansion_cause();

   811 

   812   // An auxiliary method used to record the ends of

   813   // used regions of each generation to limit the extent of sweep

   814   void save_sweep_limits();

   815 

   816   // A work method used by the foreground collector to do

   817   // a mark-sweep-compact.

   818   void do_compaction_work(bool clear_all_soft_refs);

   819 

   820   // Work methods for reporting concurrent mode interruption or failure

   821   bool is_external_interruption();

   822   void report_concurrent_mode_interruption();

   823 

   824   // If the background GC is active, acquire control from the background

   825   // GC and do the collection.

   826   void acquire_control_and_collect(bool   full, bool clear_all_soft_refs);

   827 

   828   // For synchronizing passing of control from background to foreground

   829   // GC.  waitForForegroundGC() is called by the background

   830   // collector.  It if had to wait for a foreground collection,

   831   // it returns true and the background collection should assume

   832   // that the collection was finished by the foreground

   833   // collector.

   834   bool waitForForegroundGC();

   835 

   836   size_t block_size_using_printezis_bits(HeapWord* addr) const;

   837   size_t block_size_if_printezis_bits(HeapWord* addr) const;

   838   HeapWord* next_card_start_after_block(HeapWord* addr) const;

   839 

   840   void setup_cms_unloading_and_verification_state();

   841  public:

   842   CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,

   843                CardTableRS*                   ct,

   844                ConcurrentMarkSweepPolicy*     cp);

   845   ConcurrentMarkSweepThread* cmsThread() { return _cmsThread; }

   846 

   847   ReferenceProcessor* ref_processor() { return _ref_processor; }

   848   void ref_processor_init();

   849 

   850   Mutex* bitMapLock()        const { return _markBitMap.lock();    }

   851   static CollectorState abstract_state() { return _collectorState;  }

   852 

   853   bool should_abort_preclean() const; // Whether preclean should be aborted.

   854   size_t get_eden_used() const;

   855   size_t get_eden_capacity() const;

   856 

   857   ConcurrentMarkSweepGeneration* cmsGen() { return _cmsGen; }

   858 

   859   // Locking checks

   860   NOT_PRODUCT(static bool have_cms_token();)

   861 

   862   bool shouldConcurrentCollect();

   863 

   864   void collect(bool   full,

   865                bool   clear_all_soft_refs,

   866                size_t size,

   867                bool   tlab);

   868   void collect_in_background(GCCause::Cause cause);

   869 

   870   // In support of ExplicitGCInvokesConcurrent

   871   static void request_full_gc(unsigned int full_gc_count, GCCause::Cause cause);

   872   // Should we unload classes in a particular concurrent cycle?

   873   bool should_unload_classes() const {

   874     return _should_unload_classes;

   875   }

   876   void update_should_unload_classes();

   877 

   878   void direct_allocated(HeapWord* start, size_t size);

   879 

   880   // Object is dead if not marked and current phase is sweeping.

   881   bool is_dead_obj(oop obj) const;

   882 

   883   // After a promotion (of "start"), do any necessary marking.

   884   // If "par", then it's being done by a parallel GC thread.

   885   // The last two args indicate if we need precise marking

   886   // and if so the size of the object so it can be dirtied

   887   // in its entirety.

   888   void promoted(bool par, HeapWord* start,

   889                 bool is_obj_array, size_t obj_size);

   890 

   891   void getFreelistLocks() const;

   892   void releaseFreelistLocks() const;

   893   bool haveFreelistLocks() const;

   894 

   895   // Adjust size of underlying generation

   896   void compute_new_size();

   897 

   898   // GC prologue and epilogue

   899   void gc_prologue(bool full);

   900   void gc_epilogue(bool full);

   901 

   902   jlong time_of_last_gc(jlong now) {

   903     if (_collectorState <= Idling) {

   904       // gc not in progress

   905       return _time_of_last_gc;

   906     } else {

   907       // collection in progress

   908       return now;

   909     }

   910   }

   911 

   912   // Support for parallel remark of survivor space

   913   void* get_data_recorder(int thr_num);

   914   void sample_eden_chunk();

   915 

   916   CMSBitMap* markBitMap()  { return &_markBitMap; }

   917   void directAllocated(HeapWord* start, size_t size);

   918 

   919   // Main CMS steps and related support

   920   void checkpointRootsInitial();

   921   bool markFromRoots();  // a return value of false indicates failure

   922                          // due to stack overflow

   923   void preclean();

   924   void checkpointRootsFinal();

   925   void sweep();

   926 

   927   // Check that the currently executing thread is the expected

   928   // one (foreground collector or background collector).

   929   static void check_correct_thread_executing() PRODUCT_RETURN;

   930 

   931   bool is_cms_reachable(HeapWord* addr);

   932 

   933   // Performance Counter Support

   934   CollectorCounters* counters()    { return _gc_counters; }

   935 

   936   // Timer stuff

   937   void    startTimer() { assert(!_timer.is_active(), "Error"); _timer.start();   }

   938   void    stopTimer()  { assert( _timer.is_active(), "Error"); _timer.stop();    }

   939   void    resetTimer() { assert(!_timer.is_active(), "Error"); _timer.reset();   }

   940   double  timerValue() { assert(!_timer.is_active(), "Error"); return _timer.seconds(); }

   941 

   942   int  yields()          { return _numYields; }

   943   void resetYields()     { _numYields = 0;    }

   944   void incrementYields() { _numYields++;      }

   945   void resetNumDirtyCards()               { _numDirtyCards = 0; }

   946   void incrementNumDirtyCards(size_t num) { _numDirtyCards += num; }

   947   size_t  numDirtyCards()                 { return _numDirtyCards; }

   948 

   949   static bool foregroundGCShouldWait() { return _foregroundGCShouldWait; }

   950   static void set_foregroundGCShouldWait(bool v) { _foregroundGCShouldWait = v; }

   951   static bool foregroundGCIsActive() { return _foregroundGCIsActive; }

   952   static void set_foregroundGCIsActive(bool v) { _foregroundGCIsActive = v; }

   953   size_t sweep_count() const             { return _sweep_count; }

   954   void   increment_sweep_count()         { _sweep_count++; }

   955 

   956   // Timers/stats for gc scheduling and incremental mode pacing.

   957   CMSStats& stats() { return _stats; }

   958 

   959   // Adaptive size policy

   960   AdaptiveSizePolicy* size_policy();

   961 

   962   static void print_on_error(outputStream* st);

   963 

   964   // Debugging

   965   void verify();

   966   bool verify_after_remark(bool silent = VerifySilently);

   967   void verify_ok_to_terminate() const PRODUCT_RETURN;

   968   void verify_work_stacks_empty() const PRODUCT_RETURN;

   969   void verify_overflow_empty() const PRODUCT_RETURN;

   970 

   971   // Convenience methods in support of debugging

   972   static const size_t skip_header_HeapWords() PRODUCT_RETURN0;

   973   HeapWord* block_start(const void* p) const PRODUCT_RETURN0;

   974 

   975   // Accessors

   976   CMSMarkStack* verification_mark_stack() { return &_markStack; }

   977   CMSBitMap*    verification_mark_bm()    { return &_verification_mark_bm; }

   978 

   979   // Initialization errors

   980   bool completed_initialization() { return _completed_initialization; }

   981 

   982   void print_eden_and_survivor_chunk_arrays();

   983 };

   984 

   985 class CMSExpansionCause : public AllStatic  {

   986  public:

   987   enum Cause {

   988     _no_expansion,

   989     _satisfy_free_ratio,

   990     _satisfy_promotion,

   991     _satisfy_allocation,

   992     _allocate_par_lab,

   993     _allocate_par_spooling_space,

   994     _adaptive_size_policy

   995   };

   996   // Return a string describing the cause of the expansion.

   997   static const char* to_string(CMSExpansionCause::Cause cause);

   998 };

   999 

  1000 class ConcurrentMarkSweepGeneration: public CardGeneration {

  1001   friend class VMStructs;

  1002   friend class ConcurrentMarkSweepThread;

  1003   friend class ConcurrentMarkSweep;

  1004   friend class CMSCollector;

  1005  protected:

  1006   static CMSCollector*       _collector; // the collector that collects us

  1007   CompactibleFreeListSpace*  _cmsSpace;  // underlying space (only one for now)

  1008 

  1009   // Performance Counters

  1010   GenerationCounters*      _gen_counters;

  1011   GSpaceCounters*          _space_counters;

  1012 

  1013   // Words directly allocated, used by CMSStats.

  1014   size_t _direct_allocated_words;

  1015 

  1016   // Non-product stat counters

  1017   NOT_PRODUCT(

  1018     size_t _numObjectsPromoted;

  1019     size_t _numWordsPromoted;

  1020     size_t _numObjectsAllocated;

  1021     size_t _numWordsAllocated;

  1022   )

  1023 

  1024   // Used for sizing decisions

  1025   bool _incremental_collection_failed;

  1026   bool incremental_collection_failed() {

  1027     return _incremental_collection_failed;

  1028   }

  1029   void set_incremental_collection_failed() {

  1030     _incremental_collection_failed = true;

  1031   }

  1032   void clear_incremental_collection_failed() {

  1033     _incremental_collection_failed = false;

  1034   }

  1035 

  1036   // accessors

  1037   void set_expansion_cause(CMSExpansionCause::Cause v) { _expansion_cause = v;}

  1038   CMSExpansionCause::Cause expansion_cause() const { return _expansion_cause; }

  1039 

  1040   // Accessing spaces

  1041   CompactibleSpace* space() const { return (CompactibleSpace*)_cmsSpace; }

  1042 

  1043  private:

  1044   // For parallel young-gen GC support.

  1045   CMSParGCThreadState** _par_gc_thread_states;

  1046 

  1047   // Reason generation was expanded

  1048   CMSExpansionCause::Cause _expansion_cause;

  1049 

  1050   // In support of MinChunkSize being larger than min object size

  1051   const double _dilatation_factor;

  1052 

  1053   // True if a compacting collection was done.

  1054   bool _did_compact;

  1055   bool did_compact() { return _did_compact; }

  1056 

  1057   // Fraction of current occupancy at which to start a CMS collection which

  1058   // will collect this generation (at least).

  1059   double _initiating_occupancy;

  1060 

  1061  protected:

  1062   // Shrink generation by specified size (returns false if unable to shrink)

  1063   void shrink_free_list_by(size_t bytes);

  1064 

  1065   // Update statistics for GC

  1066   virtual void update_gc_stats(int level, bool full);

  1067 

  1068   // Maximum available space in the generation (including uncommitted)

  1069   // space.

  1070   size_t max_available() const;

  1071 

  1072   // getter and initializer for _initiating_occupancy field.

  1073   double initiating_occupancy() const { return _initiating_occupancy; }

  1074   void   init_initiating_occupancy(intx io, uintx tr);

  1075 

  1076   void expand_for_gc_cause(size_t bytes, size_t expand_bytes, CMSExpansionCause::Cause cause);

  1077 

  1078   void assert_correct_size_change_locking();

  1079 

  1080  public:

  1081   ConcurrentMarkSweepGeneration(ReservedSpace rs, size_t initial_byte_size,

  1082                                 int level, CardTableRS* ct,

  1083                                 bool use_adaptive_freelists,

  1084                                 FreeBlockDictionary<FreeChunk>::DictionaryChoice);

  1085 

  1086   // Accessors

  1087   CMSCollector* collector() const { return _collector; }

  1088   static void set_collector(CMSCollector* collector) {

  1089     assert(_collector == NULL, "already set");

  1090     _collector = collector;

  1091   }

  1092   CompactibleFreeListSpace*  cmsSpace() const { return _cmsSpace;  }

  1093 

  1094   Mutex* freelistLock() const;

  1095 

  1096   virtual Generation::Name kind() { return Generation::ConcurrentMarkSweep; }

  1097 

  1098   void set_did_compact(bool v) { _did_compact = v; }

  1099 

  1100   bool refs_discovery_is_atomic() const { return false; }

  1101   bool refs_discovery_is_mt()     const {

  1102     // Note: CMS does MT-discovery during the parallel-remark

  1103     // phases. Use ReferenceProcessorMTMutator to make refs

  1104     // discovery MT-safe during such phases or other parallel

  1105     // discovery phases in the future. This may all go away

  1106     // if/when we decide that refs discovery is sufficiently

  1107     // rare that the cost of the CAS's involved is in the

  1108     // noise. That's a measurement that should be done, and

  1109     // the code simplified if that turns out to be the case.

  1110     return ConcGCThreads > 1;

  1111   }

  1112 

  1113   // Override

  1114   virtual void ref_processor_init();

  1115 

  1116   void clear_expansion_cause() { _expansion_cause = CMSExpansionCause::_no_expansion; }

  1117 

  1118   // Space enquiries

  1119   double occupancy() const { return ((double)used())/((double)capacity()); }

  1120   size_t contiguous_available() const;

  1121   size_t unsafe_max_alloc_nogc() const;

  1122 

  1123   // over-rides

  1124   MemRegion used_region_at_save_marks() const;

  1125 

  1126   // Does a "full" (forced) collection invoked on this generation collect

  1127   // all younger generations as well? Note that the second conjunct is a

  1128   // hack to allow the collection of the younger gen first if the flag is

  1129   // set.

  1130   virtual bool full_collects_younger_generations() const {

  1131     return !ScavengeBeforeFullGC;

  1132   }

  1133 

  1134   // Adjust quantities in the generation affected by