1 /*

     2  * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  *

    23  */

    24 

    25 #include "precompiled.hpp"

    26 #include "classfile/classLoaderDataGraph.hpp"

    27 #include "classfile/systemDictionary.hpp"

    28 #include "code/codeCache.hpp"

    29 #include "gc/cms/cmsGCStats.hpp"

    30 #include "gc/cms/cmsHeap.hpp"

    31 #include "gc/cms/cmsOopClosures.inline.hpp"

    32 #include "gc/cms/cmsVMOperations.hpp"

    33 #include "gc/cms/compactibleFreeListSpace.hpp"

    34 #include "gc/cms/concurrentMarkSweepGeneration.inline.hpp"

    35 #include "gc/cms/concurrentMarkSweepThread.hpp"

    36 #include "gc/cms/parNewGeneration.hpp"

    37 #include "gc/cms/promotionInfo.inline.hpp"

    38 #include "gc/serial/genMarkSweep.hpp"

    39 #include "gc/serial/tenuredGeneration.hpp"

    40 #include "gc/shared/adaptiveSizePolicy.hpp"

    41 #include "gc/shared/cardGeneration.inline.hpp"

    42 #include "gc/shared/cardTableRS.hpp"

    43 #include "gc/shared/collectedHeap.inline.hpp"

    44 #include "gc/shared/collectorCounters.hpp"

    45 #include "gc/shared/gcLocker.hpp"

    46 #include "gc/shared/gcPolicyCounters.hpp"

    47 #include "gc/shared/gcTimer.hpp"

    48 #include "gc/shared/gcTrace.hpp"

    49 #include "gc/shared/gcTraceTime.inline.hpp"

    50 #include "gc/shared/genCollectedHeap.hpp"

    51 #include "gc/shared/genOopClosures.inline.hpp"

    52 #include "gc/shared/isGCActiveMark.hpp"

    53 #include "gc/shared/owstTaskTerminator.hpp"

    54 #include "gc/shared/referencePolicy.hpp"

    55 #include "gc/shared/referenceProcessorPhaseTimes.hpp"

    56 #include "gc/shared/space.inline.hpp"

    57 #include "gc/shared/strongRootsScope.hpp"

    58 #include "gc/shared/taskqueue.inline.hpp"

    59 #include "gc/shared/weakProcessor.hpp"

    60 #include "gc/shared/workerPolicy.hpp"

    61 #include "logging/log.hpp"

    62 #include "logging/logStream.hpp"

    63 #include "memory/allocation.hpp"

    64 #include "memory/binaryTreeDictionary.inline.hpp"

    65 #include "memory/iterator.inline.hpp"

    66 #include "memory/padded.hpp"

    67 #include "memory/resourceArea.hpp"

    68 #include "memory/universe.hpp"

    69 #include "oops/access.inline.hpp"

    70 #include "oops/oop.inline.hpp"

    71 #include "prims/jvmtiExport.hpp"

    72 #include "runtime/atomic.hpp"

    73 #include "runtime/flags/flagSetting.hpp"

    74 #include "runtime/globals_extension.hpp"

    75 #include "runtime/handles.inline.hpp"

    76 #include "runtime/java.hpp"

    77 #include "runtime/orderAccess.hpp"

    78 #include "runtime/timer.hpp"

    79 #include "runtime/vmThread.hpp"

    80 #include "services/memoryService.hpp"

    81 #include "services/runtimeService.hpp"

    82 #include "utilities/align.hpp"

    83 #include "utilities/stack.inline.hpp"

    84 #if INCLUDE_JVMCI

    85 #include "jvmci/jvmci.hpp"

    86 #endif

    87 

    88 // statics

    89 CMSCollector* ConcurrentMarkSweepGeneration::_collector = NULL;

    90 bool CMSCollector::_full_gc_requested = false;

    91 GCCause::Cause CMSCollector::_full_gc_cause = GCCause::_no_gc;

    92 

    93 //////////////////////////////////////////////////////////////////

    94 // In support of CMS/VM thread synchronization

    95 //////////////////////////////////////////////////////////////////

    96 // We split use of the CGC_lock into 2 "levels".

    97 // The low-level locking is of the usual CGC_lock monitor. We introduce

    98 // a higher level "token" (hereafter "CMS token") built on top of the

    99 // low level monitor (hereafter "CGC lock").

   100 // The token-passing protocol gives priority to the VM thread. The

   101 // CMS-lock doesn't provide any fairness guarantees, but clients

   102 // should ensure that it is only held for very short, bounded

   103 // durations.

   104 //

   105 // When either of the CMS thread or the VM thread is involved in

   106 // collection operations during which it does not want the other

   107 // thread to interfere, it obtains the CMS token.

   108 //

   109 // If either thread tries to get the token while the other has

   110 // it, that thread waits. However, if the VM thread and CMS thread

   111 // both want the token, then the VM thread gets priority while the

   112 // CMS thread waits. This ensures, for instance, that the "concurrent"

   113 // phases of the CMS thread's work do not block out the VM thread

   114 // for long periods of time as the CMS thread continues to hog

   115 // the token. (See bug 4616232).

   116 //

   117 // The baton-passing functions are, however, controlled by the

   118 // flags _foregroundGCShouldWait and _foregroundGCIsActive,

   119 // and here the low-level CMS lock, not the high level token,

   120 // ensures mutual exclusion.

   121 //

   122 // Two important conditions that we have to satisfy:

   123 // 1. if a thread does a low-level wait on the CMS lock, then it

   124 //    relinquishes the CMS token if it were holding that token

   125 //    when it acquired the low-level CMS lock.

   126 // 2. any low-level notifications on the low-level lock

   127 //    should only be sent when a thread has relinquished the token.

   128 //

   129 // In the absence of either property, we'd have potential deadlock.

   130 //

   131 // We protect each of the CMS (concurrent and sequential) phases

   132 // with the CMS _token_, not the CMS _lock_.

   133 //

   134 // The only code protected by CMS lock is the token acquisition code

   135 // itself, see ConcurrentMarkSweepThread::[de]synchronize(), and the

   136 // baton-passing code.

   137 //

   138 // Unfortunately, i couldn't come up with a good abstraction to factor and

   139 // hide the naked CGC_lock manipulation in the baton-passing code

   140 // further below. That's something we should try to do. Also, the proof

   141 // of correctness of this 2-level locking scheme is far from obvious,

   142 // and potentially quite slippery. We have an uneasy suspicion, for instance,

   143 // that there may be a theoretical possibility of delay/starvation in the

   144 // low-level lock/wait/notify scheme used for the baton-passing because of

   145 // potential interference with the priority scheme embodied in the

   146 // CMS-token-passing protocol. See related comments at a CGC_lock->wait()

   147 // invocation further below and marked with "XXX 20011219YSR".

   148 // Indeed, as we note elsewhere, this may become yet more slippery

   149 // in the presence of multiple CMS and/or multiple VM threads. XXX

   150 

   151 class CMSTokenSync: public StackObj {

   152  private:

   153   bool _is_cms_thread;

   154  public:

   155   CMSTokenSync(bool is_cms_thread):

   156     _is_cms_thread(is_cms_thread) {

   157     assert(is_cms_thread == Thread::current()->is_ConcurrentGC_thread(),

   158            "Incorrect argument to constructor");

   159     ConcurrentMarkSweepThread::synchronize(_is_cms_thread);

   160   }

   161 

   162   ~CMSTokenSync() {

   163     assert(_is_cms_thread ?

   164              ConcurrentMarkSweepThread::cms_thread_has_cms_token() :

   165              ConcurrentMarkSweepThread::vm_thread_has_cms_token(),

   166           "Incorrect state");

   167     ConcurrentMarkSweepThread::desynchronize(_is_cms_thread);

   168   }

   169 };

   170 

   171 // Convenience class that does a CMSTokenSync, and then acquires

   172 // upto three locks.

   173 class CMSTokenSyncWithLocks: public CMSTokenSync {

   174  private:

   175   // Note: locks are acquired in textual declaration order

   176   // and released in the opposite order

   177   MutexLocker _locker1, _locker2, _locker3;

   178  public:

   179   CMSTokenSyncWithLocks(bool is_cms_thread, Mutex* mutex1,

   180                         Mutex* mutex2 = NULL, Mutex* mutex3 = NULL):

   181     CMSTokenSync(is_cms_thread),

   182     _locker1(mutex1, Mutex::_no_safepoint_check_flag),

   183     _locker2(mutex2, Mutex::_no_safepoint_check_flag),

   184     _locker3(mutex3, Mutex::_no_safepoint_check_flag)

   185   { }

   186 };

   187 

   188 

   189 //////////////////////////////////////////////////////////////////

   190 //  Concurrent Mark-Sweep Generation /////////////////////////////

   191 //////////////////////////////////////////////////////////////////

   192 

   193 NOT_PRODUCT(CompactibleFreeListSpace* debug_cms_space;)

   194 

   195 // This struct contains per-thread things necessary to support parallel

   196 // young-gen collection.

   197 class CMSParGCThreadState: public CHeapObj<mtGC> {

   198  public:

   199   CompactibleFreeListSpaceLAB lab;

   200   PromotionInfo promo;

   201 

   202   // Constructor.

   203   CMSParGCThreadState(CompactibleFreeListSpace* cfls) : lab(cfls) {

   204     promo.setSpace(cfls);

   205   }

   206 };

   207 

   208 ConcurrentMarkSweepGeneration::ConcurrentMarkSweepGeneration(

   209      ReservedSpace rs,

   210      size_t initial_byte_size,

   211      size_t min_byte_size,

   212      size_t max_byte_size,

   213      CardTableRS* ct) :

   214   CardGeneration(rs, initial_byte_size, ct),

   215   _dilatation_factor(((double)MinChunkSize)/((double)(CollectedHeap::min_fill_size()))),

   216   _did_compact(false)

   217 {

   218   HeapWord* bottom = (HeapWord*) _virtual_space.low();

   219   HeapWord* end    = (HeapWord*) _virtual_space.high();

   220 

   221   _direct_allocated_words = 0;

   222   NOT_PRODUCT(

   223     _numObjectsPromoted = 0;

   224     _numWordsPromoted = 0;

   225     _numObjectsAllocated = 0;

   226     _numWordsAllocated = 0;

   227   )

   228 

   229   _cmsSpace = new CompactibleFreeListSpace(_bts, MemRegion(bottom, end));

   230   NOT_PRODUCT(debug_cms_space = _cmsSpace;)

   231   _cmsSpace->_old_gen = this;

   232 

   233   _gc_stats = new CMSGCStats();

   234 

   235   // Verify the assumption that FreeChunk::_prev and OopDesc::_klass

   236   // offsets match. The ability to tell free chunks from objects

   237   // depends on this property.

   238   debug_only(

   239     FreeChunk* junk = NULL;

   240     assert(UseCompressedClassPointers ||

   241            junk->prev_addr() == (void*)(oop(junk)->klass_addr()),

   242            "Offset of FreeChunk::_prev within FreeChunk must match"

   243            "  that of OopDesc::_klass within OopDesc");

   244   )

   245 

   246   _par_gc_thread_states = NEW_C_HEAP_ARRAY(CMSParGCThreadState*, ParallelGCThreads, mtGC);

   247   for (uint i = 0; i < ParallelGCThreads; i++) {

   248     _par_gc_thread_states[i] = new CMSParGCThreadState(cmsSpace());

   249   }

   250 

   251   _incremental_collection_failed = false;

   252   // The "dilatation_factor" is the expansion that can occur on

   253   // account of the fact that the minimum object size in the CMS

   254   // generation may be larger than that in, say, a contiguous young

   255   //  generation.

   256   // Ideally, in the calculation below, we'd compute the dilatation

   257   // factor as: MinChunkSize/(promoting_gen's min object size)

   258   // Since we do not have such a general query interface for the

   259   // promoting generation, we'll instead just use the minimum

   260   // object size (which today is a header's worth of space);

   261   // note that all arithmetic is in units of HeapWords.

   262   assert(MinChunkSize >= CollectedHeap::min_fill_size(), "just checking");

   263   assert(_dilatation_factor >= 1.0, "from previous assert");

   264 

   265   initialize_performance_counters(min_byte_size, max_byte_size);

   266 }

   267 

   268 

   269 // The field "_initiating_occupancy" represents the occupancy percentage

   270 // at which we trigger a new collection cycle.  Unless explicitly specified

   271 // via CMSInitiatingOccupancyFraction (argument "io" below), it

   272 // is calculated by:

   273 //

   274 //   Let "f" be MinHeapFreeRatio in

   275 //

   276 //    _initiating_occupancy = 100-f +

   277 //                           f * (CMSTriggerRatio/100)

   278 //   where CMSTriggerRatio is the argument "tr" below.

   279 //

   280 // That is, if we assume the heap is at its desired maximum occupancy at the

   281 // end of a collection, we let CMSTriggerRatio of the (purported) free

   282 // space be allocated before initiating a new collection cycle.

   283 //

   284 void ConcurrentMarkSweepGeneration::init_initiating_occupancy(intx io, uintx tr) {

   285   assert(io <= 100 && tr <= 100, "Check the arguments");

   286   if (io >= 0) {

   287     _initiating_occupancy = (double)io / 100.0;

   288   } else {

   289     _initiating_occupancy = ((100 - MinHeapFreeRatio) +

   290                              (double)(tr * MinHeapFreeRatio) / 100.0)

   291                             / 100.0;

   292   }

   293 }

   294 

   295 void ConcurrentMarkSweepGeneration::ref_processor_init() {

   296   assert(collector() != NULL, "no collector");

   297   collector()->ref_processor_init();

   298 }

   299 

   300 void CMSCollector::ref_processor_init() {

   301   if (_ref_processor == NULL) {

   302     // Allocate and initialize a reference processor

   303     _ref_processor =

   304       new ReferenceProcessor(&_span_based_discoverer,

   305                              (ParallelGCThreads > 1) && ParallelRefProcEnabled, // mt processing

   306                              ParallelGCThreads,                      // mt processing degree

   307                              _cmsGen->refs_discovery_is_mt(),        // mt discovery

   308                              MAX2(ConcGCThreads, ParallelGCThreads), // mt discovery degree

   309                              _cmsGen->refs_discovery_is_atomic(),    // discovery is not atomic

   310                              &_is_alive_closure,                     // closure for liveness info

   311                              false);                                 // disable adjusting number of processing threads

   312     // Initialize the _ref_processor field of CMSGen

   313     _cmsGen->set_ref_processor(_ref_processor);

   314 

   315   }

   316 }

   317 

   318 AdaptiveSizePolicy* CMSCollector::size_policy() {

   319   return CMSHeap::heap()->size_policy();

   320 }

   321 

   322 void ConcurrentMarkSweepGeneration::initialize_performance_counters(size_t min_old_size,

   323                                                                     size_t max_old_size) {

   324 

   325   const char* gen_name = "old";

   326   // Generation Counters - generation 1, 1 subspace

   327   _gen_counters = new GenerationCounters(gen_name, 1, 1,

   328       min_old_size, max_old_size, &_virtual_space);

   329 

   330   _space_counters = new GSpaceCounters(gen_name, 0,

   331                                        _virtual_space.reserved_size(),

   332                                        this, _gen_counters);

   333 }

   334 

   335 CMSStats::CMSStats(ConcurrentMarkSweepGeneration* cms_gen, unsigned int alpha):

   336   _cms_gen(cms_gen)

   337 {

   338   assert(alpha <= 100, "bad value");

   339   _saved_alpha = alpha;

   340 

   341   // Initialize the alphas to the bootstrap value of 100.

   342   _gc0_alpha = _cms_alpha = 100;

   343 

   344   _cms_begin_time.update();

   345   _cms_end_time.update();

   346 

   347   _gc0_duration = 0.0;

   348   _gc0_period = 0.0;

   349   _gc0_promoted = 0;

   350 

   351   _cms_duration = 0.0;

   352   _cms_period = 0.0;

   353   _cms_allocated = 0;

   354 

   355   _cms_used_at_gc0_begin = 0;

   356   _cms_used_at_gc0_end = 0;

   357   _allow_duty_cycle_reduction = false;

   358   _valid_bits = 0;

   359 }

   360 

   361 double CMSStats::cms_free_adjustment_factor(size_t free) const {

   362   // TBD: CR 6909490

   363   return 1.0;

   364 }

   365 

   366 void CMSStats::adjust_cms_free_adjustment_factor(bool fail, size_t free) {

   367 }

   368 

   369 // If promotion failure handling is on use

   370 // the padded average size of the promotion for each

   371 // young generation collection.

   372 double CMSStats::time_until_cms_gen_full() const {

   373   size_t cms_free = _cms_gen->cmsSpace()->free();

   374   CMSHeap* heap = CMSHeap::heap();

   375   size_t expected_promotion = MIN2(heap->young_gen()->capacity(),

   376                                    (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average());

   377   if (cms_free > expected_promotion) {

   378     // Start a cms collection if there isn't enough space to promote

   379     // for the next young collection.  Use the padded average as

   380     // a safety factor.

   381     cms_free -= expected_promotion;

   382 

   383     // Adjust by the safety factor.

   384     double cms_free_dbl = (double)cms_free;

   385     double cms_adjustment = (100.0 - CMSIncrementalSafetyFactor) / 100.0;

   386     // Apply a further correction factor which tries to adjust

   387     // for recent occurance of concurrent mode failures.

   388     cms_adjustment = cms_adjustment * cms_free_adjustment_factor(cms_free);

   389     cms_free_dbl = cms_free_dbl * cms_adjustment;

   390 

   391     log_trace(gc)("CMSStats::time_until_cms_gen_full: cms_free " SIZE_FORMAT " expected_promotion " SIZE_FORMAT,

   392                   cms_free, expected_promotion);

   393     log_trace(gc)("  cms_free_dbl %f cms_consumption_rate %f", cms_free_dbl, cms_consumption_rate() + 1.0);

   394     // Add 1 in case the consumption rate goes to zero.

   395     return cms_free_dbl / (cms_consumption_rate() + 1.0);

   396   }

   397   return 0.0;

   398 }

   399 

   400 // Compare the duration of the cms collection to the

   401 // time remaining before the cms generation is empty.

   402 // Note that the time from the start of the cms collection

   403 // to the start of the cms sweep (less than the total

   404 // duration of the cms collection) can be used.  This

   405 // has been tried and some applications experienced

   406 // promotion failures early in execution.  This was

   407 // possibly because the averages were not accurate

   408 // enough at the beginning.

   409 double CMSStats::time_until_cms_start() const {

   410   // We add "gc0_period" to the "work" calculation

   411   // below because this query is done (mostly) at the

   412   // end of a scavenge, so we need to conservatively

   413   // account for that much possible delay

   414   // in the query so as to avoid concurrent mode failures

   415   // due to starting the collection just a wee bit too

   416   // late.

   417   double work = cms_duration() + gc0_period();

   418   double deadline = time_until_cms_gen_full();

   419   // If a concurrent mode failure occurred recently, we want to be

   420   // more conservative and halve our expected time_until_cms_gen_full()

   421   if (work > deadline) {

   422     log_develop_trace(gc)("CMSCollector: collect because of anticipated promotion before full %3.7f + %3.7f > %3.7f ",

   423                           cms_duration(), gc0_period(), time_until_cms_gen_full());

   424     return 0.0;

   425   }

   426   return work - deadline;

   427 }

   428 

   429 #ifndef PRODUCT

   430 void CMSStats::print_on(outputStream *st) const {

   431   st->print(" gc0_alpha=%d,cms_alpha=%d", _gc0_alpha, _cms_alpha);

   432   st->print(",gc0_dur=%g,gc0_per=%g,gc0_promo=" SIZE_FORMAT,

   433                gc0_duration(), gc0_period(), gc0_promoted());

   434   st->print(",cms_dur=%g,cms_per=%g,cms_alloc=" SIZE_FORMAT,

   435             cms_duration(), cms_period(), cms_allocated());

   436   st->print(",cms_since_beg=%g,cms_since_end=%g",

   437             cms_time_since_begin(), cms_time_since_end());

   438   st->print(",cms_used_beg=" SIZE_FORMAT ",cms_used_end=" SIZE_FORMAT,

   439             _cms_used_at_gc0_begin, _cms_used_at_gc0_end);

   440 

   441   if (valid()) {

   442     st->print(",promo_rate=%g,cms_alloc_rate=%g",

   443               promotion_rate(), cms_allocation_rate());

   444     st->print(",cms_consumption_rate=%g,time_until_full=%g",

   445               cms_consumption_rate(), time_until_cms_gen_full());

   446   }

   447   st->cr();

   448 }

   449 #endif // #ifndef PRODUCT

   450 

   451 CMSCollector::CollectorState CMSCollector::_collectorState =

   452                              CMSCollector::Idling;

   453 bool CMSCollector::_foregroundGCIsActive = false;

   454 bool CMSCollector::_foregroundGCShouldWait = false;

   455 

   456 CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,

   457                            CardTableRS*                   ct):

   458   _overflow_list(NULL),

   459   _conc_workers(NULL),     // may be set later

   460   _completed_initialization(false),

   461   _collection_count_start(0),

   462   _should_unload_classes(CMSClassUnloadingEnabled),

   463   _concurrent_cycles_since_last_unload(0),

   464   _roots_scanning_options(GenCollectedHeap::SO_None),

   465   _verification_mark_bm(0, Mutex::leaf + 1, "CMS_verification_mark_bm_lock"),

   466   _verifying(false),

   467   _inter_sweep_estimate(CMS_SweepWeight, CMS_SweepPadding),

   468   _intra_sweep_estimate(CMS_SweepWeight, CMS_SweepPadding),

   469   _gc_tracer_cm(new (ResourceObj::C_HEAP, mtGC) CMSTracer()),

   470   _gc_timer_cm(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),

   471   _cms_start_registered(false),

   472   _cmsGen(cmsGen),

   473   // Adjust span to cover old (cms) gen

   474   _span(cmsGen->reserved()),

   475   _ct(ct),

   476   _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"),

   477   _modUnionTable((CardTable::card_shift - LogHeapWordSize),

   478                  -1 /* lock-free */, "No_lock" /* dummy */),

   479   _restart_addr(NULL),

   480   _ser_pmc_preclean_ovflw(0),

   481   _ser_pmc_remark_ovflw(0),

   482   _par_pmc_remark_ovflw(0),

   483   _ser_kac_preclean_ovflw(0),

   484   _ser_kac_ovflw(0),

   485   _par_kac_ovflw(0),

   486 #ifndef PRODUCT

   487   _num_par_pushes(0),

   488 #endif

   489   _span_based_discoverer(_span),

   490   _ref_processor(NULL),    // will be set later

   491   // Construct the is_alive_closure with _span & markBitMap

   492   _is_alive_closure(_span, &_markBitMap),

   493   _modUnionClosurePar(&_modUnionTable),

   494   _between_prologue_and_epilogue(false),

   495   _abort_preclean(false),

   496   _start_sampling(false),

   497   _stats(cmsGen),

   498   _eden_chunk_lock(new Mutex(Mutex::leaf + 1, "CMS_eden_chunk_lock", true,

   499                              //verify that this lock should be acquired with safepoint check.

   500                              Monitor::_safepoint_check_never)),

   501   _eden_chunk_array(NULL),     // may be set in ctor body

   502   _eden_chunk_index(0),        // -- ditto --

   503   _eden_chunk_capacity(0),     // -- ditto --

   504   _survivor_chunk_array(NULL), // -- ditto --

   505   _survivor_chunk_index(0),    // -- ditto --

   506   _survivor_chunk_capacity(0), // -- ditto --

   507   _survivor_plab_array(NULL)   // -- ditto --

   508 {

   509   // Now expand the span and allocate the collection support structures

   510   // (MUT, marking bit map etc.) to cover both generations subject to

   511   // collection.

   512 

   513   // For use by dirty card to oop closures.

   514   _cmsGen->cmsSpace()->set_collector(this);

   515 

   516   // Allocate MUT and marking bit map

   517   {

   518     MutexLocker x(_markBitMap.lock(), Mutex::_no_safepoint_check_flag);

   519     if (!_markBitMap.allocate(_span)) {

   520       log_warning(gc)("Failed to allocate CMS Bit Map");

   521       return;

   522     }

   523     assert(_markBitMap.covers(_span), "_markBitMap inconsistency?");

   524   }

   525   {

   526     _modUnionTable.allocate(_span);

   527     assert(_modUnionTable.covers(_span), "_modUnionTable inconsistency?");

   528   }

   529 

   530   if (!_markStack.allocate(MarkStackSize)) {

   531     log_warning(gc)("Failed to allocate CMS Marking Stack");

   532     return;

   533   }

   534 

   535   // Support for multi-threaded concurrent phases

   536   if (CMSConcurrentMTEnabled) {

   537     if (FLAG_IS_DEFAULT(ConcGCThreads)) {

   538       // just for now

   539       FLAG_SET_DEFAULT(ConcGCThreads, (ParallelGCThreads + 3) / 4);

   540     }

   541     if (ConcGCThreads > 1) {

   542       _conc_workers = new YieldingFlexibleWorkGang("CMS Thread",

   543                                  ConcGCThreads, true);

   544       if (_conc_workers == NULL) {

   545         log_warning(gc)("GC/CMS: _conc_workers allocation failure: forcing -CMSConcurrentMTEnabled");

   546         CMSConcurrentMTEnabled = false;

   547       } else {

   548         _conc_workers->initialize_workers();

   549       }

   550     } else {

   551       CMSConcurrentMTEnabled = false;

   552     }

   553   }

   554   if (!CMSConcurrentMTEnabled) {

   555     ConcGCThreads = 0;

   556   } else {

   557     // Turn off CMSCleanOnEnter optimization temporarily for

   558     // the MT case where it's not fixed yet; see 6178663.

   559     CMSCleanOnEnter = false;

   560   }

   561   assert((_conc_workers != NULL) == (ConcGCThreads > 1),

   562          "Inconsistency");

   563   log_debug(gc)("ConcGCThreads: %u", ConcGCThreads);

   564   log_debug(gc)("ParallelGCThreads: %u", ParallelGCThreads);

   565 

   566   // Parallel task queues; these are shared for the

   567   // concurrent and stop-world phases of CMS, but

   568   // are not shared with parallel scavenge (ParNew).

   569   {

   570     uint i;

   571     uint num_queues = MAX2(ParallelGCThreads, ConcGCThreads);

   572 

   573     if ((CMSParallelRemarkEnabled || CMSConcurrentMTEnabled

   574          || ParallelRefProcEnabled)

   575         && num_queues > 0) {

   576       _task_queues = new OopTaskQueueSet(num_queues);

   577       if (_task_queues == NULL) {

   578         log_warning(gc)("task_queues allocation failure.");

   579         return;

   580       }

   581       typedef Padded<OopTaskQueue> PaddedOopTaskQueue;

   582       for (i = 0; i < num_queues; i++) {

   583         PaddedOopTaskQueue *q = new PaddedOopTaskQueue();

   584         if (q == NULL) {

   585           log_warning(gc)("work_queue allocation failure.");

   586           return;

   587         }

   588         _task_queues->register_queue(i, q);

   589       }

   590       for (i = 0; i < num_queues; i++) {

   591         _task_queues->queue(i)->initialize();

   592       }

   593     }

   594   }

   595 

   596   _cmsGen ->init_initiating_occupancy(CMSInitiatingOccupancyFraction, CMSTriggerRatio);

   597 

   598   // Clip CMSBootstrapOccupancy between 0 and 100.

   599   _bootstrap_occupancy = CMSBootstrapOccupancy / 100.0;

   600 

   601   // Now tell CMS generations the identity of their collector

   602   ConcurrentMarkSweepGeneration::set_collector(this);

   603 

   604   // Create & start a CMS thread for this CMS collector

   605   _cmsThread = ConcurrentMarkSweepThread::start(this);

   606   assert(cmsThread() != NULL, "CMS Thread should have been created");

   607   assert(cmsThread()->collector() == this,

   608          "CMS Thread should refer to this gen");

   609   assert(CGC_lock != NULL, "Where's the CGC_lock?");

   610 

   611   // Support for parallelizing young gen rescan

   612   CMSHeap* heap = CMSHeap::heap();

   613   _young_gen = heap->young_gen();

   614   if (heap->supports_inline_contig_alloc()) {

   615     _top_addr = heap->top_addr();

   616     _end_addr = heap->end_addr();

   617     assert(_young_gen != NULL, "no _young_gen");

   618     _eden_chunk_index = 0;

   619     _eden_chunk_capacity = (_young_gen->max_capacity() + CMSSamplingGrain) / CMSSamplingGrain;

   620     _eden_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, _eden_chunk_capacity, mtGC);

   621   }

   622 

   623   // Support for parallelizing survivor space rescan

   624   if ((CMSParallelRemarkEnabled && CMSParallelSurvivorRemarkEnabled) || CMSParallelInitialMarkEnabled) {

   625     const size_t max_plab_samples =

   626       _young_gen->max_survivor_size() / (PLAB::min_size() * HeapWordSize);

   627 

   628     _survivor_plab_array  = NEW_C_HEAP_ARRAY(ChunkArray, ParallelGCThreads, mtGC);

   629     _survivor_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, max_plab_samples, mtGC);

   630     _cursor               = NEW_C_HEAP_ARRAY(size_t, ParallelGCThreads, mtGC);

   631     _survivor_chunk_capacity = max_plab_samples;

   632     for (uint i = 0; i < ParallelGCThreads; i++) {

   633       HeapWord** vec = NEW_C_HEAP_ARRAY(HeapWord*, max_plab_samples, mtGC);

   634       ChunkArray* cur = ::new (&_survivor_plab_array[i]) ChunkArray(vec, max_plab_samples);

   635       assert(cur->end() == 0, "Should be 0");

   636       assert(cur->array() == vec, "Should be vec");

   637       assert(cur->capacity() == max_plab_samples, "Error");

   638     }

   639   }

   640 

   641   NOT_PRODUCT(_overflow_counter = CMSMarkStackOverflowInterval;)

   642   _gc_counters = new CollectorCounters("CMS full collection pauses", 1);

   643   _cgc_counters = new CollectorCounters("CMS concurrent cycle pauses", 2);

   644   _completed_initialization = true;

   645   _inter_sweep_timer.start();  // start of time

   646 }

   647 

   648 const char* ConcurrentMarkSweepGeneration::name() const {

   649   return "concurrent mark-sweep generation";

   650 }

   651 void ConcurrentMarkSweepGeneration::update_counters() {

   652   if (UsePerfData) {

   653     _space_counters->update_all();

   654     _gen_counters->update_all();

   655   }

   656 }

   657 

   658 // this is an optimized version of update_counters(). it takes the

   659 // used value as a parameter rather than computing it.

   660 //

   661 void ConcurrentMarkSweepGeneration::update_counters(size_t used) {

   662   if (UsePerfData) {

   663     _space_counters->update_used(used);

   664     _space_counters->update_capacity();

   665     _gen_counters->update_all();

   666   }

   667 }

   668 

   669 void ConcurrentMarkSweepGeneration::print() const {

   670   Generation::print();

   671   cmsSpace()->print();

   672 }

   673 

   674 #ifndef PRODUCT

   675 void ConcurrentMarkSweepGeneration::print_statistics() {

   676   cmsSpace()->printFLCensus(0);

   677 }

   678 #endif

   679 

   680 size_t

   681 ConcurrentMarkSweepGeneration::contiguous_available() const {

   682   // dld proposes an improvement in precision here. If the committed

   683   // part of the space ends in a free block we should add that to

   684   // uncommitted size in the calculation below. Will make this

   685   // change later, staying with the approximation below for the

   686   // time being. -- ysr.

   687   return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());

   688 }

   689 

   690 size_t

   691 ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const {

   692   return _cmsSpace->max_alloc_in_words() * HeapWordSize;

   693 }

   694 

   695 size_t ConcurrentMarkSweepGeneration::used_stable() const {

   696   return cmsSpace()->used_stable();

   697 }

   698 

   699 size_t ConcurrentMarkSweepGeneration::max_available() const {

   700   return free() + _virtual_space.uncommitted_size();

   701 }

   702 

   703 bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {

   704   size_t available = max_available();

   705   size_t av_promo  = (size_t)gc_stats()->avg_promoted()->padded_average();

   706   bool   res = (available >= av_promo) || (available >= max_promotion_in_bytes);

   707   log_trace(gc, promotion)("CMS: promo attempt is%s safe: available(" SIZE_FORMAT ") %s av_promo(" SIZE_FORMAT "), max_promo(" SIZE_FORMAT ")",

   708                            res? "":" not", available, res? ">=":"<", av_promo, max_promotion_in_bytes);

   709   return res;

   710 }

   711 

   712 // At a promotion failure dump information on block layout in heap

   713 // (cms old generation).

   714 void ConcurrentMarkSweepGeneration::promotion_failure_occurred() {

   715   Log(gc, promotion) log;

   716   if (log.is_trace()) {

   717     LogStream ls(log.trace());

   718     cmsSpace()->dump_at_safepoint_with_locks(collector(), &ls);

   719   }

   720 }

   721 

   722 void ConcurrentMarkSweepGeneration::reset_after_compaction() {

   723   // Clear the promotion information.  These pointers can be adjusted

   724   // along with all the other pointers into the heap but

   725   // compaction is expected to be a rare event with

   726   // a heap using cms so don't do it without seeing the need.

   727   for (uint i = 0; i < ParallelGCThreads; i++) {

   728     _par_gc_thread_states[i]->promo.reset();

   729   }

   730 }

   731 

   732 void ConcurrentMarkSweepGeneration::compute_new_size() {

   733   assert_locked_or_safepoint(Heap_lock);

   734 

   735   // If incremental collection failed, we just want to expand

   736   // to the limit.

   737   if (incremental_collection_failed()) {

   738     clear_incremental_collection_failed();

   739     grow_to_reserved();

   740     return;

   741   }

   742 

   743   // The heap has been compacted but not reset yet.

   744   // Any metric such as free() or used() will be incorrect.

   745 

   746   CardGeneration::compute_new_size();

   747 

   748   // Reset again after a possible resizing

   749   if (did_compact()) {

   750     cmsSpace()->reset_after_compaction();

   751   }

   752 }

   753 

   754 void ConcurrentMarkSweepGeneration::compute_new_size_free_list() {

   755   assert_locked_or_safepoint(Heap_lock);

   756 

   757   // If incremental collection failed, we just want to expand

   758   // to the limit.

   759   if (incremental_collection_failed()) {

   760     clear_incremental_collection_failed();

   761     grow_to_reserved();

   762     return;

   763   }

   764 

   765   double free_percentage = ((double) free()) / capacity();

   766   double desired_free_percentage = (double) MinHeapFreeRatio / 100;

   767   double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;

   768 

   769   // compute expansion delta needed for reaching desired free percentage

   770   if (free_percentage < desired_free_percentage) {

   771     size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));

   772     assert(desired_capacity >= capacity(), "invalid expansion size");

   773     size_t expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes);

   774     Log(gc) log;

   775     if (log.is_trace()) {

   776       size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));

   777       log.trace("From compute_new_size: ");

   778       log.trace("  Free fraction %f", free_percentage);

   779       log.trace("  Desired free fraction %f", desired_free_percentage);

   780       log.trace("  Maximum free fraction %f", maximum_free_percentage);

   781       log.trace("  Capacity " SIZE_FORMAT, capacity() / 1000);

   782       log.trace("  Desired capacity " SIZE_FORMAT, desired_capacity / 1000);

   783       CMSHeap* heap = CMSHeap::heap();

   784       size_t young_size = heap->young_gen()->capacity();

   785       log.trace("  Young gen size " SIZE_FORMAT, young_size / 1000);

   786       log.trace("  unsafe_max_alloc_nogc " SIZE_FORMAT, unsafe_max_alloc_nogc() / 1000);

   787       log.trace("  contiguous available " SIZE_FORMAT, contiguous_available() / 1000);

   788       log.trace("  Expand by " SIZE_FORMAT " (bytes)", expand_bytes);

   789     }

   790     // safe if expansion fails

   791     expand_for_gc_cause(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio);

   792     log.trace("  Expanded free fraction %f", ((double) free()) / capacity());

   793   } else {

   794     size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));

   795     assert(desired_capacity <= capacity(), "invalid expansion size");

   796     size_t shrink_bytes = capacity() - desired_capacity;

   797     // Don't shrink unless the delta is greater than the minimum shrink we want

   798     if (shrink_bytes >= MinHeapDeltaBytes) {

   799       shrink_free_list_by(shrink_bytes);

   800     }

   801   }

   802 }

   803 

   804 Mutex* ConcurrentMarkSweepGeneration::freelistLock() const {

   805   return cmsSpace()->freelistLock();

   806 }

   807 

   808 HeapWord* ConcurrentMarkSweepGeneration::allocate(size_t size, bool tlab) {

   809   CMSSynchronousYieldRequest yr;

   810   MutexLocker x(freelistLock(), Mutex::_no_safepoint_check_flag);

   811   return have_lock_and_allocate(size, tlab);

   812 }

   813 

   814 HeapWord* ConcurrentMarkSweepGeneration::have_lock_and_allocate(size_t size,

   815                                                                 bool   tlab /* ignored */) {

   816   assert_lock_strong(freelistLock());

   817   size_t adjustedSize = CompactibleFreeListSpace::adjustObjectSize(size);

   818   HeapWord* res = cmsSpace()->allocate(adjustedSize);

   819   // Allocate the object live (grey) if the background collector has

   820   // started marking. This is necessary because the marker may

   821   // have passed this address and consequently this object will

   822   // not otherwise be greyed and would be incorrectly swept up.

   823   // Note that if this object contains references, the writing

   824   // of those references will dirty the card containing this object

   825   // allowing the object to be blackened (and its references scanned)

   826   // either during a preclean phase or at the final checkpoint.

   827   if (res != NULL) {

   828     // We may block here with an uninitialized object with

   829     // its mark-bit or P-bits not yet set. Such objects need

   830     // to be safely navigable by block_start().

   831     assert(oop(res)->klass_or_null() == NULL, "Object should be uninitialized here.");

   832     assert(!((FreeChunk*)res)->is_free(), "Error, block will look free but show wrong size");

   833     collector()->direct_allocated(res, adjustedSize);

   834     _direct_allocated_words += adjustedSize;

   835     // allocation counters

   836     NOT_PRODUCT(

   837       _numObjectsAllocated++;

   838       _numWordsAllocated += (int)adjustedSize;

   839     )

   840   }

   841   return res;

   842 }

   843 

   844 // In the case of direct allocation by mutators in a generation that

   845 // is being concurrently collected, the object must be allocated

   846 // live (grey) if the background collector has started marking.

   847 // This is necessary because the marker may

   848 // have passed this address and consequently this object will

   849 // not otherwise be greyed and would be incorrectly swept up.

   850 // Note that if this object contains references, the writing

   851 // of those references will dirty the card containing this object

   852 // allowing the object to be blackened (and its references scanned)

   853 // either during a preclean phase or at the final checkpoint.

   854 void CMSCollector::direct_allocated(HeapWord* start, size_t size) {

   855   assert(_markBitMap.covers(start, size), "Out of bounds");

   856   if (_collectorState >= Marking) {

   857     MutexLocker y(_markBitMap.lock(),

   858                   Mutex::_no_safepoint_check_flag);

   859     // [see comments preceding SweepClosure::do_blk() below for details]

   860     //

   861     // Can the P-bits be deleted now?  JJJ

   862     //

   863     // 1. need to mark the object as live so it isn't collected

   864     // 2. need to mark the 2nd bit to indicate the object may be uninitialized

   865     // 3. need to mark the end of the object so marking, precleaning or sweeping

   866     //    can skip over uninitialized or unparsable objects. An allocated

   867     //    object is considered uninitialized for our purposes as long as

   868     //    its klass word is NULL.  All old gen objects are parsable

   869     //    as soon as they are initialized.)

   870     _markBitMap.mark(start);          // object is live

   871     _markBitMap.mark(start + 1);      // object is potentially uninitialized?

   872     _markBitMap.mark(start + size - 1);

   873                                       // mark end of object

   874   }

   875   // check that oop looks uninitialized

   876   assert(oop(start)->klass_or_null() == NULL, "_klass should be NULL");

   877 }

   878 

   879 void CMSCollector::promoted(bool par, HeapWord* start,

   880                             bool is_obj_array, size_t obj_size) {

   881   assert(_markBitMap.covers(start), "Out of bounds");

   882   // See comment in direct_allocated() about when objects should

   883   // be allocated live.

   884   if (_collectorState >= Marking) {

   885     // we already hold the marking bit map lock, taken in

   886     // the prologue

   887     if (par) {

   888       _markBitMap.par_mark(start);

   889     } else {

   890       _markBitMap.mark(start);

   891     }

   892     // We don't need to mark the object as uninitialized (as

   893     // in direct_allocated above) because this is being done with the

   894     // world stopped and the object will be initialized by the

   895     // time the marking, precleaning or sweeping get to look at it.

   896     // But see the code for copying objects into the CMS generation,

   897     // where we need to ensure that concurrent readers of the

   898     // block offset table are able to safely navigate a block that

   899     // is in flux from being free to being allocated (and in

   900     // transition while being copied into) and subsequently

   901     // becoming a bona-fide object when the copy/promotion is complete.

   902     assert(SafepointSynchronize::is_at_safepoint(),

   903            "expect promotion only at safepoints");

   904 

   905     if (_collectorState < Sweeping) {

   906       // Mark the appropriate cards in the modUnionTable, so that

   907       // this object gets scanned before the sweep. If this is

   908       // not done, CMS generation references in the object might

   909       // not get marked.

   910       // For the case of arrays, which are otherwise precisely

   911       // marked, we need to dirty the entire array, not just its head.

   912       if (is_obj_array) {

   913         // The [par_]mark_range() method expects mr.end() below to

   914         // be aligned to the granularity of a bit's representation

   915         // in the heap. In the case of the MUT below, that's a

   916         // card size.

   917         MemRegion mr(start,

   918                      align_up(start + obj_size,

   919                               CardTable::card_size /* bytes */));

   920         if (par) {

   921           _modUnionTable.par_mark_range(mr);

   922         } else {

   923           _modUnionTable.mark_range(mr);

   924         }

   925       } else {  // not an obj array; we can just mark the head

   926         if (par) {

   927           _modUnionTable.par_mark(start);

   928         } else {

   929           _modUnionTable.mark(start);

   930         }

   931       }

   932     }

   933   }

   934 }

   935 

   936 oop ConcurrentMarkSweepGeneration::promote(oop obj, size_t obj_size) {

   937   assert(obj_size == (size_t)obj->size(), "bad obj_size passed in");

   938   // allocate, copy and if necessary update promoinfo --

   939   // delegate to underlying space.

   940   assert_lock_strong(freelistLock());

   941 

   942 #ifndef PRODUCT

   943   if (CMSHeap::heap()->promotion_should_fail()) {

   944     return NULL;

   945   }

   946 #endif  // #ifndef PRODUCT

   947 

   948   oop res = _cmsSpace->promote(obj, obj_size);

   949   if (res == NULL) {

   950     // expand and retry

   951     size_t s = _cmsSpace->expansionSpaceRequired(obj_size);  // HeapWords

   952     expand_for_gc_cause(s*HeapWordSize, MinHeapDeltaBytes, CMSExpansionCause::_satisfy_promotion);

   953     // Since this is the old generation, we don't try to promote

   954     // into a more senior generation.

   955     res = _cmsSpace->promote(obj, obj_size);

   956   }

   957   if (res != NULL) {

   958     // See comment in allocate() about when objects should

   959     // be allocated live.

   960     assert(oopDesc::is_oop(obj), "Will dereference klass pointer below");

   961     collector()->promoted(false,           // Not parallel

   962                           (HeapWord*)res, obj->is_objArray(), obj_size);

   963     // promotion counters

   964     NOT_PRODUCT(

   965       _numObjectsPromoted++;

   966       _numWordsPromoted +=

   967         (int)(CompactibleFreeListSpace::adjustObjectSize(obj->size()));

   968     )

   969   }

   970   return res;

   971 }

   972 

   973 

   974 // IMPORTANT: Notes on object size recognition in CMS.

   975 // ---------------------------------------------------

   976 // A block of storage in the CMS generation is always in

   977 // one of three states. A free block (FREE), an allocated

   978 // object (OBJECT) whose size() method reports the correct size,

   979 // and an intermediate state (TRANSIENT) in which its size cannot

   980 // be accurately determined.

   981 // STATE IDENTIFICATION:   (32 bit and 64 bit w/o COOPS)

   982 // -----------------------------------------------------

   983 // FREE:      klass_word & 1 == 1; mark_word holds block size

   984 //

   985 // OBJECT:    klass_word installed; klass_word != 0 && klass_word & 1 == 0;

   986 //            obj->size() computes correct size

   987 //

   988 // TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT

   989 //

   990 // STATE IDENTIFICATION: (64 bit+COOPS)

   991 // ------------------------------------

   992 // FREE:      mark_word & CMS_FREE_BIT == 1; mark_word & ~CMS_FREE_BIT gives block_size

   993 //

   994 // OBJECT:    klass_word installed; klass_word != 0;

   995 //            obj->size() computes correct size

   996 //

   997 // TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT

   998 //

   999 //

  1000 // STATE TRANSITION DIAGRAM

  1001 //

  1002 //        mut / parnew                     mut  /  parnew

  1003 // FREE --------------------> TRANSIENT ---------------------> OBJECT --|

  1004 //  ^                                                                   |

  1005 //  |------------------------ DEAD <------------------------------------|

  1006 //         sweep                            mut

  1007 //

  1008 // While a block is in TRANSIENT state its size cannot be determined

  1009 // so readers will either need to come back later or stall until

  1010 // the size can be determined. Note that for the case of direct

  1011 // allocation, P-bits, when available, may be used to determine the

  1012 // size of an object that may not yet have been initialized.

  1013 

  1014 // Things to support parallel young-gen collection.

  1015 oop

  1016 ConcurrentMarkSweepGeneration::par_promote(int thread_num,

  1017                                            oop old, markWord m,

  1018                                            size_t word_sz) {

  1019 #ifndef PRODUCT

  1020   if (CMSHeap::heap()->promotion_should_fail()) {

  1021     return NULL;

  1022   }

  1023 #endif  // #ifndef PRODUCT

  1024 

  1025   CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];

  1026   PromotionInfo* promoInfo = &ps->promo;

  1027   // if we are tracking promotions, then first ensure space for

  1028   // promotion (including spooling space for saving header if necessary).

  1029   // then allocate and copy, then track promoted info if needed.

  1030   // When tracking (see PromotionInfo::track()), the mark word may

  1031   // be displaced and in this case restoration of the mark word

  1032   // occurs in the (oop_since_save_marks_)iterate phase.

  1033   if (promoInfo->tracking() && !promoInfo->ensure_spooling_space()) {

  1034     // Out of space for allocating spooling buffers;

  1035     // try expanding and allocating spooling buffers.

  1036     if (!expand_and_ensure_spooling_space(promoInfo)) {

  1037       return NULL;

  1038     }

  1039   }

  1040   assert(!promoInfo->tracking() || promoInfo->has_spooling_space(), "Control point invariant");

  1041   const size_t alloc_sz = CompactibleFreeListSpace::adjustObjectSize(word_sz);

  1042   HeapWord* obj_ptr = ps->lab.alloc(alloc_sz);

  1043   if (obj_ptr == NULL) {

  1044      obj_ptr = expand_and_par_lab_allocate(ps, alloc_sz);

  1045      if (obj_ptr == NULL) {

  1046        return NULL;

  1047      }

  1048   }

  1049   oop obj = oop(obj_ptr);

  1050   OrderAccess::storestore();

  1051   assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");

  1052   assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size");

  1053   // IMPORTANT: See note on object initialization for CMS above.

  1054   // Otherwise, copy the object.  Here we must be careful to insert the

  1055   // klass pointer last, since this marks the block as an allocated object.

  1056   // Except with compressed oops it's the mark word.

  1057   HeapWord* old_ptr = (HeapWord*)old;

  1058   // Restore the mark word copied above.

  1059   obj->set_mark_raw(m);

  1060   assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");

  1061   assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size");

  1062   OrderAccess::storestore();

  1063 

  1064   if (UseCompressedClassPointers) {

  1065     // Copy gap missed by (aligned) header size calculation below

  1066     obj->set_klass_gap(old->klass_gap());

  1067   }

  1068   if (word_sz > (size_t)oopDesc::header_size()) {

  1069     Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(),

  1070                                  obj_ptr + oopDesc::header_size(),

  1071                                  word_sz - oopDesc::header_size());

  1072   }

  1073 

  1074   // Now we can track the promoted object, if necessary.  We take care

  1075   // to delay the transition from uninitialized to full object

  1076   // (i.e., insertion of klass pointer) until after, so that it

  1077   // atomically becomes a promoted object.

  1078   if (promoInfo->tracking()) {

  1079     promoInfo->track((PromotedObject*)obj, old->klass());

  1080   }

  1081   assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");

  1082   assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size");

  1083   assert(oopDesc::is_oop(old), "Will use and dereference old klass ptr below");

  1084 

  1085   // Finally, install the klass pointer (this should be volatile).

  1086   OrderAccess::storestore();

  1087   obj->set_klass(old->klass());

  1088   // We should now be able to calculate the right size for this object

  1089   assert(oopDesc::is_oop(obj) && obj->size() == (int)word_sz, "Error, incorrect size computed for promoted object");

  1090 

  1091   collector()->promoted(true,          // parallel

  1092                         obj_ptr, old->is_objArray(), word_sz);

  1093 

  1094   NOT_PRODUCT(

  1095     Atomic::inc(&_numObjectsPromoted);

  1096     Atomic::add(alloc_sz, &_numWordsPromoted);

  1097   )

  1098 

  1099   return obj;

  1100 }

  1101 

  1102 void

  1103 ConcurrentMarkSweepGeneration::

  1104 par_promote_alloc_done(int thread_num) {

  1105   CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];

  1106   ps->lab.retire(thread_num);

  1107 }

  1108 

  1109 void

  1110 ConcurrentMarkSweepGeneration::

  1111 par_oop_since_save_marks_iterate_done(int thread_num) {

  1112   CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];

  1113   ParScanWithoutBarrierClosure* dummy_cl = NULL;

  1114   ps->promo.promoted_oops_iterate(dummy_cl);

  1115 

  1116   // Because card-scanning has been completed, subsequent phases

  1117   // (e.g., reference processing) will not need to recognize which

  1118   // objects have been promoted during this GC. So, we can now disable

  1119   // promotion tracking.

  1120   ps->promo.stopTrackingPromotions();

  1121 }

  1122 

  1123 bool ConcurrentMarkSweepGeneration::should_collect(bool   full,

  1124                                                    size_t size,

  1125                                                    bool   tlab)

  1126 {

  1127   // We allow a STW collection only if a full

  1128   // collection was requested.

  1129   return full || should_allocate(size, tlab); // FIX ME !!!

  1130   // This and promotion failure handling are connected at the

  1131   // hip and should be fixed by untying them.

  1132 }

  1133 

  1134 bool CMSCollector::shouldConcurrentCollect() {