145 

   146 const size_t ParGCAllocBufferWithBOT::ChunkSizeInWords =

   147 MIN2(CardTableModRefBS::par_chunk_heapword_alignment(),

   148      ((size_t)Generation::GenGrain)/HeapWordSize);

   149 const size_t ParGCAllocBufferWithBOT::ChunkSizeInBytes =

   150 MIN2(CardTableModRefBS::par_chunk_heapword_alignment() * HeapWordSize,

   151      (size_t)Generation::GenGrain);

   152 

   153 ParGCAllocBufferWithBOT::ParGCAllocBufferWithBOT(size_t word_sz,

   154                                                  BlockOffsetSharedArray* bsa) :

   155   ParGCAllocBuffer(word_sz),

   156   _bsa(bsa),

   157   _bt(bsa, MemRegion(_bottom, _hard_end)),

   158   _true_end(_hard_end)

   159 {}

   160 

   161 // The buffer comes with its own BOT, with a shared (obviously) underlying

   162 // BlockOffsetSharedArray. We manipulate this BOT in the normal way

   163 // as we would for any contiguous space. However, on occasion we

   164 // need to do some buffer surgery at the extremities before we

   165 // start using the body of the buffer for allocations. Such surgery

   166 // (as explained elsewhere) is to prevent allocation on a card that

   167 // is in the process of being walked concurrently by another GC thread.

   168 // When such surgery happens at a point that is far removed (to the

   169 // right of the current allocation point, top), we use the "contig"

   170 // parameter below to directly manipulate the shared array without

   171 // modifying the _next_threshold state in the BOT.

   172 void ParGCAllocBufferWithBOT::fill_region_with_block(MemRegion mr,

   173                                                      bool contig) {

   174   CollectedHeap::fill_with_object(mr);

   175   if (contig) {

   176     _bt.alloc_block(mr.start(), mr.end());

   177   } else {

   178     _bt.BlockOffsetArray::alloc_block(mr.start(), mr.end());

   179   }

   180 }

   181 

   182 HeapWord* ParGCAllocBufferWithBOT::allocate_slow(size_t word_sz) {

   183   HeapWord* res = NULL;

   184   if (_true_end > _hard_end) {

   185     assert((HeapWord*)align_size_down(intptr_t(_hard_end),

   186                                       ChunkSizeInBytes) == _hard_end,

   187            "or else _true_end should be equal to _hard_end");

   188     assert(_retained, "or else _true_end should be equal to _hard_end");

   189     assert(_retained_filler.end() <= _top, "INVARIANT");

   190     CollectedHeap::fill_with_object(_retained_filler);

   191     if (_top < _hard_end) {

   192       fill_region_with_block(MemRegion(_top, _hard_end), true);

   193     }

   194     HeapWord* next_hard_end = MIN2(_true_end, _hard_end + ChunkSizeInWords);

   195     _retained_filler = MemRegion(_hard_end, FillerHeaderSize);

   196     _bt.alloc_block(_retained_filler.start(), _retained_filler.word_size());

   197     _top      = _retained_filler.end();

   198     _hard_end = next_hard_end;

   199     _end      = _hard_end - AlignmentReserve;

   200     res       = ParGCAllocBuffer::allocate(word_sz);

   201     if (res != NULL) {

   202       _bt.alloc_block(res, word_sz);

   203     }

   204   }

   205   return res;

   206 }

   207 

   208 void

   209 ParGCAllocBufferWithBOT::undo_allocation(HeapWord* obj, size_t word_sz) {

   210   ParGCAllocBuffer::undo_allocation(obj, word_sz);

   211   // This may back us up beyond the previous threshold, so reset.

   212   _bt.set_region(MemRegion(_top, _hard_end));

   213   _bt.initialize_threshold();

   214 }

   215 

   216 void ParGCAllocBufferWithBOT::retire(bool end_of_gc, bool retain) {

   217   assert(!retain || end_of_gc, "Can only retain at GC end.");

   218   if (_retained) {

   219     // We're about to make the retained_filler into a block.

   220     _bt.BlockOffsetArray::alloc_block(_retained_filler.start(),

   221                                       _retained_filler.end());

   222   }

   223   // Reset _hard_end to _true_end (and update _end)

   224   if (retain && _hard_end != NULL) {

   225     assert(_hard_end <= _true_end, "Invariant.");

   226     _hard_end = _true_end;

   227     _end      = MAX2(_top, _hard_end - AlignmentReserve);

   228     assert(_end <= _hard_end, "Invariant.");

   229   }

   230   _true_end = _hard_end;

   231   HeapWord* pre_top = _top;

   232 

   233   ParGCAllocBuffer::retire(end_of_gc, retain);

   234   // Now any old _retained_filler is cut back to size, the free part is

   235   // filled with a filler object, and top is past the header of that

   236   // object.

   237 

   238   if (retain && _top < _end) {

   239     assert(end_of_gc && retain, "Or else retain should be false.");

   240     // If the lab does not start on a card boundary, we don't want to

   241     // allocate onto that card, since that might lead to concurrent

   242     // allocation and card scanning, which we don't support.  So we fill

   243     // the first card with a garbage object.

   244     size_t first_card_index = _bsa->index_for(pre_top);

   245     HeapWord* first_card_start = _bsa->address_for_index(first_card_index);

   246     if (first_card_start < pre_top) {

   247       HeapWord* second_card_start =

   248         _bsa->inc_by_region_size(first_card_start);

   249 

   250       // Ensure enough room to fill with the smallest block

   251       second_card_start = MAX2(second_card_start, pre_top + AlignmentReserve);

   252 

   253       // If the end is already in the first card, don't go beyond it!

   254       // Or if the remainder is too small for a filler object, gobble it up.

   255       if (_hard_end < second_card_start ||

   256           pointer_delta(_hard_end, second_card_start) < AlignmentReserve) {

   257         second_card_start = _hard_end;

   258       }

   259       if (pre_top < second_card_start) {

   260         MemRegion first_card_suffix(pre_top, second_card_start);

   261         fill_region_with_block(first_card_suffix, true);

   262       }

   263       pre_top = second_card_start;

   264       _top = pre_top;

   265       _end = MAX2(_top, _hard_end - AlignmentReserve);

   266     }

   267 

   268     // If the lab does not end on a card boundary, we don't want to

   269     // allocate onto that card, since that might lead to concurrent

   270     // allocation and card scanning, which we don't support.  So we fill

   271     // the last card with a garbage object.

   272     size_t last_card_index = _bsa->index_for(_hard_end);

   273     HeapWord* last_card_start = _bsa->address_for_index(last_card_index);

   274     if (last_card_start < _hard_end) {

   275 

   276       // Ensure enough room to fill with the smallest block

   277       last_card_start = MIN2(last_card_start, _hard_end - AlignmentReserve);

   278 

   279       // If the top is already in the last card, don't go back beyond it!

   280       // Or if the remainder is too small for a filler object, gobble it up.

   281       if (_top > last_card_start ||

   282           pointer_delta(last_card_start, _top) < AlignmentReserve) {

   283         last_card_start = _top;

   284       }

   285       if (last_card_start < _hard_end) {

   286         MemRegion last_card_prefix(last_card_start, _hard_end);

   287         fill_region_with_block(last_card_prefix, false);

   288       }

   289       _hard_end = last_card_start;

   290       _end      = MAX2(_top, _hard_end - AlignmentReserve);

   291       _true_end = _hard_end;

   292       assert(_end <= _hard_end, "Invariant.");

   293     }

   294 

   295     // At this point:

   296     //   1) we had a filler object from the original top to hard_end.

   297     //   2) We've filled in any partial cards at the front and back.

   298     if (pre_top < _hard_end) {

   299       // Now we can reset the _bt to do allocation in the given area.

   300       MemRegion new_filler(pre_top, _hard_end);

   301       fill_region_with_block(new_filler, false);

   302       _top = pre_top + ParGCAllocBuffer::FillerHeaderSize;

   303       // If there's no space left, don't retain.

   304       if (_top >= _end) {

   305         _retained = false;

   306         invalidate();

   307         return;

   308       }

   309       _retained_filler = MemRegion(pre_top, _top);

   310       _bt.set_region(MemRegion(_top, _hard_end));

   311       _bt.initialize_threshold();

   312       assert(_bt.threshold() > _top, "initialize_threshold failed!");

   313 

   314       // There may be other reasons for queries into the middle of the

   315       // filler object.  When such queries are done in parallel with

   316       // allocation, bad things can happen, if the query involves object

   317       // iteration.  So we ensure that such queries do not involve object

   318       // iteration, by putting another filler object on the boundaries of

   319       // such queries.  One such is the object spanning a parallel card

   320       // chunk boundary.

   321 

   322       // "chunk_boundary" is the address of the first chunk boundary less

   323       // than "hard_end".

   324       HeapWord* chunk_boundary =

   325         (HeapWord*)align_size_down(intptr_t(_hard_end-1), ChunkSizeInBytes);

   326       assert(chunk_boundary < _hard_end, "Or else above did not work.");

   327       assert(pointer_delta(_true_end, chunk_boundary) >= AlignmentReserve,

   328              "Consequence of last card handling above.");

   329 

   330       if (_top <= chunk_boundary) {

   331         assert(_true_end == _hard_end, "Invariant.");

   332         while (_top <= chunk_boundary) {

   333           assert(pointer_delta(_hard_end, chunk_boundary) >= AlignmentReserve,

   334                  "Consequence of last card handling above.");

   335           _bt.BlockOffsetArray::alloc_block(chunk_boundary, _hard_end);

   336           CollectedHeap::fill_with_object(chunk_boundary, _hard_end);

   337           _hard_end = chunk_boundary;

   338           chunk_boundary -= ChunkSizeInWords;

   339         }

   340         _end = _hard_end - AlignmentReserve;

   341         assert(_top <= _end, "Invariant.");

   342         // Now reset the initial filler chunk so it doesn't overlap with

   343         // the one(s) inserted above.

   344         MemRegion new_filler(pre_top, _hard_end);

   345         fill_region_with_block(new_filler, false);

   346       }

   347     } else {

   348       _retained = false;

   349       invalidate();

   350     }

   351   } else {

   352     assert(!end_of_gc ||

   353            (!_retained && _true_end == _hard_end), "Checking.");

   354   }

   355   assert(_end <= _hard_end, "Invariant.");

   356   assert(_top < _end || _top == _hard_end, "Invariant");

   357 }