1 /*

     2  * Copyright (c) 2013, 2018, Red Hat, Inc. All rights reserved.

     3  *

     4  * This code is free software; you can redistribute it and/or modify it

     5  * under the terms of the GNU General Public License version 2 only, as

     6  * published by the Free Software Foundation.

     7  *

     8  * This code is distributed in the hope that it will be useful, but WITHOUT

     9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    11  * version 2 for more details (a copy is included in the LICENSE file that

    12  * accompanied this code).

    13  *

    14  * You should have received a copy of the GNU General Public License version

    15  * 2 along with this work; if not, write to the Free Software Foundation,

    16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    17  *

    18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    19  * or visit www.oracle.com if you need additional information or have any

    20  * questions.

    21  *

    22  */

    23 

    24 #include "precompiled.hpp"

    25 #include "memory/allocation.hpp"

    26 #include "gc/shenandoah/shenandoahBrooksPointer.hpp"

    27 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"

    28 #include "gc/shenandoah/shenandoahHeap.hpp"

    29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"

    30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"

    31 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"

    32 #include "gc/shenandoah/shenandoahTraversalGC.hpp"

    33 #include "gc/shared/space.inline.hpp"

    34 #include "memory/iterator.inline.hpp"

    35 #include "memory/resourceArea.hpp"

    36 #include "memory/universe.hpp"

    37 #include "oops/oop.inline.hpp"

    38 #include "runtime/java.hpp"

    39 #include "runtime/mutexLocker.hpp"

    40 #include "runtime/os.hpp"

    41 #include "runtime/safepoint.hpp"

    42 

    43 size_t ShenandoahHeapRegion::RegionCount = 0;

    44 size_t ShenandoahHeapRegion::RegionSizeBytes = 0;

    45 size_t ShenandoahHeapRegion::RegionSizeWords = 0;

    46 size_t ShenandoahHeapRegion::RegionSizeBytesShift = 0;

    47 size_t ShenandoahHeapRegion::RegionSizeWordsShift = 0;

    48 size_t ShenandoahHeapRegion::RegionSizeBytesMask = 0;

    49 size_t ShenandoahHeapRegion::RegionSizeWordsMask = 0;

    50 size_t ShenandoahHeapRegion::HumongousThresholdBytes = 0;

    51 size_t ShenandoahHeapRegion::HumongousThresholdWords = 0;

    52 size_t ShenandoahHeapRegion::MaxTLABSizeBytes = 0;

    53 size_t ShenandoahHeapRegion::MaxTLABSizeWords = 0;

    54 

    55 ShenandoahHeapRegion::PaddedAllocSeqNum ShenandoahHeapRegion::_alloc_seq_num;

    56 

    57 ShenandoahHeapRegion::ShenandoahHeapRegion(ShenandoahHeap* heap, HeapWord* start,

    58                                            size_t size_words, size_t index, bool committed) :

    59   _heap(heap),

    60   _pacer(ShenandoahPacing ? heap->pacer() : NULL),

    61   _reserved(MemRegion(start, size_words)),

    62   _region_number(index),

    63   _new_top(NULL),

    64   _critical_pins(0),

    65   _empty_time(os::elapsedTime()),

    66   _state(committed ? _empty_committed : _empty_uncommitted),

    67   _tlab_allocs(0),

    68   _gclab_allocs(0),

    69   _shared_allocs(0),

    70   _seqnum_first_alloc_mutator(0),

    71   _seqnum_first_alloc_gc(0),

    72   _seqnum_last_alloc_mutator(0),

    73   _seqnum_last_alloc_gc(0),

    74   _live_data(0) {

    75 

    76   ContiguousSpace::initialize(_reserved, true, committed);

    77 }

    78 

    79 size_t ShenandoahHeapRegion::region_number() const {

    80   return _region_number;

    81 }

    82 

    83 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {

    84   ResourceMark rm;

    85   stringStream ss;

    86   ss.print("Illegal region state transition from \"%s\", at %s\n  ", region_state_to_string(_state), method);

    87   print_on(&ss);

    88   fatal("%s", ss.as_string());

    89 }

    90 

    91 void ShenandoahHeapRegion::make_regular_allocation() {

    92   _heap->assert_heaplock_owned_by_current_thread();

    93 

    94   switch (_state) {

    95     case _empty_uncommitted:

    96       do_commit();

    97     case _empty_committed:

    98       _state = _regular;

    99     case _regular:

   100     case _pinned:

   101       return;

   102     default:

   103       report_illegal_transition("regular allocation");

   104   }

   105 }

   106 

   107 void ShenandoahHeapRegion::make_regular_bypass() {

   108   _heap->assert_heaplock_owned_by_current_thread();

   109   assert (_heap->is_full_gc_in_progress() || _heap->is_degenerated_gc_in_progress(),

   110           "only for full or degen GC");

   111 

   112   switch (_state) {

   113     case _empty_uncommitted:

   114       do_commit();

   115     case _empty_committed:

   116     case _cset:

   117     case _humongous_start:

   118     case _humongous_cont:

   119       _state = _regular;

   120       return;

   121     case _pinned_cset:

   122       _state = _pinned;

   123       return;

   124     case _regular:

   125     case _pinned:

   126       return;

   127     default:

   128       report_illegal_transition("regular bypass");

   129   }

   130 }

   131 

   132 void ShenandoahHeapRegion::make_humongous_start() {

   133   _heap->assert_heaplock_owned_by_current_thread();

   134   switch (_state) {

   135     case _empty_uncommitted:

   136       do_commit();

   137     case _empty_committed:

   138       _state = _humongous_start;

   139       return;

   140     default:

   141       report_illegal_transition("humongous start allocation");

   142   }

   143 }

   144 

   145 void ShenandoahHeapRegion::make_humongous_start_bypass() {

   146   _heap->assert_heaplock_owned_by_current_thread();

   147   assert (_heap->is_full_gc_in_progress(), "only for full GC");

   148 

   149   switch (_state) {

   150     case _empty_committed:

   151     case _regular:

   152     case _humongous_start:

   153     case _humongous_cont:

   154       _state = _humongous_start;

   155       return;

   156     default:

   157       report_illegal_transition("humongous start bypass");

   158   }

   159 }

   160 

   161 void ShenandoahHeapRegion::make_humongous_cont() {

   162   _heap->assert_heaplock_owned_by_current_thread();

   163   switch (_state) {

   164     case _empty_uncommitted:

   165       do_commit();

   166     case _empty_committed:

   167       _state = _humongous_cont;

   168       return;

   169     default:

   170       report_illegal_transition("humongous continuation allocation");

   171   }

   172 }

   173 

   174 void ShenandoahHeapRegion::make_humongous_cont_bypass() {

   175   _heap->assert_heaplock_owned_by_current_thread();

   176   assert (_heap->is_full_gc_in_progress(), "only for full GC");

   177 

   178   switch (_state) {

   179     case _empty_committed:

   180     case _regular:

   181     case _humongous_start:

   182     case _humongous_cont:

   183       _state = _humongous_cont;

   184       return;

   185     default:

   186       report_illegal_transition("humongous continuation bypass");

   187   }

   188 }

   189 

   190 void ShenandoahHeapRegion::make_pinned() {

   191   _heap->assert_heaplock_owned_by_current_thread();

   192   switch (_state) {

   193     case _regular:

   194       assert (_critical_pins == 0, "sanity");

   195       _state = _pinned;

   196     case _pinned_cset:

   197     case _pinned:

   198       _critical_pins++;

   199       return;

   200     case _humongous_start:

   201       assert (_critical_pins == 0, "sanity");

   202       _state = _pinned_humongous_start;

   203     case _pinned_humongous_start:

   204       _critical_pins++;

   205       return;

   206     case _cset:

   207       guarantee(_heap->cancelled_gc(), "only valid when evac has been cancelled");

   208       assert (_critical_pins == 0, "sanity");

   209       _state = _pinned_cset;

   210       _critical_pins++;

   211       return;

   212     default:

   213       report_illegal_transition("pinning");

   214   }

   215 }

   216 

   217 void ShenandoahHeapRegion::make_unpinned() {

   218   _heap->assert_heaplock_owned_by_current_thread();

   219   switch (_state) {

   220     case _pinned:

   221       assert (_critical_pins > 0, "sanity");

   222       _critical_pins--;

   223       if (_critical_pins == 0) {

   224         _state = _regular;

   225       }

   226       return;

   227     case _regular:

   228     case _humongous_start:

   229       assert (_critical_pins == 0, "sanity");

   230       return;

   231     case _pinned_cset:

   232       guarantee(_heap->cancelled_gc(), "only valid when evac has been cancelled");

   233       assert (_critical_pins > 0, "sanity");

   234       _critical_pins--;

   235       if (_critical_pins == 0) {

   236         _state = _cset;

   237       }

   238       return;

   239     case _pinned_humongous_start:

   240       assert (_critical_pins > 0, "sanity");

   241       _critical_pins--;

   242       if (_critical_pins == 0) {

   243         _state = _humongous_start;

   244       }

   245       return;

   246     default:

   247       report_illegal_transition("unpinning");

   248   }

   249 }

   250 

   251 void ShenandoahHeapRegion::make_cset() {

   252   _heap->assert_heaplock_owned_by_current_thread();

   253   switch (_state) {

   254     case _regular:

   255       _state = _cset;

   256     case _cset:

   257       return;

   258     default:

   259       report_illegal_transition("cset");

   260   }

   261 }

   262 

   263 void ShenandoahHeapRegion::make_trash() {

   264   _heap->assert_heaplock_owned_by_current_thread();

   265   switch (_state) {

   266     case _cset:

   267       // Reclaiming cset regions

   268     case _humongous_start:

   269     case _humongous_cont:

   270       // Reclaiming humongous regions

   271     case _regular:

   272       // Immediate region reclaim

   273       _state = _trash;

   274       return;

   275     default:

   276       report_illegal_transition("trashing");

   277   }

   278 }

   279 

   280 void ShenandoahHeapRegion::make_trash_immediate() {

   281   make_trash();

   282 

   283   // On this path, we know there are no marked objects in the region,

   284   // tell marking context about it to bypass bitmap resets.

   285   _heap->complete_marking_context()->reset_top_bitmap(this);

   286 }

   287 

   288 void ShenandoahHeapRegion::make_empty() {

   289   _heap->assert_heaplock_owned_by_current_thread();

   290   switch (_state) {

   291     case _trash:

   292       _state = _empty_committed;

   293       _empty_time = os::elapsedTime();

   294       return;

   295     default:

   296       report_illegal_transition("emptying");

   297   }

   298 }

   299 

   300 void ShenandoahHeapRegion::make_uncommitted() {

   301   _heap->assert_heaplock_owned_by_current_thread();

   302   switch (_state) {

   303     case _empty_committed:

   304       do_uncommit();

   305       _state = _empty_uncommitted;

   306       return;

   307     default:

   308       report_illegal_transition("uncommiting");

   309   }

   310 }

   311 

   312 void ShenandoahHeapRegion::make_committed_bypass() {

   313   _heap->assert_heaplock_owned_by_current_thread();

   314   assert (_heap->is_full_gc_in_progress(), "only for full GC");

   315 

   316   switch (_state) {

   317     case _empty_uncommitted:

   318       do_commit();

   319       _state = _empty_committed;

   320       return;

   321     default:

   322       report_illegal_transition("commit bypass");

   323   }

   324 }

   325 

   326 void ShenandoahHeapRegion::clear_live_data() {

   327   OrderAccess::release_store_fence<size_t>(&_live_data, 0);

   328 }

   329 

   330 void ShenandoahHeapRegion::reset_alloc_metadata() {

   331   _tlab_allocs = 0;

   332   _gclab_allocs = 0;

   333   _shared_allocs = 0;

   334   _seqnum_first_alloc_mutator = 0;

   335   _seqnum_last_alloc_mutator = 0;

   336   _seqnum_first_alloc_gc = 0;

   337   _seqnum_last_alloc_gc = 0;

   338 }

   339 

   340 void ShenandoahHeapRegion::reset_alloc_metadata_to_shared() {

   341   if (used() > 0) {

   342     _tlab_allocs = 0;

   343     _gclab_allocs = 0;

   344     _shared_allocs = used() >> LogHeapWordSize;

   345     uint64_t next = _alloc_seq_num.value++;

   346     _seqnum_first_alloc_mutator = next;

   347     _seqnum_last_alloc_mutator = next;

   348     _seqnum_first_alloc_gc = 0;

   349     _seqnum_last_alloc_gc = 0;

   350   } else {

   351     reset_alloc_metadata();

   352   }

   353 }

   354 

   355 size_t ShenandoahHeapRegion::get_shared_allocs() const {

   356   return _shared_allocs * HeapWordSize;

   357 }

   358 

   359 size_t ShenandoahHeapRegion::get_tlab_allocs() const {

   360   return _tlab_allocs * HeapWordSize;

   361 }

   362 

   363 size_t ShenandoahHeapRegion::get_gclab_allocs() const {

   364   return _gclab_allocs * HeapWordSize;

   365 }

   366 

   367 void ShenandoahHeapRegion::set_live_data(size_t s) {

   368   assert(Thread::current()->is_VM_thread(), "by VM thread");

   369   _live_data = (s >> LogHeapWordSize);

   370 }

   371 

   372 size_t ShenandoahHeapRegion::get_live_data_words() const {

   373   return OrderAccess::load_acquire(&_live_data);

   374 }

   375 

   376 size_t ShenandoahHeapRegion::get_live_data_bytes() const {

   377   return get_live_data_words() * HeapWordSize;

   378 }

   379 

   380 bool ShenandoahHeapRegion::has_live() const {

   381   return get_live_data_words() != 0;

   382 }

   383 

   384 size_t ShenandoahHeapRegion::garbage() const {

   385   assert(used() >= get_live_data_bytes(), "Live Data must be a subset of used() live: " SIZE_FORMAT " used: " SIZE_FORMAT,

   386          get_live_data_bytes(), used());

   387 

   388   size_t result = used() - get_live_data_bytes();

   389   return result;

   390 }

   391 

   392 void ShenandoahHeapRegion::print_on(outputStream* st) const {

   393   st->print("|");

   394   st->print(SIZE_FORMAT_W(5), this->_region_number);

   395 

   396   switch (_state) {

   397     case _empty_uncommitted:

   398       st->print("|EU ");

   399       break;

   400     case _empty_committed:

   401       st->print("|EC ");

   402       break;

   403     case _regular:

   404       st->print("|R  ");

   405       break;

   406     case _humongous_start:

   407       st->print("|H  ");

   408       break;

   409     case _pinned_humongous_start:

   410       st->print("|HP ");

   411       break;

   412     case _humongous_cont:

   413       st->print("|HC ");

   414       break;

   415     case _cset:

   416       st->print("|CS ");

   417       break;

   418     case _trash:

   419       st->print("|T  ");

   420       break;

   421     case _pinned:

   422       st->print("|P  ");

   423       break;

   424     case _pinned_cset:

   425       st->print("|CSP");

   426       break;

   427     default:

   428       ShouldNotReachHere();

   429   }

   430   st->print("|BTE " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12),

   431             p2i(bottom()), p2i(top()), p2i(end()));

   432   st->print("|TAMS " INTPTR_FORMAT_W(12),

   433             p2i(_heap->marking_context()->top_at_mark_start(const_cast<ShenandoahHeapRegion*>(this))));

   434   st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()),                proper_unit_for_byte_size(used()));

   435   st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()),     proper_unit_for_byte_size(get_tlab_allocs()));

   436   st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()),    proper_unit_for_byte_size(get_gclab_allocs()));

   437   st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()),   proper_unit_for_byte_size(get_shared_allocs()));

   438   st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes()));

   439   st->print("|CP " SIZE_FORMAT_W(3), _critical_pins);

   440   st->print("|SN " UINT64_FORMAT_X_W(12) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8) ", " UINT64_FORMAT_X_W(8),

   441             seqnum_first_alloc_mutator(), seqnum_last_alloc_mutator(),

   442             seqnum_first_alloc_gc(), seqnum_last_alloc_gc());

   443   st->cr();

   444 }

   445 

   446 void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk) {

   447   if (!is_active()) return;

   448   if (is_humongous()) {

   449     oop_iterate_humongous(blk);

   450   } else {

   451     oop_iterate_objects(blk);

   452   }

   453 }

   454 

   455 void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk) {

   456   assert(! is_humongous(), "no humongous region here");

   457   HeapWord* obj_addr = bottom() + ShenandoahBrooksPointer::word_size();

   458   HeapWord* t = top();

   459   // Could call objects iterate, but this is easier.

   460   while (obj_addr < t) {

   461     oop obj = oop(obj_addr);

   462     obj_addr += obj->oop_iterate_size(blk) + ShenandoahBrooksPointer::word_size();

   463   }

   464 }

   465 

   466 void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) {

   467   assert(is_humongous(), "only humongous region here");

   468   // Find head.

   469   ShenandoahHeapRegion* r = humongous_start_region();

   470   assert(r->is_humongous_start(), "need humongous head here");

   471   oop obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());

   472   obj->oop_iterate(blk, MemRegion(bottom(), top()));

   473 }

   474 

   475 ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {

   476   assert(is_humongous(), "Must be a part of the humongous region");

   477   size_t reg_num = region_number();

   478   ShenandoahHeapRegion* r = const_cast<ShenandoahHeapRegion*>(this);

   479   while (!r->is_humongous_start()) {

   480     assert(reg_num > 0, "Sanity");

   481     reg_num --;

   482     r = _heap->get_region(reg_num);

   483     assert(r->is_humongous(), "Must be a part of the humongous region");

   484   }

   485   assert(r->is_humongous_start(), "Must be");

   486   return r;

   487 }

   488 

   489 void ShenandoahHeapRegion::recycle() {

   490   ContiguousSpace::clear(false);

   491   if (ZapUnusedHeapArea) {

   492     ContiguousSpace::mangle_unused_area_complete();

   493   }

   494   clear_live_data();

   495 

   496   reset_alloc_metadata();

   497 

   498   _heap->marking_context()->reset_top_at_mark_start(this);

   499 

   500   make_empty();

   501 }

   502 

   503 HeapWord* ShenandoahHeapRegion::block_start_const(const void* p) const {

   504   assert(MemRegion(bottom(), end()).contains(p),

   505          "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",

   506          p2i(p), p2i(bottom()), p2i(end()));

   507   if (p >= top()) {

   508     return top();

   509   } else {

   510     HeapWord* last = bottom() + ShenandoahBrooksPointer::word_size();

   511     HeapWord* cur = last;

   512     while (cur <= p) {

   513       last = cur;

   514       cur += oop(cur)->size() + ShenandoahBrooksPointer::word_size();

   515     }

   516     shenandoah_assert_correct(NULL, oop(last));

   517     return last;

   518   }

   519 }

   520 

   521 void ShenandoahHeapRegion::setup_sizes(size_t initial_heap_size, size_t max_heap_size) {

   522   // Absolute minimums we should not ever break.

   523   static const size_t MIN_REGION_SIZE = 256*K;

   524 

   525   if (FLAG_IS_DEFAULT(ShenandoahMinRegionSize)) {

   526     FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE);

   527   }

   528 

   529   size_t region_size;

   530   if (FLAG_IS_DEFAULT(ShenandoahHeapRegionSize)) {

   531     if (ShenandoahMinRegionSize > initial_heap_size / MIN_NUM_REGIONS) {

   532       err_msg message("Initial heap size (" SIZE_FORMAT "K) is too low to afford the minimum number "

   533                       "of regions (" SIZE_FORMAT ") of minimum region size (" SIZE_FORMAT "K).",

   534                       initial_heap_size/K, MIN_NUM_REGIONS, ShenandoahMinRegionSize/K);

   535       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);

   536     }

   537     if (ShenandoahMinRegionSize < MIN_REGION_SIZE) {

   538       err_msg message("" SIZE_FORMAT "K should not be lower than minimum region size (" SIZE_FORMAT "K).",

   539                       ShenandoahMinRegionSize/K,  MIN_REGION_SIZE/K);

   540       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);

   541     }

   542     if (ShenandoahMinRegionSize < MinTLABSize) {

   543       err_msg message("" SIZE_FORMAT "K should not be lower than TLAB size size (" SIZE_FORMAT "K).",

   544                       ShenandoahMinRegionSize/K,  MinTLABSize/K);

   545       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize option", message);

   546     }

   547     if (ShenandoahMaxRegionSize < MIN_REGION_SIZE) {

   548       err_msg message("" SIZE_FORMAT "K should not be lower than min region size (" SIZE_FORMAT "K).",

   549                       ShenandoahMaxRegionSize/K,  MIN_REGION_SIZE/K);

   550       vm_exit_during_initialization("Invalid -XX:ShenandoahMaxRegionSize option", message);

   551     }

   552     if (ShenandoahMinRegionSize > ShenandoahMaxRegionSize) {

   553       err_msg message("Minimum (" SIZE_FORMAT "K) should be larger than maximum (" SIZE_FORMAT "K).",

   554                       ShenandoahMinRegionSize/K, ShenandoahMaxRegionSize/K);

   555       vm_exit_during_initialization("Invalid -XX:ShenandoahMinRegionSize or -XX:ShenandoahMaxRegionSize", message);

   556     }

   557 

   558     // We rapidly expand to max_heap_size in most scenarios, so that is the measure

   559     // for usual heap sizes. Do not depend on initial_heap_size here.

   560     region_size = max_heap_size / ShenandoahTargetNumRegions;

   561 

   562     // Now make sure that we don't go over or under our limits.

   563     region_size = MAX2(ShenandoahMinRegionSize, region_size);

   564     region_size = MIN2(ShenandoahMaxRegionSize, region_size);

   565 

   566   } else {

   567     if (ShenandoahHeapRegionSize > initial_heap_size / MIN_NUM_REGIONS) {

   568       err_msg message("Initial heap size (" SIZE_FORMAT "K) is too low to afford the minimum number "

   569                               "of regions (" SIZE_FORMAT ") of requested size (" SIZE_FORMAT "K).",

   570                       initial_heap_size/K, MIN_NUM_REGIONS, ShenandoahHeapRegionSize/K);

   571       vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);

   572     }

   573     if (ShenandoahHeapRegionSize < ShenandoahMinRegionSize) {

   574       err_msg message("Heap region size (" SIZE_FORMAT "K) should be larger than min region size (" SIZE_FORMAT "K).",

   575                       ShenandoahHeapRegionSize/K, ShenandoahMinRegionSize/K);

   576       vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);

   577     }

   578     if (ShenandoahHeapRegionSize > ShenandoahMaxRegionSize) {

   579       err_msg message("Heap region size (" SIZE_FORMAT "K) should be lower than max region size (" SIZE_FORMAT "K).",

   580                       ShenandoahHeapRegionSize/K, ShenandoahMaxRegionSize/K);

   581       vm_exit_during_initialization("Invalid -XX:ShenandoahHeapRegionSize option", message);

   582     }

   583     region_size = ShenandoahHeapRegionSize;

   584   }

   585 

   586   // Make sure region size is at least one large page, if enabled.

   587   // Otherwise, uncommitting one region may falsely uncommit the adjacent

   588   // regions too.

   589   // Also see shenandoahArguments.cpp, where it handles UseLargePages.

   590   if (UseLargePages && ShenandoahUncommit) {

   591     region_size = MAX2(region_size, os::large_page_size());

   592   }

   593 

   594   int region_size_log = log2_long((jlong) region_size);

   595   // Recalculate the region size to make sure it's a power of

   596   // 2. This means that region_size is the largest power of 2 that's

   597   // <= what we've calculated so far.

   598   region_size = size_t(1) << region_size_log;

   599 

   600   // Now, set up the globals.

   601   guarantee(RegionSizeBytesShift == 0, "we should only set it once");

   602   RegionSizeBytesShift = (size_t)region_size_log;

   603 

   604   guarantee(RegionSizeWordsShift == 0, "we should only set it once");

   605   RegionSizeWordsShift = RegionSizeBytesShift - LogHeapWordSize;

   606 

   607   guarantee(RegionSizeBytes == 0, "we should only set it once");

   608   RegionSizeBytes = region_size;

   609   RegionSizeWords = RegionSizeBytes >> LogHeapWordSize;

   610   assert (RegionSizeWords*HeapWordSize == RegionSizeBytes, "sanity");

   611 

   612   guarantee(RegionSizeWordsMask == 0, "we should only set it once");

   613   RegionSizeWordsMask = RegionSizeWords - 1;

   614 

   615   guarantee(RegionSizeBytesMask == 0, "we should only set it once");

   616   RegionSizeBytesMask = RegionSizeBytes - 1;

   617 

   618   guarantee(RegionCount == 0, "we should only set it once");

   619   RegionCount = max_heap_size / RegionSizeBytes;

   620   guarantee(RegionCount >= MIN_NUM_REGIONS, "Should have at least minimum regions");

   621 

   622   guarantee(HumongousThresholdWords == 0, "we should only set it once");

   623   HumongousThresholdWords = RegionSizeWords * ShenandoahHumongousThreshold / 100;

   624   assert (HumongousThresholdWords <= RegionSizeWords, "sanity");

   625 

   626   guarantee(HumongousThresholdBytes == 0, "we should only set it once");

   627   HumongousThresholdBytes = HumongousThresholdWords * HeapWordSize;

   628   assert (HumongousThresholdBytes <= RegionSizeBytes, "sanity");

   629 

   630   // The rationale for trimming the TLAB sizes has to do with the raciness in

   631   // TLAB allocation machinery. It may happen that TLAB sizing policy polls Shenandoah

   632   // about next free size, gets the answer for region #N, goes away for a while, then

   633   // tries to allocate in region #N, and fail because some other thread have claimed part

   634   // of the region #N, and then the freeset allocation code has to retire the region #N,

   635   // before moving the allocation to region #N+1.

   636   //

   637   // The worst case realizes when "answer" is "region size", which means it could

   638   // prematurely retire an entire region. Having smaller TLABs does not fix that

   639   // completely, but reduces the probability of too wasteful region retirement.

   640   // With current divisor, we will waste no more than 1/8 of region size in the worst

   641   // case. This also has a secondary effect on collection set selection: even under

   642   // the race, the regions would be at least 7/8 used, which allows relying on

   643   // "used" - "live" for cset selection. Otherwise, we can get the fragmented region

   644   // below the garbage threshold that would never be considered for collection.

   645   //

   646   // The whole thing is mitigated if Elastic TLABs are enabled.

   647   //

   648   guarantee(MaxTLABSizeBytes == 0, "we should only set it once");

   649   MaxTLABSizeBytes = MIN2(ShenandoahElasticTLAB ? RegionSizeBytes : (RegionSizeBytes / 8), HumongousThresholdBytes);

   650   assert (MaxTLABSizeBytes > MinTLABSize, "should be larger");

   651 

   652   guarantee(MaxTLABSizeWords == 0, "we should only set it once");

   653   MaxTLABSizeWords = MaxTLABSizeBytes / HeapWordSize;

   654 

   655   log_info(gc, init)("Regions: " SIZE_FORMAT " x " SIZE_FORMAT "%s",

   656                      RegionCount, byte_size_in_proper_unit(RegionSizeBytes), proper_unit_for_byte_size(RegionSizeBytes));

   657   log_info(gc, init)("Humongous object threshold: " SIZE_FORMAT "%s",

   658                      byte_size_in_proper_unit(HumongousThresholdBytes), proper_unit_for_byte_size(HumongousThresholdBytes));

   659   log_info(gc, init)("Max TLAB size: " SIZE_FORMAT "%s",

   660                      byte_size_in_proper_unit(MaxTLABSizeBytes), proper_unit_for_byte_size(MaxTLABSizeBytes));

   661 }

   662 

   663 void ShenandoahHeapRegion::do_commit() {

   664   if (!os::commit_memory((char *) _reserved.start(), _reserved.byte_size(), false)) {

   665     report_java_out_of_memory("Unable to commit region");

   666   }

   667   if (!_heap->commit_bitmap_slice(this)) {

   668     report_java_out_of_memory("Unable to commit bitmaps for region");

   669   }

   670   _heap->increase_committed(ShenandoahHeapRegion::region_size_bytes());

   671 }

   672 

   673 void ShenandoahHeapRegion::do_uncommit() {

   674   if (!os::uncommit_memory((char *) _reserved.start(), _reserved.byte_size())) {

   675     report_java_out_of_memory("Unable to uncommit region");

   676   }

   677   if (!_heap->uncommit_bitmap_slice(this)) {

   678     report_java_out_of_memory("Unable to uncommit bitmaps for region");

   679   }

   680   _heap->decrease_committed(ShenandoahHeapRegion::region_size_bytes());

   681 }