1 /*

     2  * Copyright (c) 2016, 2018, Red Hat, Inc. All rights reserved.

     3  *

     4  * This code is free software; you can redistribute it and/or modify it

     5  * under the terms of the GNU General Public License version 2 only, as

     6  * published by the Free Software Foundation.

     7  *

     8  * This code is distributed in the hope that it will be useful, but WITHOUT

     9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    11  * version 2 for more details (a copy is included in the LICENSE file that

    12  * accompanied this code).

    13  *

    14  * You should have received a copy of the GNU General Public License version

    15  * 2 along with this work; if not, write to the Free Software Foundation,

    16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    17  *

    18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    19  * or visit www.oracle.com if you need additional information or have any

    20  * questions.

    21  *

    22  */

    23 

    24 #include "precompiled.hpp"

    25 

    26 #include "gc/shenandoah/shenandoahFreeSet.hpp"

    27 #include "gc/shenandoah/shenandoahHeap.inline.hpp"

    28 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"

    29 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"

    30 #include "gc/shenandoah/shenandoahTraversalGC.hpp"

    31 #include "logging/logStream.hpp"

    32 

    33 ShenandoahFreeSet::ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions) :

    34   _heap(heap),

    35   _mutator_free_bitmap(max_regions, mtGC),

    36   _collector_free_bitmap(max_regions, mtGC),

    37   _max(max_regions)

    38 {

    39   clear_internal();

    40 }

    41 

    42 void ShenandoahFreeSet::increase_used(size_t num_bytes) {

    43   assert_heaplock_owned_by_current_thread();

    44   _used += num_bytes;

    45 

    46   assert(_used <= _capacity, "must not use more than we have: used: " SIZE_FORMAT

    47          ", capacity: " SIZE_FORMAT ", num_bytes: " SIZE_FORMAT, _used, _capacity, num_bytes);

    48 }

    49 

    50 bool ShenandoahFreeSet::is_mutator_free(size_t idx) const {

    51   assert (idx < _max, "index is sane: " SIZE_FORMAT " < " SIZE_FORMAT " (left: " SIZE_FORMAT ", right: " SIZE_FORMAT ")",

    52           idx, _max, _mutator_leftmost, _mutator_rightmost);

    53   return _mutator_free_bitmap.at(idx);

    54 }

    55 

    56 bool ShenandoahFreeSet::is_collector_free(size_t idx) const {

    57   assert (idx < _max, "index is sane: " SIZE_FORMAT " < " SIZE_FORMAT " (left: " SIZE_FORMAT ", right: " SIZE_FORMAT ")",

    58           idx, _max, _collector_leftmost, _collector_rightmost);

    59   return _collector_free_bitmap.at(idx);

    60 }

    61 

    62 HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& in_new_region) {

    63   // Scan the bitmap looking for a first fit.

    64   //

    65   // Leftmost and rightmost bounds provide enough caching to walk bitmap efficiently. Normally,

    66   // we would find the region to allocate at right away.

    67   //

    68   // Allocations are biased: new application allocs go to beginning of the heap, and GC allocs

    69   // go to the end. This makes application allocation faster, because we would clear lots

    70   // of regions from the beginning most of the time.

    71   //

    72   // Free set maintains mutator and collector views, and normally they allocate in their views only,

    73   // unless we special cases for stealing and mixed allocations.

    74 

    75   switch (req.type()) {

    76     case ShenandoahAllocRequest::_alloc_tlab:

    77     case ShenandoahAllocRequest::_alloc_shared: {

    78 

    79       // Try to allocate in the mutator view

    80       for (size_t idx = _mutator_leftmost; idx <= _mutator_rightmost; idx++) {

    81         if (is_mutator_free(idx)) {

    82           HeapWord* result = try_allocate_in(_heap->get_region(idx), req, in_new_region);

    83           if (result != NULL) {

    84             return result;

    85           }

    86         }

    87       }

    88 

    89       // There is no recovery. Mutator does not touch collector view at all.

    90       break;

    91     }

    92     case ShenandoahAllocRequest::_alloc_gclab:

    93     case ShenandoahAllocRequest::_alloc_shared_gc: {

    94       // size_t is unsigned, need to dodge underflow when _leftmost = 0

    95 

    96       // Fast-path: try to allocate in the collector view first

    97       for (size_t c = _collector_rightmost + 1; c > _collector_leftmost; c--) {

    98         size_t idx = c - 1;

    99         if (is_collector_free(idx)) {

   100           HeapWord* result = try_allocate_in(_heap->get_region(idx), req, in_new_region);

   101           if (result != NULL) {

   102             return result;

   103           }

   104         }

   105       }

   106 

   107       // No dice. Can we borrow space from mutator view?

   108       if (!ShenandoahEvacReserveOverflow) {

   109         return NULL;

   110       }

   111 

   112       // Try to steal the empty region from the mutator view

   113       for (size_t c = _mutator_rightmost + 1; c > _mutator_leftmost; c--) {

   114         size_t idx = c - 1;

   115         if (is_mutator_free(idx)) {

   116           ShenandoahHeapRegion* r = _heap->get_region(idx);

   117           if (is_empty_or_trash(r)) {

   118             flip_to_gc(r);

   119             HeapWord *result = try_allocate_in(r, req, in_new_region);

   120             if (result != NULL) {

   121               return result;

   122             }

   123           }

   124         }

   125       }

   126 

   127       // Try to mix the allocation into the mutator view:

   128       if (ShenandoahAllowMixedAllocs) {

   129         for (size_t c = _mutator_rightmost + 1; c > _mutator_leftmost; c--) {

   130           size_t idx = c - 1;

   131           if (is_mutator_free(idx)) {

   132             HeapWord* result = try_allocate_in(_heap->get_region(idx), req, in_new_region);

   133             if (result != NULL) {

   134               return result;

   135             }

   136           }

   137         }

   138       }

   139       break;

   140     }

   141     default:

   142       ShouldNotReachHere();

   143   }

   144 

   145   return NULL;

   146 }

   147 

   148 HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, ShenandoahAllocRequest& req, bool& in_new_region) {

   149   assert (!has_no_alloc_capacity(r), "Performance: should avoid full regions on this path: " SIZE_FORMAT, r->region_number());

   150 

   151   try_recycle_trashed(r);

   152 

   153   in_new_region = r->is_empty();

   154 

   155   HeapWord* result = NULL;

   156   size_t size = req.size();

   157 

   158   if (ShenandoahElasticTLAB && req.is_lab_alloc()) {

   159     size_t free = align_down(r->free() >> LogHeapWordSize, MinObjAlignment);

   160     if (size > free) {

   161       size = free;

   162     }

   163     if (size >= req.min_size()) {

   164       result = r->allocate(size, req.type());

   165       assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, free, size);

   166     }

   167   } else {

   168     result = r->allocate(size, req.type());

   169   }

   170 

   171   if (result != NULL) {

   172     // Allocation successful, bump stats:

   173     if (req.is_mutator_alloc()) {

   174       increase_used(size * HeapWordSize);

   175     }

   176 

   177     // Record actual allocation size

   178     req.set_actual_size(size);

   179 

   180     if (req.is_gc_alloc() && _heap->is_concurrent_traversal_in_progress()) {

   181       // Traversal needs to traverse through GC allocs. Adjust TAMS to the new top

   182       // so that these allocations appear below TAMS, and thus get traversed.

   183       // See top of shenandoahTraversal.cpp for an explanation.

   184       _heap->marking_context()->capture_top_at_mark_start(r);

   185       _heap->traversal_gc()->traversal_set()->add_region_check_for_duplicates(r);

   186       OrderAccess::fence();

   187     }

   188   }

   189 

   190   if (result == NULL || has_no_alloc_capacity(r)) {

   191     // Region cannot afford this or future allocations. Retire it.

   192     //

   193     // While this seems a bit harsh, especially in the case when this large allocation does not

   194     // fit, but the next small one would, we are risking to inflate scan times when lots of

   195     // almost-full regions precede the fully-empty region where we want allocate the entire TLAB.

   196     // TODO: Record first fully-empty region, and use that for large allocations

   197 

   198     // Record the remainder as allocation waste

   199     if (req.is_mutator_alloc()) {

   200       size_t waste = r->free();

   201       if (waste > 0) {

   202         increase_used(waste);

   203         _heap->notify_mutator_alloc_words(waste >> LogHeapWordSize, true);

   204       }

   205     }

   206 

   207     size_t num = r->region_number();

   208     _collector_free_bitmap.clear_bit(num);

   209     _mutator_free_bitmap.clear_bit(num);

   210     // Touched the bounds? Need to update:

   211     if (touches_bounds(num)) {

   212       adjust_bounds();

   213     }

   214     assert_bounds();

   215   }

   216   return result;

   217 }

   218 

   219 bool ShenandoahFreeSet::touches_bounds(size_t num) const {

   220   return num == _collector_leftmost || num == _collector_rightmost || num == _mutator_leftmost || num == _mutator_rightmost;

   221 }

   222 

   223 void ShenandoahFreeSet::recompute_bounds() {

   224   // Reset to the most pessimistic case:

   225   _mutator_rightmost = _max - 1;

   226   _mutator_leftmost = 0;

   227   _collector_rightmost = _max - 1;

   228   _collector_leftmost = 0;

   229 

   230   // ...and adjust from there

   231   adjust_bounds();

   232 }

   233 

   234 void ShenandoahFreeSet::adjust_bounds() {

   235   // Rewind both mutator bounds until the next bit.

   236   while (_mutator_leftmost < _max && !is_mutator_free(_mutator_leftmost)) {

   237     _mutator_leftmost++;

   238   }

   239   while (_mutator_rightmost > 0 && !is_mutator_free(_mutator_rightmost)) {

   240     _mutator_rightmost--;

   241   }

   242   // Rewind both collector bounds until the next bit.

   243   while (_collector_leftmost < _max && !is_collector_free(_collector_leftmost)) {

   244     _collector_leftmost++;

   245   }

   246   while (_collector_rightmost > 0 && !is_collector_free(_collector_rightmost)) {

   247     _collector_rightmost--;

   248   }

   249 }

   250 

   251 HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) {

   252   assert_heaplock_owned_by_current_thread();

   253 

   254   size_t words_size = req.size();

   255   size_t num = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);

   256 

   257   // No regions left to satisfy allocation, bye.

   258   if (num > mutator_count()) {

   259     return NULL;

   260   }

   261 

   262   // Find the continuous interval of $num regions, starting from $beg and ending in $end,

   263   // inclusive. Contiguous allocations are biased to the beginning.

   264 

   265   size_t beg = _mutator_leftmost;

   266   size_t end = beg;

   267 

   268   while (true) {

   269     if (end >= _max) {

   270       // Hit the end, goodbye

   271       return NULL;

   272     }

   273 

   274     // If regions are not adjacent, then current [beg; end] is useless, and we may fast-forward.

   275     // If region is not completely free, the current [beg; end] is useless, and we may fast-forward.

   276     if (!is_mutator_free(end) || !is_empty_or_trash(_heap->get_region(end))) {

   277       end++;

   278       beg = end;

   279       continue;

   280     }

   281 

   282     if ((end - beg + 1) == num) {

   283       // found the match

   284       break;

   285     }

   286 

   287     end++;

   288   };

   289 

   290   size_t remainder = words_size & ShenandoahHeapRegion::region_size_words_mask();

   291 

   292   // Initialize regions:

   293   for (size_t i = beg; i <= end; i++) {

   294     ShenandoahHeapRegion* r = _heap->get_region(i);

   295     try_recycle_trashed(r);

   296 

   297     assert(i == beg || _heap->get_region(i-1)->region_number() + 1 == r->region_number(), "Should be contiguous");

   298     assert(r->is_empty(), "Should be empty");

   299 

   300     if (i == beg) {

   301       r->make_humongous_start();

   302     } else {

   303       r->make_humongous_cont();

   304     }

   305 

   306     // Trailing region may be non-full, record the remainder there

   307     size_t used_words;

   308     if ((i == end) && (remainder != 0)) {

   309       used_words = remainder;

   310     } else {

   311       used_words = ShenandoahHeapRegion::region_size_words();

   312     }

   313 

   314     r->set_top(r->bottom() + used_words);

   315     r->reset_alloc_metadata_to_shared();

   316 

   317     _mutator_free_bitmap.clear_bit(r->region_number());

   318   }

   319 

   320   // While individual regions report their true use, all humongous regions are

   321   // marked used in the free set.

   322   increase_used(ShenandoahHeapRegion::region_size_bytes() * num);

   323 

   324   if (remainder != 0) {

   325     // Record this remainder as allocation waste

   326     _heap->notify_mutator_alloc_words(ShenandoahHeapRegion::region_size_words() - remainder, true);

   327   }

   328 

   329   // Allocated at left/rightmost? Move the bounds appropriately.

   330   if (beg == _mutator_leftmost || end == _mutator_rightmost) {

   331     adjust_bounds();

   332   }

   333   assert_bounds();

   334 

   335   req.set_actual_size(words_size);

   336   return _heap->get_region(beg)->bottom();

   337 }

   338 

   339 bool ShenandoahFreeSet::is_empty_or_trash(ShenandoahHeapRegion *r) {

   340   return r->is_empty() || r->is_trash();

   341 }

   342 

   343 size_t ShenandoahFreeSet::alloc_capacity(ShenandoahHeapRegion *r) {

   344   if (r->is_trash()) {

   345     // This would be recycled on allocation path

   346     return ShenandoahHeapRegion::region_size_bytes();

   347   } else {

   348     return r->free();

   349   }

   350 }

   351 

   352 bool ShenandoahFreeSet::has_no_alloc_capacity(ShenandoahHeapRegion *r) {

   353   return alloc_capacity(r) == 0;

   354 }

   355 

   356 void ShenandoahFreeSet::try_recycle_trashed(ShenandoahHeapRegion *r) {

   357   if (r->is_trash()) {

   358     _heap->decrease_used(r->used());

   359     r->recycle();

   360   }

   361 }

   362 

   363 void ShenandoahFreeSet::recycle_trash() {

   364   // lock is not reentrable, check we don't have it

   365   assert_heaplock_not_owned_by_current_thread();

   366 

   367   for (size_t i = 0; i < _heap->num_regions(); i++) {

   368     ShenandoahHeapRegion* r = _heap->get_region(i);

   369     if (r->is_trash()) {

   370       ShenandoahHeapLocker locker(_heap->lock());

   371       try_recycle_trashed(r);

   372     }

   373     SpinPause(); // allow allocators to take the lock

   374   }

   375 }

   376 

   377 void ShenandoahFreeSet::flip_to_gc(ShenandoahHeapRegion* r) {

   378   size_t idx = r->region_number();

   379 

   380   assert(_mutator_free_bitmap.at(idx), "Should be in mutator view");

   381   assert(is_empty_or_trash(r), "Should not be allocated");

   382 

   383   _mutator_free_bitmap.clear_bit(idx);

   384   _collector_free_bitmap.set_bit(idx);

   385   _collector_leftmost = MIN2(idx, _collector_leftmost);

   386   _collector_rightmost = MAX2(idx, _collector_rightmost);

   387 

   388   _capacity -= alloc_capacity(r);

   389 

   390   if (touches_bounds(idx)) {

   391     adjust_bounds();

   392   }

   393   assert_bounds();

   394 }

   395 

   396 void ShenandoahFreeSet::clear() {

   397   assert_heaplock_owned_by_current_thread();

   398   clear_internal();

   399 }

   400 

   401 void ShenandoahFreeSet::clear_internal() {

   402   _mutator_free_bitmap.clear();

   403   _collector_free_bitmap.clear();

   404   _mutator_leftmost = _max;

   405   _mutator_rightmost = 0;

   406   _collector_leftmost = _max;

   407   _collector_rightmost = 0;

   408   _capacity = 0;

   409   _used = 0;

   410 }

   411 

   412 void ShenandoahFreeSet::rebuild() {

   413   assert_heaplock_owned_by_current_thread();

   414   clear();

   415 

   416   for (size_t idx = 0; idx < _heap->num_regions(); idx++) {

   417     ShenandoahHeapRegion* region = _heap->get_region(idx);

   418     if (region->is_alloc_allowed() || region->is_trash()) {

   419       assert(!region->is_cset(), "Shouldn't be adding those to the free set");

   420 

   421       // Do not add regions that would surely fail allocation

   422       if (has_no_alloc_capacity(region)) continue;

   423 

   424       _capacity += alloc_capacity(region);

   425       assert(_used <= _capacity, "must not use more than we have");

   426 

   427       assert(!is_mutator_free(idx), "We are about to add it, it shouldn't be there already");

   428       _mutator_free_bitmap.set_bit(idx);

   429     }

   430   }

   431 

   432   // Evac reserve: reserve trailing space for evacuations

   433   size_t to_reserve = ShenandoahEvacReserve * _heap->capacity() / 100;

   434   size_t reserved = 0;

   435 

   436   for (size_t idx = _heap->num_regions() - 1; idx > 0; idx--) {

   437     if (reserved >= to_reserve) break;

   438 

   439     ShenandoahHeapRegion* region = _heap->get_region(idx);

   440     if (_mutator_free_bitmap.at(idx) && is_empty_or_trash(region)) {

   441       _mutator_free_bitmap.clear_bit(idx);

   442       _collector_free_bitmap.set_bit(idx);

   443       size_t ac = alloc_capacity(region);

   444       _capacity -= ac;

   445       reserved += ac;

   446     }

   447   }

   448 

   449   recompute_bounds();

   450   assert_bounds();

   451 }

   452 

   453 void ShenandoahFreeSet::log_status() {

   454   assert_heaplock_owned_by_current_thread();

   455 

   456   LogTarget(Info, gc, ergo) lt;

   457   if (lt.is_enabled()) {

   458     ResourceMark rm;

   459     LogStream ls(lt);

   460 

   461     {

   462       size_t last_idx = 0;

   463       size_t max = 0;

   464       size_t max_contig = 0;

   465       size_t empty_contig = 0;

   466 

   467       size_t total_used = 0;

   468       size_t total_free = 0;

   469 

   470       for (size_t idx = _mutator_leftmost; idx <= _mutator_rightmost; idx++) {

   471         if (is_mutator_free(idx)) {

   472           ShenandoahHeapRegion *r = _heap->get_region(idx);

   473           size_t free = alloc_capacity(r);

   474 

   475           max = MAX2(max, free);

   476 

   477           if (r->is_empty() && (last_idx + 1 == idx)) {

   478             empty_contig++;

   479           } else {

   480             empty_contig = 0;

   481           }

   482 

   483           total_used += r->used();

   484           total_free += free;

   485 

   486           max_contig = MAX2(max_contig, empty_contig);

   487           last_idx = idx;

   488         }

   489       }

   490 

   491       size_t max_humongous = max_contig * ShenandoahHeapRegion::region_size_bytes();

   492       size_t free = capacity() - used();

   493 

   494       ls.print("Free: " SIZE_FORMAT "M (" SIZE_FORMAT " regions), Max regular: " SIZE_FORMAT "K, Max humongous: " SIZE_FORMAT "K, ",

   495                total_free / M, mutator_count(), max / K, max_humongous / K);

   496 

   497       size_t frag_ext;

   498       if (free > 0) {

   499         frag_ext = 100 - (100 * max_humongous / free);

   500       } else {

   501         frag_ext = 0;

   502       }

   503       ls.print("External frag: " SIZE_FORMAT "%%, ", frag_ext);

   504 

   505       size_t frag_int;

   506       if (mutator_count() > 0) {

   507         frag_int = (100 * (total_used / mutator_count()) / ShenandoahHeapRegion::region_size_bytes());

   508       } else {

   509         frag_int = 0;

   510       }

   511       ls.print("Internal frag: " SIZE_FORMAT "%%", frag_int);

   512       ls.cr();

   513     }

   514 

   515     {

   516       size_t max = 0;

   517       size_t total_free = 0;

   518 

   519       for (size_t idx = _collector_leftmost; idx <= _collector_rightmost; idx++) {

   520         if (is_collector_free(idx)) {

   521           ShenandoahHeapRegion *r = _heap->get_region(idx);

   522           size_t free = alloc_capacity(r);

   523           max = MAX2(max, free);

   524           total_free += free;

   525         }

   526       }

   527 

   528       ls.print_cr("Evacuation Reserve: " SIZE_FORMAT "M (" SIZE_FORMAT " regions), Max regular: " SIZE_FORMAT "K",

   529                   total_free / M, collector_count(), max / K);

   530     }

   531   }

   532 }

   533 

   534 HeapWord* ShenandoahFreeSet::allocate(ShenandoahAllocRequest& req, bool& in_new_region) {

   535   assert_heaplock_owned_by_current_thread();

   536   assert_bounds();

   537 

   538   if (req.size() > ShenandoahHeapRegion::humongous_threshold_words()) {

   539     switch (req.type()) {

   540       case ShenandoahAllocRequest::_alloc_shared:

   541       case ShenandoahAllocRequest::_alloc_shared_gc:

   542         in_new_region = true;

   543         return allocate_contiguous(req);

   544       case ShenandoahAllocRequest::_alloc_gclab:

   545       case ShenandoahAllocRequest::_alloc_tlab:

   546         in_new_region = false;

   547         assert(false, "Trying to allocate TLAB larger than the humongous threshold: " SIZE_FORMAT " > " SIZE_FORMAT,

   548                req.size(), ShenandoahHeapRegion::humongous_threshold_words());

   549         return NULL;

   550       default:

   551         ShouldNotReachHere();

   552         return NULL;

   553     }

   554   } else {

   555     return allocate_single(req, in_new_region);

   556   }

   557 }

   558 

   559 size_t ShenandoahFreeSet::unsafe_peek_free() const {

   560   // Deliberately not locked, this method is unsafe when free set is modified.

   561 

   562   for (size_t index = _mutator_leftmost; index <= _mutator_rightmost; index++) {

   563     if (index < _max && is_mutator_free(index)) {

   564       ShenandoahHeapRegion* r = _heap->get_region(index);

   565       if (r->free() >= MinTLABSize) {

   566         return r->free();

   567       }

   568     }

   569   }

   570 

   571   // It appears that no regions left

   572   return 0;

   573 }

   574 

   575 void ShenandoahFreeSet::print_on(outputStream* out) const {

   576   out->print_cr("Mutator Free Set: " SIZE_FORMAT "", mutator_count());

   577   for (size_t index = _mutator_leftmost; index <= _mutator_rightmost; index++) {

   578     if (is_mutator_free(index)) {

   579       _heap->get_region(index)->print_on(out);

   580     }

   581   }

   582   out->print_cr("Collector Free Set: " SIZE_FORMAT "", collector_count());

   583   for (size_t index = _collector_leftmost; index <= _collector_rightmost; index++) {

   584     if (is_collector_free(index)) {

   585       _heap->get_region(index)->print_on(out);

   586     }

   587   }

   588 }

   589 

   590 #ifdef ASSERT

   591 void ShenandoahFreeSet::assert_heaplock_owned_by_current_thread() const {

   592   _heap->assert_heaplock_owned_by_current_thread();

   593 }

   594 

   595 void ShenandoahFreeSet::assert_heaplock_not_owned_by_current_thread() const {

   596   _heap->assert_heaplock_not_owned_by_current_thread();

   597 }

   598 

   599 void ShenandoahFreeSet::assert_bounds() const {

   600   // Performance invariants. Failing these would not break the free set, but performance

   601   // would suffer.

   602   assert (_mutator_leftmost <= _max, "leftmost in bounds: "  SIZE_FORMAT " < " SIZE_FORMAT, _mutator_leftmost,  _max);

   603   assert (_mutator_rightmost < _max, "rightmost in bounds: " SIZE_FORMAT " < " SIZE_FORMAT, _mutator_rightmost, _max);

   604 

   605   assert (_mutator_leftmost == _max || is_mutator_free(_mutator_leftmost),  "leftmost region should be free: " SIZE_FORMAT,  _mutator_leftmost);

   606   assert (_mutator_rightmost == 0   || is_mutator_free(_mutator_rightmost), "rightmost region should be free: " SIZE_FORMAT, _mutator_rightmost);

   607 

   608   size_t beg_off = _mutator_free_bitmap.get_next_one_offset(0);

   609   size_t end_off = _mutator_free_bitmap.get_next_one_offset(_mutator_rightmost + 1);

   610   assert (beg_off >= _mutator_leftmost, "free regions before the leftmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, beg_off, _mutator_leftmost);

   611   assert (end_off == _max,      "free regions past the rightmost: " SIZE_FORMAT ", bound " SIZE_FORMAT,  end_off, _mutator_rightmost);

   612 

   613   assert (_collector_leftmost <= _max, "leftmost in bounds: "  SIZE_FORMAT " < " SIZE_FORMAT, _collector_leftmost,  _max);

   614   assert (_collector_rightmost < _max, "rightmost in bounds: " SIZE_FORMAT " < " SIZE_FORMAT, _collector_rightmost, _max);

   615 

   616   assert (_collector_leftmost == _max || is_collector_free(_collector_leftmost),  "leftmost region should be free: " SIZE_FORMAT,  _collector_leftmost);

   617   assert (_collector_rightmost == 0   || is_collector_free(_collector_rightmost), "rightmost region should be free: " SIZE_FORMAT, _collector_rightmost);

   618 

   619   beg_off = _collector_free_bitmap.get_next_one_offset(0);

   620   end_off = _collector_free_bitmap.get_next_one_offset(_collector_rightmost + 1);

   621   assert (beg_off >= _collector_leftmost, "free regions before the leftmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, beg_off, _collector_leftmost);

   622   assert (end_off == _max,      "free regions past the rightmost: " SIZE_FORMAT ", bound " SIZE_FORMAT,  end_off, _collector_rightmost);

   623 }

   624 #endif