1 /*

     2  * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  */

    23 

    24 #include "precompiled.hpp"

    25 #include "gc/z/zLock.inline.hpp"

    26 #include "gc/z/zMarkStack.inline.hpp"

    27 #include "gc/z/zMarkStackAllocator.hpp"

    28 #include "logging/log.hpp"

    29 #include "runtime/atomic.hpp"

    30 #include "runtime/os.hpp"

    31 #include "utilities/debug.hpp"

    32 

    33 uintptr_t ZMarkStackSpaceStart;

    34 

    35 ZMarkStackSpace::ZMarkStackSpace() :

    36     _expand_lock(),

    37     _start(0),

    38     _top(0),

    39     _end(0) {

    40   assert(ZMarkStackSpaceLimit >= ZMarkStackSpaceExpandSize, "ZMarkStackSpaceLimit too small");

    41 

    42   // Reserve address space

    43   const size_t size = ZMarkStackSpaceLimit;

    44   const size_t alignment = (size_t)os::vm_allocation_granularity();

    45   const uintptr_t addr = (uintptr_t)os::reserve_memory(size, NULL, alignment, mtGC);

    46   if (addr == 0) {

    47     log_error(gc, marking)("Failed to reserve address space for mark stacks");

    48     return;

    49   }

    50 

    51   // Successfully initialized

    52   _start = _top = _end = addr;

    53 

    54   // Register mark stack space start

    55   ZMarkStackSpaceStart = _start;

    56 }

    57 

    58 bool ZMarkStackSpace::is_initialized() const {

    59   return _start != 0;

    60 }

    61 

    62 uintptr_t ZMarkStackSpace::alloc_space(size_t size) {

    63   uintptr_t top = Atomic::load(&_top);

    64 

    65   for (;;) {

    66     const uintptr_t end = Atomic::load(&_end);

    67     const uintptr_t new_top = top + size;

    68     if (new_top > end) {

    69       // Not enough space left

    70       return 0;

    71     }

    72 

    73     const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, top);

    74     if (prev_top == top) {

    75       // Success

    76       return top;

    77     }

    78 

    79     // Retry

    80     top = prev_top;

    81   }

    82 }

    83 

    84 uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) {

    85   ZLocker locker(&_expand_lock);

    86 

    87   // Retry allocation before expanding

    88   uintptr_t addr = alloc_space(size);

    89   if (addr != 0) {

    90     return addr;

    91   }

    92 

    93   // Check expansion limit

    94   const size_t expand_size = ZMarkStackSpaceExpandSize;

    95   const size_t old_size = _end - _start;

    96   const size_t new_size = old_size + expand_size;

    97   if (new_size > ZMarkStackSpaceLimit) {

    98     // Expansion limit reached. This is a fatal error since we

    99     // currently can't recover from running out of mark stack space.

   100     fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit=<size> to increase the "

   101           "maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.",

   102           ZMarkStackSpaceLimit / M);

   103   }

   104 

   105   log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M",

   106                          old_size / M, new_size / M);

   107 

   108   // Expand

   109   os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space");

   110 

   111   // Increment top before end to make sure another

   112   // thread can't steal out newly expanded space.

   113   addr = Atomic::add(size, &_top) - size;

   114   Atomic::add(expand_size, &_end);

   115 

   116   return addr;

   117 }

   118 

   119 uintptr_t ZMarkStackSpace::alloc(size_t size) {

   120   const uintptr_t addr = alloc_space(size);

   121   if (addr != 0) {

   122     return addr;

   123   }

   124 

   125   return expand_and_alloc_space(size);

   126 }

   127 

   128 ZMarkStackAllocator::ZMarkStackAllocator() :

   129     _freelist(),

   130     _space() {

   131   guarantee(sizeof(ZMarkStack) == ZMarkStackSize, "Size mismatch");

   132   guarantee(sizeof(ZMarkStackMagazine) <= ZMarkStackSize, "Size mismatch");

   133 

   134   // Prime free list to avoid an immediate space

   135   // expansion when marking starts.

   136   if (_space.is_initialized()) {

   137     prime_freelist();

   138   }

   139 }

   140 

   141 bool ZMarkStackAllocator::is_initialized() const {

   142   return _space.is_initialized();

   143 }

   144 

   145 void ZMarkStackAllocator::prime_freelist() {

   146   for (size_t size = 0; size < ZMarkStackSpaceExpandSize; size += ZMarkStackMagazineSize) {

   147     const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize);

   148     ZMarkStackMagazine* const magazine = create_magazine_from_space(addr, ZMarkStackMagazineSize);

   149     free_magazine(magazine);

   150   }

   151 }

   152 

   153 ZMarkStackMagazine* ZMarkStackAllocator::create_magazine_from_space(uintptr_t addr, size_t size) {

   154   assert(is_aligned(size, ZMarkStackSize), "Invalid size");

   155 

   156   // Use first stack as magazine

   157   ZMarkStackMagazine* const magazine = new ((void*)addr) ZMarkStackMagazine();

   158   for (size_t i = ZMarkStackSize; i < size; i += ZMarkStackSize) {

   159     ZMarkStack* const stack = new ((void*)(addr + i)) ZMarkStack();

   160     const bool success = magazine->push(stack);

   161     assert(success, "Magazine should never get full");

   162   }

   163 

   164   return magazine;

   165 }

   166 

   167 ZMarkStackMagazine* ZMarkStackAllocator::alloc_magazine() {

   168   // Try allocating from the free list first

   169   ZMarkStackMagazine* const magazine = _freelist.pop_atomic();

   170   if (magazine != NULL) {

   171     return magazine;

   172   }

   173 

   174   // Allocate new magazine

   175   const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize);

   176   if (addr == 0) {

   177     return NULL;

   178   }

   179 

   180   return create_magazine_from_space(addr, ZMarkStackMagazineSize);

   181 }

   182 

   183 void ZMarkStackAllocator::free_magazine(ZMarkStackMagazine* magazine) {

   184   _freelist.push_atomic(magazine);

   185 }