1 /*

     2  * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  */

    23 

    24 #include "precompiled.hpp"

    25 #include "gc/shared/threadLocalAllocBuffer.inline.hpp"

    26 #include "gc/z/zCollectedHeap.hpp"

    27 #include "gc/z/zGlobals.hpp"

    28 #include "gc/z/zHeap.inline.hpp"

    29 #include "gc/z/zObjectAllocator.hpp"

    30 #include "gc/z/zPage.inline.hpp"

    31 #include "gc/z/zStat.hpp"

    32 #include "gc/z/zThread.hpp"

    33 #include "gc/z/zUtils.inline.hpp"

    34 #include "logging/log.hpp"

    35 #include "runtime/atomic.hpp"

    36 #include "runtime/safepoint.hpp"

    37 #include "runtime/thread.hpp"

    38 #include "runtime/threadSMR.hpp"

    39 #include "utilities/align.hpp"

    40 #include "utilities/debug.hpp"

    41 

    42 static const ZStatCounter ZCounterUndoObjectAllocationSucceeded("Memory", "Undo Object Allocation Succeeded", ZStatUnitOpsPerSecond);

    43 static const ZStatCounter ZCounterUndoObjectAllocationFailed("Memory", "Undo Object Allocation Failed", ZStatUnitOpsPerSecond);

    44 static const ZStatSubPhase ZSubPhasePauseRetireTLABS("Pause Retire TLABS");

    45 static const ZStatSubPhase ZSubPhasePauseRemapTLABS("Pause Remap TLABS");

    46 

    47 ZObjectAllocator::ZObjectAllocator(uint nworkers) :

    48     _nworkers(nworkers),

    49     _used(0),

    50     _shared_medium_page(NULL),

    51     _shared_small_page(NULL),

    52     _worker_small_page(NULL) {}

    53 

    54 ZPage* ZObjectAllocator::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) {

    55   ZPage* const page = ZHeap::heap()->alloc_page(type, size, flags);

    56   if (page != NULL) {

    57     // Increment used bytes

    58     Atomic::add(size, _used.addr());

    59   }

    60 

    61   return page;

    62 }

    63 

    64 uintptr_t ZObjectAllocator::alloc_object_in_shared_page(ZPage** shared_page,

    65                                                         uint8_t page_type,

    66                                                         size_t page_size,

    67                                                         size_t size,

    68                                                         ZAllocationFlags flags) {

    69   uintptr_t addr = 0;

    70   ZPage* page = *shared_page;

    71 

    72   if (page != NULL) {

    73     addr = page->alloc_object_atomic(size);

    74   }

    75 

    76   if (addr == 0) {

    77     // Allocate new page

    78     ZPage* const new_page = alloc_page(page_type, page_size, flags);

    79     if (new_page != NULL) {

    80       // Allocate object before installing the new page

    81       addr = new_page->alloc_object(size);

    82 

    83     retry:

    84       // Install new page

    85       ZPage* const prev_page = Atomic::cmpxchg(new_page, shared_page, page);

    86       if (prev_page != page) {

    87         if (prev_page == NULL) {

    88           // Previous page was retired, retry installing the new page

    89           page = prev_page;

    90           goto retry;

    91         }

    92 

    93         // Another page already installed, try allocation there first

    94         const uintptr_t prev_addr = prev_page->alloc_object_atomic(size);

    95         if (prev_addr == 0) {

    96           // Allocation failed, retry installing the new page

    97           page = prev_page;

    98           goto retry;

    99         }

   100 

   101         // Allocation succeeded in already installed page

   102         addr = prev_addr;

   103 

   104         // Undo new page allocation

   105         ZHeap::heap()->undo_alloc_page(new_page);

   106       }

   107     }

   108   }

   109 

   110   return addr;

   111 }

   112 

   113 uintptr_t ZObjectAllocator::alloc_large_object(size_t size, ZAllocationFlags flags) {

   114   assert(ZThread::is_java(), "Should be a Java thread");

   115 

   116   uintptr_t addr = 0;

   117 

   118   // Allocate new large page

   119   const size_t page_size = align_up(size, ZPageSizeMin);

   120   ZPage* const page = alloc_page(ZPageTypeLarge, page_size, flags);

   121   if (page != NULL) {

   122     // Allocate the object

   123     addr = page->alloc_object(size);

   124   }

   125 

   126   return addr;

   127 }

   128 

   129 uintptr_t ZObjectAllocator::alloc_medium_object(size_t size, ZAllocationFlags flags) {

   130   return alloc_object_in_shared_page(_shared_medium_page.addr(), ZPageTypeMedium, ZPageSizeMedium, size, flags);

   131 }

   132 

   133 uintptr_t ZObjectAllocator::alloc_small_object_from_nonworker(size_t size, ZAllocationFlags flags) {

   134   assert(ZThread::is_java() || ZThread::is_vm(), "Should be a Java or VM thread");

   135 

   136   if (flags.relocation() && flags.java_thread() && UseTLAB) {

   137     // For relocations from Java threads, try TLAB allocation first

   138     const uintptr_t addr = (uintptr_t)Thread::current()->tlab().allocate(ZUtils::bytes_to_words(size));

   139     if (addr != 0) {

   140       return addr;

   141     }

   142   }

   143 

   144   // Non-worker small page allocation can never use the reserve

   145   flags.set_no_reserve();

   146 

   147   return alloc_object_in_shared_page(_shared_small_page.addr(), ZPageTypeSmall, ZPageSizeSmall, size, flags);

   148 }

   149 

   150 uintptr_t ZObjectAllocator::alloc_small_object_from_worker(size_t size, ZAllocationFlags flags) {

   151   assert(ZThread::is_worker(), "Should be a worker thread");

   152 

   153   ZPage* page = _worker_small_page.get();

   154   uintptr_t addr = 0;

   155 

   156   if (page != NULL) {

   157     addr = page->alloc_object(size);

   158   }

   159 

   160   if (addr == 0) {

   161     // Allocate new page

   162     page = alloc_page(ZPageTypeSmall, ZPageSizeSmall, flags);

   163     if (page != NULL) {

   164       addr = page->alloc_object(size);

   165     }

   166     _worker_small_page.set(page);

   167   }

   168 

   169   return addr;

   170 }

   171 

   172 uintptr_t ZObjectAllocator::alloc_small_object(size_t size, ZAllocationFlags flags) {

   173   if (flags.worker_thread()) {

   174     return alloc_small_object_from_worker(size, flags);

   175   } else {

   176     return alloc_small_object_from_nonworker(size, flags);

   177   }

   178 }

   179 

   180 uintptr_t ZObjectAllocator::alloc_object(size_t size, ZAllocationFlags flags) {

   181   if (size <= ZObjectSizeLimitSmall) {

   182     // Small

   183     return alloc_small_object(size, flags);

   184   } else if (size <= ZObjectSizeLimitMedium) {

   185     // Medium

   186     return alloc_medium_object(size, flags);

   187   } else {

   188     // Large

   189     return alloc_large_object(size, flags);

   190   }

   191 }

   192 

   193 uintptr_t ZObjectAllocator::alloc_object(size_t size) {

   194   assert(ZThread::is_java(), "Must be a Java thread");

   195 

   196   ZAllocationFlags flags;

   197   flags.set_java_thread();

   198   flags.set_no_reserve();

   199 

   200   if (!ZStallOnOutOfMemory) {

   201     flags.set_non_blocking();

   202   }

   203 

   204   return alloc_object(size, flags);

   205 }

   206 

   207 uintptr_t ZObjectAllocator::alloc_object_for_relocation(size_t size) {

   208   assert(ZThread::is_java() || ZThread::is_worker() || ZThread::is_vm(), "Unknown thread");

   209 

   210   ZAllocationFlags flags;

   211   flags.set_relocation();

   212   flags.set_non_blocking();

   213 

   214   if (ZThread::is_worker()) {

   215     flags.set_worker_thread();

   216   } else if (ZThread::is_java()) {

   217     flags.set_java_thread();

   218   }

   219 

   220   return alloc_object(size, flags);

   221 }

   222 

   223 bool ZObjectAllocator::undo_alloc_large_object(ZPage* page) {

   224   assert(page->type() == ZPageTypeLarge, "Invalid page type");

   225 

   226   // Undo page allocation

   227   ZHeap::heap()->undo_alloc_page(page);

   228   return true;

   229 }

   230 

   231 bool ZObjectAllocator::undo_alloc_medium_object(ZPage* page, uintptr_t addr, size_t size) {

   232   assert(page->type() == ZPageTypeMedium, "Invalid page type");

   233 

   234   // Try atomic undo on shared page

   235   return page->undo_alloc_object_atomic(addr, size);

   236 }

   237 

   238 bool ZObjectAllocator::undo_alloc_small_object_from_nonworker(ZPage* page, uintptr_t addr, size_t size) {

   239   assert(page->type() == ZPageTypeSmall, "Invalid page type");

   240 

   241   if (ZThread::is_java()) {

   242     // Try undo allocation in TLAB

   243     if (Thread::current()->tlab().undo_allocate((HeapWord*)addr, ZUtils::bytes_to_words(size))) {

   244       return true;

   245     }

   246   }

   247 

   248   // Try atomic undo on shared page

   249   return page->undo_alloc_object_atomic(addr, size);

   250 }

   251 

   252 bool ZObjectAllocator::undo_alloc_small_object_from_worker(ZPage* page, uintptr_t addr, size_t size) {

   253   assert(page->type() == ZPageTypeSmall, "Invalid page type");

   254   assert(page == _worker_small_page.get(), "Invalid page");

   255 

   256   // Non-atomic undo on worker-local page

   257   const bool success = page->undo_alloc_object(addr, size);

   258   assert(success, "Should always succeed");

   259   return success;

   260 }

   261 

   262 bool ZObjectAllocator::undo_alloc_small_object(ZPage* page, uintptr_t addr, size_t size) {

   263   if (ZThread::is_worker()) {

   264     return undo_alloc_small_object_from_worker(page, addr, size);

   265   } else {

   266     return undo_alloc_small_object_from_nonworker(page, addr, size);

   267   }

   268 }

   269 

   270 bool ZObjectAllocator::undo_alloc_object(ZPage* page, uintptr_t addr, size_t size) {

   271   const uint8_t type = page->type();

   272 

   273   if (type == ZPageTypeSmall) {

   274     return undo_alloc_small_object(page, addr, size);

   275   } else if (type == ZPageTypeMedium) {

   276     return undo_alloc_medium_object(page, addr, size);

   277   } else {

   278     return undo_alloc_large_object(page);

   279   }

   280 }

   281 

   282 void ZObjectAllocator::undo_alloc_object_for_relocation(ZPage* page, uintptr_t addr, size_t size) {

   283   if (undo_alloc_object(page, addr, size)) {

   284     ZStatInc(ZCounterUndoObjectAllocationSucceeded);

   285   } else {

   286     ZStatInc(ZCounterUndoObjectAllocationFailed);

   287     log_trace(gc)("Failed to undo object allocation: " PTR_FORMAT ", Size: " SIZE_FORMAT ", Thread: " PTR_FORMAT " (%s)",

   288                   addr, size, ZThread::id(), ZThread::name());

   289   }

   290 }

   291 

   292 size_t ZObjectAllocator::used() const {

   293   size_t total_used = 0;

   294 

   295   ZPerCPUConstIterator<size_t> iter(&_used);

   296   for (const size_t* cpu_used; iter.next(&cpu_used);) {

   297     total_used += *cpu_used;

   298   }

   299 

   300   return total_used;

   301 }

   302 

   303 size_t ZObjectAllocator::remaining() const {

   304   assert(ZThread::is_java(), "Should be a Java thread");

   305 

   306   ZPage* page = _shared_small_page.get();

   307   if (page != NULL) {

   308     return page->remaining();

   309   }

   310 

   311   return 0;

   312 }

   313 

   314 void ZObjectAllocator::retire_tlabs() {

   315   ZStatTimer timer(ZSubPhasePauseRetireTLABS);

   316   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");

   317 

   318   // Retire TLABs

   319   if (UseTLAB) {

   320     ZCollectedHeap* heap = ZCollectedHeap::heap();

   321     heap->accumulate_statistics_all_tlabs();

   322     heap->ensure_parsability(true /* retire_tlabs */);

   323     heap->resize_all_tlabs();

   324   }

   325 

   326   // Reset used

   327   _used.set_all(0);

   328 

   329   // Reset allocation pages

   330   _shared_medium_page.set(NULL);

   331   _shared_small_page.set_all(NULL);

   332   _worker_small_page.set_all(NULL);

   333 }

   334 

   335 static void remap_tlab_address(HeapWord** p) {

   336   *p = (HeapWord*)ZAddress::good_or_null((uintptr_t)*p);

   337 }

   338 

   339 void ZObjectAllocator::remap_tlabs() {

   340   ZStatTimer timer(ZSubPhasePauseRemapTLABS);

   341   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");

   342 

   343   if (UseTLAB) {

   344     for (JavaThreadIteratorWithHandle iter; JavaThread* thread = iter.next(); ) {

   345       thread->tlab().addresses_do(remap_tlab_address);

   346     }

   347   }

   348 }