1 /*

     2  * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  */

    23 

    24 #ifndef SHARE_GC_Z_ZPAGE_INLINE_HPP

    25 #define SHARE_GC_Z_ZPAGE_INLINE_HPP

    26 

    27 #include "gc/z/zAddress.inline.hpp"

    28 #include "gc/z/zForwardingTable.inline.hpp"

    29 #include "gc/z/zGlobals.hpp"

    30 #include "gc/z/zLiveMap.inline.hpp"

    31 #include "gc/z/zMark.hpp"

    32 #include "gc/z/zNUMA.hpp"

    33 #include "gc/z/zPage.hpp"

    34 #include "gc/z/zPhysicalMemory.inline.hpp"

    35 #include "gc/z/zUtils.inline.hpp"

    36 #include "gc/z/zVirtualMemory.inline.hpp"

    37 #include "oops/oop.inline.hpp"

    38 #include "runtime/atomic.hpp"

    39 #include "utilities/align.hpp"

    40 #include "utilities/debug.hpp"

    41 

    42 inline const char* ZPage::type_to_string() const {

    43   switch (type()) {

    44   case ZPageTypeSmall:

    45     return "Small";

    46 

    47   case ZPageTypeMedium:

    48     return "Medium";

    49 

    50   default:

    51     assert(type() == ZPageTypeLarge, "Invalid page type");

    52     return "Large";

    53   }

    54 }

    55 

    56 inline uint32_t ZPage::object_max_count() const {

    57   switch (type()) {

    58   case ZPageTypeLarge:

    59     // A large page can only contain a single

    60     // object aligned to the start of the page.

    61     return 1;

    62 

    63   default:

    64     return (uint32_t)(size() >> object_alignment_shift());

    65   }

    66 }

    67 

    68 inline size_t ZPage::object_alignment_shift() const {

    69   switch (type()) {

    70   case ZPageTypeSmall:

    71     return ZObjectAlignmentSmallShift;

    72 

    73   case ZPageTypeMedium:

    74     return ZObjectAlignmentMediumShift;

    75 

    76   default:

    77     assert(type() == ZPageTypeLarge, "Invalid page type");

    78     return ZObjectAlignmentLargeShift;

    79   }

    80 }

    81 

    82 inline size_t ZPage::object_alignment() const {

    83   switch (type()) {

    84   case ZPageTypeSmall:

    85     return ZObjectAlignmentSmall;

    86 

    87   case ZPageTypeMedium:

    88     return ZObjectAlignmentMedium;

    89 

    90   default:

    91     assert(type() == ZPageTypeLarge, "Invalid page type");

    92     return ZObjectAlignmentLarge;

    93   }

    94 }

    95 

    96 inline uint8_t ZPage::type() const {

    97   return _type;

    98 }

    99 

   100 inline uintptr_t ZPage::start() const {

   101   return _virtual.start();

   102 }

   103 

   104 inline uintptr_t ZPage::end() const {

   105   return _virtual.end();

   106 }

   107 

   108 inline size_t ZPage::size() const {

   109   return _virtual.size();

   110 }

   111 

   112 inline uintptr_t ZPage::top() const {

   113   return _top;

   114 }

   115 

   116 inline size_t ZPage::remaining() const {

   117   return end() - top();

   118 }

   119 

   120 inline ZPhysicalMemory& ZPage::physical_memory() {

   121   return _physical;

   122 }

   123 

   124 inline const ZVirtualMemory& ZPage::virtual_memory() const {

   125   return _virtual;

   126 }

   127 

   128 inline uint8_t ZPage::numa_id() {

   129   if (_numa_id == (uint8_t)-1) {

   130     _numa_id = (uint8_t)ZNUMA::memory_id(ZAddress::good(start()));

   131   }

   132 

   133   return _numa_id;

   134 }

   135 

   136 inline bool ZPage::inc_refcount() {

   137   for (uint32_t prev_refcount = _refcount; prev_refcount > 0; prev_refcount = _refcount) {

   138     if (Atomic::cmpxchg(prev_refcount + 1, &_refcount, prev_refcount) == prev_refcount) {

   139       return true;

   140     }

   141   }

   142   return false;

   143 }

   144 

   145 inline bool ZPage::dec_refcount() {

   146   assert(is_active(), "Should be active");

   147   return Atomic::sub(1u, &_refcount) == 0;

   148 }

   149 

   150 inline bool ZPage::is_in(uintptr_t addr) const {

   151   const uintptr_t offset = ZAddress::offset(addr);

   152   return offset >= start() && offset < top();

   153 }

   154 

   155 inline uintptr_t ZPage::block_start(uintptr_t addr) const {

   156   if (block_is_obj(addr)) {

   157     return addr;

   158   } else {

   159     return ZAddress::good(top());

   160   }

   161 }

   162 

   163 inline size_t ZPage::block_size(uintptr_t addr) const {

   164   if (block_is_obj(addr)) {

   165     return ZUtils::object_size(addr);

   166   } else {

   167     return end() - top();

   168   }

   169 }

   170 

   171 inline bool ZPage::block_is_obj(uintptr_t addr) const {

   172   return ZAddress::offset(addr) < top();

   173 }

   174 

   175 inline bool ZPage::is_active() const {

   176   return _refcount > 0;

   177 }

   178 

   179 inline bool ZPage::is_allocating() const {

   180   return is_active() && _seqnum == ZGlobalSeqNum;

   181 }

   182 

   183 inline bool ZPage::is_relocatable() const {

   184   return is_active() && _seqnum < ZGlobalSeqNum;

   185 }

   186 

   187 inline bool ZPage::is_detached() const {

   188   return _physical.is_null();

   189 }

   190 

   191 inline bool ZPage::is_mapped() const {

   192   return _seqnum > 0;

   193 }

   194 

   195 inline void ZPage::set_pre_mapped() {

   196   // The _seqnum variable is also used to signal that the virtual and physical

   197   // memory has been mapped. So, we need to set it to non-zero when the memory

   198   // has been pre-mapped.

   199   _seqnum = 1;

   200 }

   201 

   202 inline bool ZPage::is_pinned() const {

   203   return _pinned;

   204 }

   205 

   206 inline void ZPage::set_pinned() {

   207   _pinned = 1;

   208 }

   209 

   210 inline bool ZPage::is_forwarding() const {

   211   return !_forwarding.is_null();

   212 }

   213 

   214 inline void ZPage::set_forwarding() {

   215   assert(is_marked(), "Should be marked");

   216   _forwarding.setup(_livemap.live_objects());

   217 }

   218 

   219 inline void ZPage::reset_forwarding() {

   220   _forwarding.reset();

   221   _pinned = 0;

   222 }

   223 

   224 inline void ZPage::verify_forwarding() const {

   225   _forwarding.verify(object_max_count(), _livemap.live_objects());

   226 }

   227 

   228 inline bool ZPage::is_marked() const {

   229   assert(is_relocatable(), "Invalid page state");

   230   return _livemap.is_marked();

   231 }

   232 

   233 inline bool ZPage::is_object_marked(uintptr_t addr) const {

   234   const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;

   235   return _livemap.get(index);

   236 }

   237 

   238 inline bool ZPage::is_object_strongly_marked(uintptr_t addr) const {

   239   const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;

   240   return _livemap.get(index + 1);

   241 }

   242 

   243 inline bool ZPage::is_object_live(uintptr_t addr) const {

   244   return is_allocating() || is_object_marked(addr);

   245 }

   246 

   247 inline bool ZPage::is_object_strongly_live(uintptr_t addr) const {

   248   return is_allocating() || is_object_strongly_marked(addr);

   249 }

   250 

   251 inline bool ZPage::mark_object(uintptr_t addr, bool finalizable, bool& inc_live) {

   252   assert(ZAddress::is_marked(addr), "Invalid address");

   253   assert(is_relocatable(), "Invalid page state");

   254   assert(is_in(addr), "Invalid address");

   255 

   256   // Set mark bit

   257   const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;

   258   return _livemap.set_atomic(index, finalizable, inc_live);

   259 }

   260 

   261 inline void ZPage::inc_live_atomic(uint32_t objects, size_t bytes) {

   262   _livemap.inc_live_atomic(objects, bytes);

   263 }

   264 

   265 inline size_t ZPage::live_bytes() const {

   266   assert(is_marked(), "Should be marked");

   267   return _livemap.live_bytes();

   268 }

   269 

   270 inline void ZPage::object_iterate(ObjectClosure* cl) {

   271   _livemap.iterate(cl, ZAddress::good(start()), object_alignment_shift());

   272 }

   273 

   274 inline uintptr_t ZPage::alloc_object(size_t size) {

   275   assert(is_allocating(), "Invalid state");

   276 

   277   const size_t aligned_size = align_up(size, object_alignment());

   278   const uintptr_t addr = top();

   279   const uintptr_t new_top = addr + aligned_size;

   280 

   281   if (new_top > end()) {

   282     // Not enough space left

   283     return 0;

   284   }

   285 

   286   _top = new_top;

   287 

   288   // Fill alignment padding if needed

   289   if (aligned_size != size) {

   290     ZUtils::insert_filler_object(addr + size, aligned_size - size);

   291   }

   292 

   293   return ZAddress::good(addr);

   294 }

   295 

   296 inline uintptr_t ZPage::alloc_object_atomic(size_t size) {

   297   assert(is_allocating(), "Invalid state");

   298 

   299   const size_t aligned_size = align_up(size, object_alignment());

   300   uintptr_t addr = top();

   301 

   302   for (;;) {

   303     const uintptr_t new_top = addr + aligned_size;

   304     if (new_top > end()) {

   305       // Not enough space left

   306       return 0;

   307     }

   308 

   309     const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, addr);

   310     if (prev_top == addr) {

   311       // Fill alignment padding if needed

   312       if (aligned_size != size) {

   313         ZUtils::insert_filler_object(addr + size, aligned_size - size);

   314       }

   315 

   316       // Success

   317       return ZAddress::good(addr);

   318     }

   319 

   320     // Retry

   321     addr = prev_top;

   322   }

   323 }

   324 

   325 inline bool ZPage::undo_alloc_object(uintptr_t addr, size_t size) {

   326   assert(is_allocating(), "Invalid state");

   327 

   328   const uintptr_t offset = ZAddress::offset(addr);

   329   const size_t aligned_size = align_up(size, object_alignment());

   330   const uintptr_t old_top = top();

   331   const uintptr_t new_top = old_top - aligned_size;

   332 

   333   if (new_top != offset) {

   334     // Failed to undo allocation, not the last allocated object

   335     return false;

   336   }

   337 

   338   _top = new_top;

   339 

   340   // Success

   341   return true;

   342 }

   343 

   344 inline bool ZPage::undo_alloc_object_atomic(uintptr_t addr, size_t size) {

   345   assert(is_allocating(), "Invalid state");

   346 

   347   const uintptr_t offset = ZAddress::offset(addr);

   348   const size_t aligned_size = align_up(size, object_alignment());

   349   uintptr_t old_top = top();

   350 

   351   for (;;) {

   352     const uintptr_t new_top = old_top - aligned_size;

   353     if (new_top != offset) {

   354       // Failed to undo allocation, not the last allocated object

   355       return false;

   356     }

   357 

   358     const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, old_top);

   359     if (prev_top == old_top) {

   360       // Success

   361       return true;

   362     }

   363 

   364     // Retry

   365     old_top = prev_top;

   366   }

   367 }

   368 

   369 #endif // SHARE_GC_Z_ZPAGE_INLINE_HPP