1 /*

     2  * Copyright (c) 2015, 2018, Red Hat, Inc. All rights reserved.

     3  *

     4  * This code is free software; you can redistribute it and/or modify it

     5  * under the terms of the GNU General Public License version 2 only, as

     6  * published by the Free Software Foundation.

     7  *

     8  * This code is distributed in the hope that it will be useful, but WITHOUT

     9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    11  * version 2 for more details (a copy is included in the LICENSE file that

    12  * accompanied this code).

    13  *

    14  * You should have received a copy of the GNU General Public License version

    15  * 2 along with this work; if not, write to the Free Software Foundation,

    16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    17  *

    18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    19  * or visit www.oracle.com if you need additional information or have any

    20  * questions.

    21  *

    22  */

    23 

    24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP

    25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP

    26 

    27 #include "gc/shared/barrierSet.hpp"

    28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"

    29 #include "gc/shenandoah/shenandoahBrooksPointer.inline.hpp"

    30 #include "gc/shenandoah/shenandoahHeap.inline.hpp"

    31 

    32 bool ShenandoahBarrierSet::need_update_refs_barrier() {

    33   return _heap->is_update_refs_in_progress() ||

    34          _heap->is_concurrent_traversal_in_progress() ||

    35          (_heap->is_concurrent_mark_in_progress() && _heap->has_forwarded_objects());

    36 }

    37 

    38 inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) {

    39   return ShenandoahBrooksPointer::forwardee(p);

    40 }

    41 

    42 inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) {

    43   if (((HeapWord*) p) != NULL) {

    44     return resolve_forwarded_not_null(p);

    45   } else {

    46     return p;

    47   }

    48 }

    49 

    50 template <DecoratorSet decorators, typename BarrierSetT>

    51 template <typename T>

    52 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(oop new_value, T* addr, oop compare_value) {

    53   oop res;

    54   oop expected = compare_value;

    55   do {

    56     compare_value = expected;

    57     res = Raw::oop_atomic_cmpxchg(new_value, addr, compare_value);

    58     expected = res;

    59   } while ((! oopDesc::equals_raw(compare_value, expected)) && oopDesc::equals_raw(resolve_forwarded(compare_value), resolve_forwarded(expected)));

    60   if (oopDesc::equals_raw(expected, compare_value)) {

    61     if (ShenandoahSATBBarrier && !CompressedOops::is_null(compare_value)) {

    62       ShenandoahBarrierSet::barrier_set()->enqueue(compare_value);

    63     }

    64   }

    65   return res;

    66 }

    67 

    68 template <DecoratorSet decorators, typename BarrierSetT>

    69 template <typename T>

    70 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(oop new_value, T* addr) {

    71   oop previous = Raw::oop_atomic_xchg(new_value, addr);

    72   if (ShenandoahSATBBarrier) {

    73     if (!CompressedOops::is_null(previous)) {

    74       ShenandoahBarrierSet::barrier_set()->enqueue(previous);

    75     }

    76   }

    77   return previous;

    78 }

    79 

    80 template <DecoratorSet decorators, typename BarrierSetT>

    81 template <typename T>

    82 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,

    83                                                                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,

    84                                                                                      size_t length) {

    85   if (!CompressedOops::is_null(src_obj)) {

    86     src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj));

    87   }

    88   if (!CompressedOops::is_null(dst_obj)) {

    89     dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj));

    90   }

    91   Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);

    92 }

    93 

    94 template <typename T>

    95 bool ShenandoahBarrierSet::arraycopy_loop_1(T* src, T* dst, size_t length, Klass* bound,

    96                                             bool checkcast, bool satb, bool disjoint,

    97                                             ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {

    98   if (checkcast) {

    99     return arraycopy_loop_2<T, true>(src, dst, length, bound, satb, disjoint, storeval_mode);

   100   } else {

   101     return arraycopy_loop_2<T, false>(src, dst, length, bound, satb, disjoint, storeval_mode);

   102   }

   103 }

   104 

   105 template <typename T, bool CHECKCAST>

   106 bool ShenandoahBarrierSet::arraycopy_loop_2(T* src, T* dst, size_t length, Klass* bound,

   107                                             bool satb, bool disjoint,

   108                                             ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {

   109   if (satb) {

   110     return arraycopy_loop_3<T, CHECKCAST, true>(src, dst, length, bound, disjoint, storeval_mode);

   111   } else {

   112     return arraycopy_loop_3<T, CHECKCAST, false>(src, dst, length, bound, disjoint, storeval_mode);

   113   }

   114 }

   115 

   116 template <typename T, bool CHECKCAST, bool SATB>

   117 bool ShenandoahBarrierSet::arraycopy_loop_3(T* src, T* dst, size_t length, Klass* bound, bool disjoint,

   118                                             ShenandoahBarrierSet::ArrayCopyStoreValMode storeval_mode) {

   119   switch (storeval_mode) {

   120     case NONE:

   121       return arraycopy_loop<T, CHECKCAST, SATB, NONE>(src, dst, length, bound, disjoint);

   122     case READ_BARRIER:

   123       return arraycopy_loop<T, CHECKCAST, SATB, READ_BARRIER>(src, dst, length, bound, disjoint);

   124     case WRITE_BARRIER:

   125       return arraycopy_loop<T, CHECKCAST, SATB, WRITE_BARRIER>(src, dst, length, bound, disjoint);

   126     default:

   127       ShouldNotReachHere();

   128       return true; // happy compiler

   129   }

   130 }

   131 

   132 template <typename T, bool CHECKCAST, bool SATB, ShenandoahBarrierSet::ArrayCopyStoreValMode STOREVAL_MODE>

   133 bool ShenandoahBarrierSet::arraycopy_loop(T* src, T* dst, size_t length, Klass* bound, bool disjoint) {

   134   Thread* thread = Thread::current();

   135 

   136   ShenandoahEvacOOMScope oom_evac_scope;

   137 

   138   // We need to handle four cases:

   139   //

   140   // a) src < dst, conjoint, can only copy backward only

   141   //   [...src...]

   142   //         [...dst...]

   143   //

   144   // b) src < dst, disjoint, can only copy forward, because types may mismatch

   145   //   [...src...]

   146   //              [...dst...]

   147   //

   148   // c) src > dst, conjoint, can copy forward only

   149   //         [...src...]

   150   //   [...dst...]

   151   //

   152   // d) src > dst, disjoint, can only copy forward, because types may mismatch

   153   //              [...src...]

   154   //   [...dst...]

   155   //

   156   if (src > dst || disjoint) {

   157     // copy forward:

   158     T* cur_src = src;

   159     T* cur_dst = dst;

   160     T* src_end = src + length;

   161     for (; cur_src < src_end; cur_src++, cur_dst++) {

   162       if (!arraycopy_element<T, CHECKCAST, SATB, STOREVAL_MODE>(cur_src, cur_dst, bound, thread)) {

   163         return false;

   164       }

   165     }

   166   } else {

   167     // copy backward:

   168     T* cur_src = src + length - 1;

   169     T* cur_dst = dst + length - 1;

   170     for (; cur_src >= src; cur_src--, cur_dst--) {

   171       if (!arraycopy_element<T, CHECKCAST, SATB, STOREVAL_MODE>(cur_src, cur_dst, bound, thread)) {

   172         return false;

   173       }

   174     }

   175   }

   176   return true;

   177 }

   178 

   179 template <typename T, bool CHECKCAST, bool SATB, ShenandoahBarrierSet::ArrayCopyStoreValMode STOREVAL_MODE>

   180 bool ShenandoahBarrierSet::arraycopy_element(T* cur_src, T* cur_dst, Klass* bound, Thread* thread) {

   181   T o = RawAccess<>::oop_load(cur_src);

   182 

   183   if (SATB) {

   184     T prev = RawAccess<>::oop_load(cur_dst);

   185     if (!CompressedOops::is_null(prev)) {

   186       oop prev_obj = CompressedOops::decode_not_null(prev);

   187       enqueue(prev_obj);

   188     }

   189   }

   190 

   191   if (!CompressedOops::is_null(o)) {

   192     oop obj = CompressedOops::decode_not_null(o);

   193 

   194     if (CHECKCAST) {

   195       assert(bound != NULL, "need element klass for checkcast");

   196       if (!oopDesc::is_instanceof_or_null(obj, bound)) {

   197         return false;

   198       }

   199     }

   200 

   201     switch (STOREVAL_MODE) {

   202     case NONE:

   203       break;

   204     case READ_BARRIER:

   205       obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);

   206       break;

   207     case WRITE_BARRIER:

   208       if (_heap->in_collection_set(obj)) {

   209         oop forw = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);

   210         if (oopDesc::equals_raw(forw, obj)) {

   211           forw = _heap->evacuate_object(forw, thread);

   212         }

   213         obj = forw;

   214       }

   215       enqueue(obj);

   216       break;

   217     default:

   218       ShouldNotReachHere();

   219     }

   220 

   221     RawAccess<IS_NOT_NULL>::oop_store(cur_dst, obj);

   222   } else {

   223     // Store null.

   224     RawAccess<>::oop_store(cur_dst, o);

   225   }

   226   return true;

   227 }

   228 

   229 // Clone barrier support

   230 template <DecoratorSet decorators, typename BarrierSetT>

   231 void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {

   232   src = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src));

   233   dst = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst));

   234   Raw::clone(src, dst, size);

   235   ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) dst, size));

   236 }

   237 

   238 template <DecoratorSet decorators, typename BarrierSetT>

   239 template <typename T>

   240 bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,

   241                                                                                          arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,

   242                                                                                          size_t length) {

   243   ShenandoahHeap* heap = ShenandoahHeap::heap();

   244   if (!CompressedOops::is_null(src_obj)) {

   245     src_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->read_barrier(src_obj));

   246   }

   247   if (!CompressedOops::is_null(dst_obj)) {

   248     dst_obj = arrayOop(ShenandoahBarrierSet::barrier_set()->write_barrier(dst_obj));

   249   }

   250 

   251   bool satb = ShenandoahSATBBarrier && heap->is_concurrent_mark_in_progress();

   252   bool checkcast = HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value;

   253   bool disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;

   254   ArrayCopyStoreValMode storeval_mode;

   255   if (heap->has_forwarded_objects()) {

   256     if (heap->is_concurrent_traversal_in_progress()) {

   257       storeval_mode = WRITE_BARRIER;

   258     } else if (heap->is_concurrent_mark_in_progress() || heap->is_update_refs_in_progress()) {

   259       storeval_mode = READ_BARRIER;

   260     } else {

   261       assert(heap->is_idle() || heap->is_evacuation_in_progress(), "must not have anything in progress");

   262       storeval_mode = NONE; // E.g. during evac or outside cycle

   263     }

   264   } else {

   265     assert(heap->is_stable() || heap->is_concurrent_mark_in_progress(), "must not have anything in progress");

   266     storeval_mode = NONE;

   267   }

   268 

   269   if (!satb && !checkcast && storeval_mode == NONE) {

   270     // Short-circuit to bulk copy.

   271     return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);

   272   }

   273 

   274   src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);

   275   dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);

   276 

   277   Klass* bound = objArrayOop(dst_obj)->element_klass();

   278   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();

   279   return bs->arraycopy_loop_1(src_raw, dst_raw, length, bound, checkcast, satb, disjoint, storeval_mode);

   280 }

   281 

   282 #endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP