1 /*

     2  * Copyright (c) 2018, Red Hat, Inc. All rights reserved.

     3  *

     4  * This code is free software; you can redistribute it and/or modify it

     5  * under the terms of the GNU General Public License version 2 only, as

     6  * published by the Free Software Foundation.

     7  *

     8  * This code is distributed in the hope that it will be useful, but WITHOUT

     9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    11  * version 2 for more details (a copy is included in the LICENSE file that

    12  * accompanied this code).

    13  *

    14  * You should have received a copy of the GNU General Public License version

    15  * 2 along with this work; if not, write to the Free Software Foundation,

    16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    17  *

    18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    19  * or visit www.oracle.com if you need additional information or have any

    20  * questions.

    21  *

    22  */

    23 

    24 #include "precompiled.hpp"

    25 #include "gc/shenandoah/shenandoahHeap.hpp"

    26 #include "gc/shenandoah/shenandoahHeuristics.hpp"

    27 #include "gc/shenandoah/shenandoahRuntime.hpp"

    28 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"

    29 #include "gc/shenandoah/c2/shenandoahSupport.hpp"

    30 #include "opto/arraycopynode.hpp"

    31 #include "opto/escape.hpp"

    32 #include "opto/graphKit.hpp"

    33 #include "opto/idealKit.hpp"

    34 #include "opto/macro.hpp"

    35 #include "opto/movenode.hpp"

    36 #include "opto/narrowptrnode.hpp"

    37 #include "opto/rootnode.hpp"

    38 

    39 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {

    40   return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());

    41 }

    42 

    43 ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)

    44   : _shenandoah_barriers(new (comp_arena) GrowableArray<ShenandoahWriteBarrierNode*>(comp_arena, 8,  0, NULL)) {

    45 }

    46 

    47 int ShenandoahBarrierSetC2State::shenandoah_barriers_count() const {

    48   return _shenandoah_barriers->length();

    49 }

    50 

    51 ShenandoahWriteBarrierNode* ShenandoahBarrierSetC2State::shenandoah_barrier(int idx) const {

    52   return _shenandoah_barriers->at(idx);

    53 }

    54 

    55 void ShenandoahBarrierSetC2State::add_shenandoah_barrier(ShenandoahWriteBarrierNode * n) {

    56   assert(!_shenandoah_barriers->contains(n), "duplicate entry in barrier list");

    57   _shenandoah_barriers->append(n);

    58 }

    59 

    60 void ShenandoahBarrierSetC2State::remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n) {

    61   if (_shenandoah_barriers->contains(n)) {

    62     _shenandoah_barriers->remove(n);

    63   }

    64 }

    65 

    66 #define __ kit->

    67 

    68 Node* ShenandoahBarrierSetC2::shenandoah_read_barrier(GraphKit* kit, Node* obj) const {

    69   if (ShenandoahReadBarrier) {

    70     obj = shenandoah_read_barrier_impl(kit, obj, false, true, true);

    71   }

    72   return obj;

    73 }

    74 

    75 Node* ShenandoahBarrierSetC2::shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const {

    76   if (ShenandoahStoreValEnqueueBarrier) {

    77     obj = shenandoah_write_barrier(kit, obj);

    78     obj = shenandoah_enqueue_barrier(kit, obj);

    79   }

    80   if (ShenandoahStoreValReadBarrier) {

    81     obj = shenandoah_read_barrier_impl(kit, obj, true, false, false);

    82   }

    83   return obj;

    84 }

    85 

    86 Node* ShenandoahBarrierSetC2::shenandoah_read_barrier_impl(GraphKit* kit, Node* obj, bool use_ctrl, bool use_mem, bool allow_fromspace) const {

    87   const Type* obj_type = obj->bottom_type();

    88   if (obj_type->higher_equal(TypePtr::NULL_PTR)) {

    89     return obj;

    90   }

    91   const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);

    92   Node* mem = use_mem ? __ memory(adr_type) : __ immutable_memory();

    93 

    94   if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, mem, allow_fromspace)) {

    95     // We know it is null, no barrier needed.

    96     return obj;

    97   }

    98 

    99   if (obj_type->meet(TypePtr::NULL_PTR) == obj_type->remove_speculative()) {

   100 

   101     // We don't know if it's null or not. Need null-check.

   102     enum { _not_null_path = 1, _null_path, PATH_LIMIT };

   103     RegionNode* region = new RegionNode(PATH_LIMIT);

   104     Node*       phi    = new PhiNode(region, obj_type);

   105     Node* null_ctrl = __ top();

   106     Node* not_null_obj = __ null_check_oop(obj, &null_ctrl);

   107 

   108     region->init_req(_null_path, null_ctrl);

   109     phi   ->init_req(_null_path, __ zerocon(T_OBJECT));

   110 

   111     Node* ctrl = use_ctrl ? __ control() : NULL;

   112     ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, not_null_obj, allow_fromspace);

   113     Node* n = __ gvn().transform(rb);

   114 

   115     region->init_req(_not_null_path, __ control());

   116     phi   ->init_req(_not_null_path, n);

   117 

   118     __ set_control(__ gvn().transform(region));

   119     __ record_for_igvn(region);

   120     return __ gvn().transform(phi);

   121 

   122   } else {

   123     // We know it is not null. Simple barrier is sufficient.

   124     Node* ctrl = use_ctrl ? __ control() : NULL;

   125     ShenandoahReadBarrierNode* rb = new ShenandoahReadBarrierNode(ctrl, mem, obj, allow_fromspace);

   126     Node* n = __ gvn().transform(rb);

   127     __ record_for_igvn(n);

   128     return n;

   129   }

   130 }

   131 

   132 Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_helper(GraphKit* kit, Node* obj, const TypePtr* adr_type) const {

   133   ShenandoahWriteBarrierNode* wb = new ShenandoahWriteBarrierNode(kit->C, kit->control(), kit->memory(adr_type), obj);

   134   Node* n = __ gvn().transform(wb);

   135   if (n == wb) { // New barrier needs memory projection.

   136     Node* proj = __ gvn().transform(new ShenandoahWBMemProjNode(n));

   137     __ set_memory(proj, adr_type);

   138   }

   139   return n;

   140 }

   141 

   142 Node* ShenandoahBarrierSetC2::shenandoah_write_barrier(GraphKit* kit, Node* obj) const {

   143   if (ShenandoahWriteBarrier) {

   144     obj = shenandoah_write_barrier_impl(kit, obj);

   145   }

   146   return obj;

   147 }

   148 

   149 Node* ShenandoahBarrierSetC2::shenandoah_write_barrier_impl(GraphKit* kit, Node* obj) const {

   150   if (! ShenandoahBarrierNode::needs_barrier(&__ gvn(), NULL, obj, NULL, true)) {

   151     return obj;

   152   }

   153   const Type* obj_type = obj->bottom_type();

   154   const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(obj_type);

   155   Node* n = shenandoah_write_barrier_helper(kit, obj, adr_type);

   156   __ record_for_igvn(n);

   157   return n;

   158 }

   159 

   160 bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr,

   161                                                          BasicType bt, uint adr_idx) const {

   162   intptr_t offset = 0;

   163   Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);

   164   AllocateNode* alloc = AllocateNode::Ideal_allocation(base, phase);

   165 

   166   if (offset == Type::OffsetBot) {

   167     return false; // cannot unalias unless there are precise offsets

   168   }

   169 

   170   if (alloc == NULL) {

   171     return false; // No allocation found

   172   }

   173 

   174   intptr_t size_in_bytes = type2aelembytes(bt);

   175 

   176   Node* mem = __ memory(adr_idx); // start searching here...

   177 

   178   for (int cnt = 0; cnt < 50; cnt++) {

   179 

   180     if (mem->is_Store()) {

   181 

   182       Node* st_adr = mem->in(MemNode::Address);

   183       intptr_t st_offset = 0;

   184       Node* st_base = AddPNode::Ideal_base_and_offset(st_adr, phase, st_offset);

   185 

   186       if (st_base == NULL) {

   187         break; // inscrutable pointer

   188       }

   189 

   190       // Break we have found a store with same base and offset as ours so break

   191       if (st_base == base && st_offset == offset) {

   192         break;

   193       }

   194 

   195       if (st_offset != offset && st_offset != Type::OffsetBot) {

   196         const int MAX_STORE = BytesPerLong;

   197         if (st_offset >= offset + size_in_bytes ||

   198             st_offset <= offset - MAX_STORE ||

   199             st_offset <= offset - mem->as_Store()->memory_size()) {

   200           // Success:  The offsets are provably independent.

   201           // (You may ask, why not just test st_offset != offset and be done?

   202           // The answer is that stores of different sizes can co-exist

   203           // in the same sequence of RawMem effects.  We sometimes initialize

   204           // a whole 'tile' of array elements with a single jint or jlong.)

   205           mem = mem->in(MemNode::Memory);

   206           continue; // advance through independent store memory

   207         }

   208       }

   209 

   210       if (st_base != base

   211           && MemNode::detect_ptr_independence(base, alloc, st_base,

   212                                               AllocateNode::Ideal_allocation(st_base, phase),

   213                                               phase)) {

   214         // Success:  The bases are provably independent.

   215         mem = mem->in(MemNode::Memory);

   216         continue; // advance through independent store memory

   217       }

   218     } else if (mem->is_Proj() && mem->in(0)->is_Initialize()) {

   219 

   220       InitializeNode* st_init = mem->in(0)->as_Initialize();

   221       AllocateNode* st_alloc = st_init->allocation();

   222 

   223       // Make sure that we are looking at the same allocation site.

   224       // The alloc variable is guaranteed to not be null here from earlier check.

   225       if (alloc == st_alloc) {

   226         // Check that the initialization is storing NULL so that no previous store

   227         // has been moved up and directly write a reference

   228         Node* captured_store = st_init->find_captured_store(offset,

   229                                                             type2aelembytes(T_OBJECT),

   230                                                             phase);

   231         if (captured_store == NULL || captured_store == st_init->zero_memory()) {

   232           return true;

   233         }

   234       }

   235     }

   236 

   237     // Unless there is an explicit 'continue', we must bail out here,

   238     // because 'mem' is an inscrutable memory state (e.g., a call).

   239     break;

   240   }

   241 

   242   return false;

   243 }

   244 

   245 #undef __

   246 #define __ ideal.

   247 

   248 void ShenandoahBarrierSetC2::satb_write_barrier_pre(GraphKit* kit,

   249                                                     bool do_load,

   250                                                     Node* obj,

   251                                                     Node* adr,

   252                                                     uint alias_idx,

   253                                                     Node* val,

   254                                                     const TypeOopPtr* val_type,

   255                                                     Node* pre_val,

   256                                                     BasicType bt) const {

   257   // Some sanity checks

   258   // Note: val is unused in this routine.

   259 

   260   if (do_load) {

   261     // We need to generate the load of the previous value

   262     assert(obj != NULL, "must have a base");

   263     assert(adr != NULL, "where are loading from?");

   264     assert(pre_val == NULL, "loaded already?");

   265     assert(val_type != NULL, "need a type");

   266 

   267     if (ReduceInitialCardMarks

   268         && satb_can_remove_pre_barrier(kit, &kit->gvn(), adr, bt, alias_idx)) {

   269       return;

   270     }

   271 

   272   } else {

   273     // In this case both val_type and alias_idx are unused.

   274     assert(pre_val != NULL, "must be loaded already");

   275     // Nothing to be done if pre_val is null.

   276     if (pre_val->bottom_type() == TypePtr::NULL_PTR) return;

   277     assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here");

   278   }

   279   assert(bt == T_OBJECT, "or we shouldn't be here");

   280 

   281   IdealKit ideal(kit, true);

   282 

   283   Node* tls = __ thread(); // ThreadLocalStorage

   284 

   285   Node* no_base = __ top();

   286   Node* zero  = __ ConI(0);

   287   Node* zeroX = __ ConX(0);

   288 

   289   float likely  = PROB_LIKELY(0.999);

   290   float unlikely  = PROB_UNLIKELY(0.999);

   291 

   292   // Offsets into the thread

   293   const int index_offset   = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());

   294   const int buffer_offset  = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());

   295 

   296   // Now the actual pointers into the thread

   297   Node* buffer_adr  = __ AddP(no_base, tls, __ ConX(buffer_offset));

   298   Node* index_adr   = __ AddP(no_base, tls, __ ConX(index_offset));

   299 

   300   // Now some of the values

   301   Node* marking;

   302   Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())));

   303   Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw);

   304   marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING));

   305   assert(ShenandoahWriteBarrierNode::is_gc_state_load(ld), "Should match the shape");

   306 

   307   // if (!marking)

   308   __ if_then(marking, BoolTest::ne, zero, unlikely); {

   309     BasicType index_bt = TypeX_X->basic_type();

   310     assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size.");

   311     Node* index   = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw);

   312 

   313     if (do_load) {

   314       // load original value

   315       // alias_idx correct??

   316       pre_val = __ load(__ ctrl(), adr, val_type, bt, alias_idx);

   317     }

   318 

   319     // if (pre_val != NULL)

   320     __ if_then(pre_val, BoolTest::ne, kit->null()); {

   321       Node* buffer  = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);

   322 

   323       // is the queue for this thread full?

   324       __ if_then(index, BoolTest::ne, zeroX, likely); {

   325 

   326         // decrement the index

   327         Node* next_index = kit->gvn().transform(new SubXNode(index, __ ConX(sizeof(intptr_t))));

   328 

   329         // Now get the buffer location we will log the previous value into and store it

   330         Node *log_addr = __ AddP(no_base, buffer, next_index);

   331         __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw, MemNode::unordered);

   332         // update the index

   333         __ store(__ ctrl(), index_adr, next_index, index_bt, Compile::AliasIdxRaw, MemNode::unordered);

   334 

   335       } __ else_(); {

   336 

   337         // logging buffer is full, call the runtime

   338         const TypeFunc *tf = ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type();

   339         __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", pre_val, tls);

   340       } __ end_if();  // (!index)

   341     } __ end_if();  // (pre_val != NULL)

   342   } __ end_if();  // (!marking)

   343 

   344   // Final sync IdealKit and GraphKit.

   345   kit->final_sync(ideal);

   346 

   347   if (ShenandoahSATBBarrier && adr != NULL) {

   348     Node* c = kit->control();

   349     Node* call = c->in(1)->in(1)->in(1)->in(0);

   350     assert(is_shenandoah_wb_pre_call(call), "shenandoah_wb_pre call expected");

   351     call->add_req(adr);

   352   }

   353 }

   354 

   355 bool ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(Node* call) {

   356   return call->is_CallLeaf() &&

   357          call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry);

   358 }

   359 

   360 bool ShenandoahBarrierSetC2::is_shenandoah_wb_call(Node* call) {

   361   return call->is_CallLeaf() &&

   362          call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT);

   363 }

   364 

   365 bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {

   366   if (n->Opcode() != Op_If) {

   367     return false;

   368   }

   369 

   370   Node* bol = n->in(1);

   371   assert(bol->is_Bool(), "");

   372   Node* cmpx = bol->in(1);

   373   if (bol->as_Bool()->_test._test == BoolTest::ne &&

   374       cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) &&

   375       is_shenandoah_state_load(cmpx->in(1)->in(1)) &&

   376       cmpx->in(1)->in(2)->is_Con() &&

   377       cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING)) {

   378     return true;

   379   }

   380 

   381   return false;

   382 }

   383 

   384 bool ShenandoahBarrierSetC2::is_shenandoah_state_load(Node* n) {

   385   if (!n->is_Load()) return false;

   386   const int state_offset = in_bytes(ShenandoahThreadLocalData::gc_state_offset());

   387   return n->in(2)->is_AddP() && n->in(2)->in(2)->Opcode() == Op_ThreadLocal

   388          && n->in(2)->in(3)->is_Con()

   389          && n->in(2)->in(3)->bottom_type()->is_intptr_t()->get_con() == state_offset;

   390 }

   391 

   392 void ShenandoahBarrierSetC2::shenandoah_write_barrier_pre(GraphKit* kit,

   393                                                           bool do_load,

   394                                                           Node* obj,

   395                                                           Node* adr,

   396                                                           uint alias_idx,

   397                                                           Node* val,

   398                                                           const TypeOopPtr* val_type,

   399                                                           Node* pre_val,

   400                                                           BasicType bt) const {

   401   if (ShenandoahSATBBarrier) {

   402     IdealKit ideal(kit);

   403     kit->sync_kit(ideal);

   404 

   405     satb_write_barrier_pre(kit, do_load, obj, adr, alias_idx, val, val_type, pre_val, bt);

   406 

   407     ideal.sync_kit(kit);

   408     kit->final_sync(ideal);

   409   }

   410 }

   411 

   412 Node* ShenandoahBarrierSetC2::shenandoah_enqueue_barrier(GraphKit* kit, Node* pre_val) const {

   413   return kit->gvn().transform(new ShenandoahEnqueueBarrierNode(pre_val));

   414 }

   415 

   416 // Helper that guards and inserts a pre-barrier.

   417 void ShenandoahBarrierSetC2::insert_pre_barrier(GraphKit* kit, Node* base_oop, Node* offset,

   418                                                 Node* pre_val, bool need_mem_bar) const {

   419   // We could be accessing the referent field of a reference object. If so, when G1

   420   // is enabled, we need to log the value in the referent field in an SATB buffer.

   421   // This routine performs some compile time filters and generates suitable

   422   // runtime filters that guard the pre-barrier code.

   423   // Also add memory barrier for non volatile load from the referent field

   424   // to prevent commoning of loads across safepoint.

   425 

   426   // Some compile time checks.

   427 

   428   // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?

   429   const TypeX* otype = offset->find_intptr_t_type();

   430   if (otype != NULL && otype->is_con() &&

   431       otype->get_con() != java_lang_ref_Reference::referent_offset) {

   432     // Constant offset but not the reference_offset so just return

   433     return;

   434   }

   435 

   436   // We only need to generate the runtime guards for instances.

   437   const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();

   438   if (btype != NULL) {

   439     if (btype->isa_aryptr()) {

   440       // Array type so nothing to do

   441       return;

   442     }

   443 

   444     const TypeInstPtr* itype = btype->isa_instptr();

   445     if (itype != NULL) {

   446       // Can the klass of base_oop be statically determined to be

   447       // _not_ a sub-class of Reference and _not_ Object?

   448       ciKlass* klass = itype->klass();

   449       if ( klass->is_loaded() &&

   450           !klass->is_subtype_of(kit->env()->Reference_klass()) &&

   451           !kit->env()->Object_klass()->is_subtype_of(klass)) {

   452         return;

   453       }

   454     }

   455   }

   456 

   457   // The compile time filters did not reject base_oop/offset so

   458   // we need to generate the following runtime filters

   459   //

   460   // if (offset == java_lang_ref_Reference::_reference_offset) {

   461   //   if (instance_of(base, java.lang.ref.Reference)) {

   462   //     pre_barrier(_, pre_val, ...);

   463   //   }

   464   // }

   465 

   466   float likely   = PROB_LIKELY(  0.999);

   467   float unlikely = PROB_UNLIKELY(0.999);

   468 

   469   IdealKit ideal(kit);

   470 

   471   Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);

   472 

   473   __ if_then(offset, BoolTest::eq, referent_off, unlikely); {

   474       // Update graphKit memory and control from IdealKit.

   475       kit->sync_kit(ideal);

   476 

   477       Node* ref_klass_con = kit->makecon(TypeKlassPtr::make(kit->env()->Reference_klass()));

   478       Node* is_instof = kit->gen_instanceof(base_oop, ref_klass_con);

   479 

   480       // Update IdealKit memory and control from graphKit.

   481       __ sync_kit(kit);

   482 

   483       Node* one = __ ConI(1);

   484       // is_instof == 0 if base_oop == NULL

   485       __ if_then(is_instof, BoolTest::eq, one, unlikely); {

   486 

   487         // Update graphKit from IdeakKit.

   488         kit->sync_kit(ideal);

   489 

   490         // Use the pre-barrier to record the value in the referent field

   491         satb_write_barrier_pre(kit, false /* do_load */,

   492                                NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,

   493                                pre_val /* pre_val */,

   494                                T_OBJECT);

   495         if (need_mem_bar) {

   496           // Add memory barrier to prevent commoning reads from this field

   497           // across safepoint since GC can change its value.

   498           kit->insert_mem_bar(Op_MemBarCPUOrder);

   499         }

   500         // Update IdealKit from graphKit.

   501         __ sync_kit(kit);

   502 

   503       } __ end_if(); // _ref_type != ref_none

   504   } __ end_if(); // offset == referent_offset

   505 

   506   // Final sync IdealKit and GraphKit.

   507   kit->final_sync(ideal);

   508 }

   509 

   510 #undef __

   511 

   512 const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {

   513   const Type **fields = TypeTuple::fields(2);

   514   fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value

   515   fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // thread

   516   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);

   517 

   518   // create result type (range)

   519   fields = TypeTuple::fields(0);

   520   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);

   521 

   522   return TypeFunc::make(domain, range);

   523 }

   524 

   525 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {

   526   const Type **fields = TypeTuple::fields(1);

   527   fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value

   528   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);

   529 

   530   // create result type (range)

   531   fields = TypeTuple::fields(0);

   532   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);

   533 

   534   return TypeFunc::make(domain, range);

   535 }

   536 

   537 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_write_barrier_Type() {

   538   const Type **fields = TypeTuple::fields(1);

   539   fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value

   540   const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);

   541 

   542   // create result type (range)

   543   fields = TypeTuple::fields(1);

   544   fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;

   545   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);

   546 

   547   return TypeFunc::make(domain, range);

   548 }

   549 

   550 void ShenandoahBarrierSetC2::resolve_address(C2Access& access) const {

   551   const TypePtr* adr_type = access.addr().type();

   552 

   553   if ((access.decorators() & IN_NATIVE) == 0 && (adr_type->isa_instptr() || adr_type->isa_aryptr())) {

   554     int off = adr_type->is_ptr()->offset();

   555     int base_off = adr_type->isa_instptr() ? instanceOopDesc::base_offset_in_bytes() :

   556       arrayOopDesc::base_offset_in_bytes(adr_type->is_aryptr()->elem()->array_element_basic_type());

   557     assert(off != Type::OffsetTop, "unexpected offset");

   558     if (off == Type::OffsetBot || off >= base_off) {

   559       DecoratorSet decorators = access.decorators();

   560       bool is_write = (decorators & C2_WRITE_ACCESS) != 0;

   561       GraphKit* kit = NULL;

   562       if (access.is_parse_access()) {

   563         C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);

   564         kit = parse_access.kit();

   565       }

   566       Node* adr = access.addr().node();

   567       assert(adr->is_AddP(), "unexpected address shape");

   568       Node* base = adr->in(AddPNode::Base);

   569 

   570       if (is_write) {

   571         if (kit != NULL) {

   572           base = shenandoah_write_barrier(kit, base);

   573         } else {

   574           assert(access.is_opt_access(), "either parse or opt access");

   575           assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for clone");

   576         }

   577       } else {

   578         if (adr_type->isa_instptr()) {

   579           Compile* C = access.gvn().C;

   580           ciField* field = C->alias_type(adr_type)->field();

   581 

   582           // Insert read barrier for Shenandoah.

   583           if (field != NULL &&

   584               ((ShenandoahOptimizeStaticFinals   && field->is_static()  && field->is_final()) ||

   585                (ShenandoahOptimizeInstanceFinals && !field->is_static() && field->is_final()) ||

   586                (ShenandoahOptimizeStableFinals   && field->is_stable()))) {

   587             // Skip the barrier for special fields

   588           } else {

   589             if (kit != NULL) {

   590               base = shenandoah_read_barrier(kit, base);

   591             } else {

   592               assert(access.is_opt_access(), "either parse or opt access");

   593               assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy");

   594             }

   595           }

   596         } else {

   597           if (kit != NULL) {

   598             base = shenandoah_read_barrier(kit, base);

   599           } else {

   600             assert(access.is_opt_access(), "either parse or opt access");

   601             assert((access.decorators() & C2_ARRAY_COPY) != 0, "can be skipped for arraycopy");

   602           }

   603         }

   604       }

   605       if (base != adr->in(AddPNode::Base)) {

   606         assert(kit != NULL, "no barrier should have been added");

   607 

   608         Node* address = adr->in(AddPNode::Address);

   609 

   610         if (address->is_AddP()) {

   611           assert(address->in(AddPNode::Base) == adr->in(AddPNode::Base), "unexpected address shape");

   612           assert(!address->in(AddPNode::Address)->is_AddP(), "unexpected address shape");

   613           assert(address->in(AddPNode::Address) == adr->in(AddPNode::Base), "unexpected address shape");

   614           address = address->clone();

   615           address->set_req(AddPNode::Base, base);

   616           address->set_req(AddPNode::Address, base);

   617           address = kit->gvn().transform(address);

   618         } else {

   619           assert(address == adr->in(AddPNode::Base), "unexpected address shape");

   620           address = base;

   621         }

   622         adr = adr->clone();

   623         adr->set_req(AddPNode::Base, base);

   624         adr->set_req(AddPNode::Address, address);

   625         adr = kit->gvn().transform(adr);

   626         access.addr().set_node(adr);

   627       }

   628     }

   629   }

   630 }

   631 

   632 Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {

   633   DecoratorSet decorators = access.decorators();

   634 

   635   const TypePtr* adr_type = access.addr().type();

   636   Node* adr = access.addr().node();

   637 

   638   bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;

   639   bool on_heap = (decorators & IN_HEAP) != 0;

   640 

   641   if (!access.is_oop() || (!on_heap && !anonymous)) {

   642     return BarrierSetC2::store_at_resolved(access, val);

   643   }

   644 

   645   if (access.is_parse_access()) {

   646     C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);

   647     GraphKit* kit = parse_access.kit();

   648 

   649     uint adr_idx = kit->C->get_alias_index(adr_type);

   650     assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );

   651     Node* value = val.node();

   652     value = shenandoah_storeval_barrier(kit, value);

   653     val.set_node(value);

   654     shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),

   655                                  static_cast<const TypeOopPtr*>(val.type()), NULL /* pre_val */, access.type());

   656   } else {

   657     assert(access.is_opt_access(), "only for optimization passes");

   658     assert(((decorators & C2_TIGHLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");

   659     C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);

   660     PhaseGVN& gvn =  opt_access.gvn();

   661     MergeMemNode* mm = opt_access.mem();

   662 

   663     if (ShenandoahStoreValReadBarrier) {

   664       RegionNode* region = new RegionNode(3);

   665       const Type* v_t = gvn.type(val.node());

   666       Node* phi = new PhiNode(region, v_t->isa_oopptr() ? v_t->is_oopptr()->cast_to_nonconst() : v_t);

   667       Node* cmp = gvn.transform(new CmpPNode(val.node(), gvn.zerocon(T_OBJECT)));

   668       Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne));

   669       IfNode* iff = new IfNode(opt_access.ctl(), bol, PROB_LIKELY_MAG(3), COUNT_UNKNOWN);

   670 

   671       gvn.transform(iff);

   672       if (gvn.is_IterGVN()) {

   673         gvn.is_IterGVN()->_worklist.push(iff);

   674       } else {

   675         gvn.record_for_igvn(iff);

   676       }

   677 

   678       Node* null_true = gvn.transform(new IfFalseNode(iff));

   679       Node* null_false = gvn.transform(new IfTrueNode(iff));

   680       region->init_req(1, null_true);

   681       region->init_req(2, null_false);

   682       phi->init_req(1, gvn.zerocon(T_OBJECT));

   683       Node* cast = new CastPPNode(val.node(), gvn.type(val.node())->join_speculative(TypePtr::NOTNULL));

   684       cast->set_req(0, null_false);

   685       cast = gvn.transform(cast);

   686       Node* rb = gvn.transform(new ShenandoahReadBarrierNode(null_false, gvn.C->immutable_memory(), cast, false));

   687       phi->init_req(2, rb);

   688       opt_access.set_ctl(gvn.transform(region));

   689       val.set_node(gvn.transform(phi));

   690     }

   691     if (ShenandoahStoreValEnqueueBarrier) {

   692       const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(gvn.type(val.node()));

   693       int alias = gvn.C->get_alias_index(adr_type);

   694       Node* wb = new ShenandoahWriteBarrierNode(gvn.C, opt_access.ctl(), mm->memory_at(alias), val.node());

   695       Node* wb_transformed = gvn.transform(wb);

   696       Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(wb_transformed));

   697       if (wb_transformed == wb) {

   698         Node* proj = gvn.transform(new ShenandoahWBMemProjNode(wb));

   699         mm->set_memory_at(alias, proj);

   700       }

   701       val.set_node(enqueue);

   702     }

   703   }

   704   return BarrierSetC2::store_at_resolved(access, val);

   705 }

   706 

   707 Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {

   708   DecoratorSet decorators = access.decorators();

   709 

   710   Node* adr = access.addr().node();

   711   Node* obj = access.base();

   712 

   713   bool mismatched = (decorators & C2_MISMATCHED) != 0;

   714   bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;

   715   bool on_heap = (decorators & IN_HEAP) != 0;

   716   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;

   717   bool is_unordered = (decorators & MO_UNORDERED) != 0;

   718   bool need_cpu_mem_bar = !is_unordered || mismatched || !on_heap;

   719 

   720   Node* top = Compile::current()->top();

   721 

   722   Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;

   723   Node* load = BarrierSetC2::load_at_resolved(access, val_type);

   724 

   725   // If we are reading the value of the referent field of a Reference

   726   // object (either by using Unsafe directly or through reflection)

   727   // then, if SATB is enabled, we need to record the referent in an

   728   // SATB log buffer using the pre-barrier mechanism.

   729   // Also we need to add memory barrier to prevent commoning reads

   730   // from this field across safepoint since GC can change its value.

   731   bool need_read_barrier = ShenandoahKeepAliveBarrier &&

   732     (on_heap && (on_weak || (unknown && offset != top && obj != top)));

   733 

   734   if (!access.is_oop() || !need_read_barrier) {

   735     return load;

   736   }

   737 

   738   assert(access.is_parse_access(), "entry not supported at optimization time");

   739   C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);

   740   GraphKit* kit = parse_access.kit();

   741 

   742   if (on_weak) {

   743     // Use the pre-barrier to record the value in the referent field

   744     satb_write_barrier_pre(kit, false /* do_load */,

   745                            NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,

   746                            load /* pre_val */, T_OBJECT);

   747     // Add memory barrier to prevent commoning reads from this field

   748     // across safepoint since GC can change its value.

   749     kit->insert_mem_bar(Op_MemBarCPUOrder);

   750   } else if (unknown) {

   751     // We do not require a mem bar inside pre_barrier if need_mem_bar

   752     // is set: the barriers would be emitted by us.

   753     insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);

   754   }

   755 

   756   return load;

   757 }

   758 

   759 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,

   760                                                    Node* new_val, const Type* value_type) const {

   761   GraphKit* kit = access.kit();

   762   if (access.is_oop()) {

   763     new_val = shenandoah_storeval_barrier(kit, new_val);

   764     shenandoah_write_barrier_pre(kit, false /* do_load */,

   765                                  NULL, NULL, max_juint, NULL, NULL,

   766                                  expected_val /* pre_val */, T_OBJECT);

   767 

   768     MemNode::MemOrd mo = access.mem_node_mo();

   769     Node* mem = access.memory();

   770     Node* adr = access.addr().node();

   771     const TypePtr* adr_type = access.addr().type();

   772     Node* load_store = NULL;

   773 

   774 #ifdef _LP64

   775     if (adr->bottom_type()->is_ptr_to_narrowoop()) {

   776       Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));

   777       Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));

   778       load_store = kit->gvn().transform(new ShenandoahCompareAndExchangeNNode(kit->control(), mem, adr, newval_enc, oldval_enc, adr_type, value_type->make_narrowoop(), mo));

   779     } else

   780 #endif

   781     {

   782       load_store = kit->gvn().transform(new ShenandoahCompareAndExchangePNode(kit->control(), mem, adr, new_val, expected_val, adr_type, value_type->is_oopptr(), mo));

   783     }

   784 

   785     access.set_raw_access(load_store);

   786     pin_atomic_op(access);

   787 

   788 #ifdef _LP64

   789     if (adr->bottom_type()->is_ptr_to_narrowoop()) {

   790       return kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));

   791     }

   792 #endif

   793     return load_store;

   794   }

   795   return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);

   796 }

   797 

   798 Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,

   799                                                               Node* new_val, const Type* value_type) const {

   800   GraphKit* kit = access.kit();

   801   if (access.is_oop()) {

   802     new_val = shenandoah_storeval_barrier(kit, new_val);

   803     shenandoah_write_barrier_pre(kit, false /* do_load */,

   804                                  NULL, NULL, max_juint, NULL, NULL,

   805                                  expected_val /* pre_val */, T_OBJECT);

   806     DecoratorSet decorators = access.decorators();

   807     MemNode::MemOrd mo = access.mem_node_mo();

   808     Node* mem = access.memory();

   809     bool is_weak_cas = (decorators & C2_WEAK_CMPXCHG) != 0;

   810     Node* load_store = NULL;

   811     Node* adr = access.addr().node();

   812 #ifdef _LP64

   813     if (adr->bottom_type()->is_ptr_to_narrowoop()) {

   814       Node *newval_enc = kit->gvn().transform(new EncodePNode(new_val, new_val->bottom_type()->make_narrowoop()));

   815       Node *oldval_enc = kit->gvn().transform(new EncodePNode(expected_val, expected_val->bottom_type()->make_narrowoop()));

   816       if (is_weak_cas) {

   817         load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));

   818       } else {

   819         load_store = kit->gvn().transform(new ShenandoahCompareAndSwapNNode(kit->control(), mem, adr, newval_enc, oldval_enc, mo));

   820       }

   821     } else

   822 #endif

   823     {

   824       if (is_weak_cas) {

   825         load_store = kit->gvn().transform(new ShenandoahWeakCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));

   826       } else {

   827         load_store = kit->gvn().transform(new ShenandoahCompareAndSwapPNode(kit->control(), mem, adr, new_val, expected_val, mo));

   828       }

   829     }

   830     access.set_raw_access(load_store);

   831     pin_atomic_op(access);

   832     return load_store;

   833   }

   834   return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);

   835 }

   836 

   837 Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {

   838   GraphKit* kit = access.kit();

   839   if (access.is_oop()) {

   840     val = shenandoah_storeval_barrier(kit, val);

   841   }

   842   Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);

   843   if (access.is_oop()) {

   844     shenandoah_write_barrier_pre(kit, false /* do_load */,

   845                                  NULL, NULL, max_juint, NULL, NULL,

   846                                  result /* pre_val */, T_OBJECT);

   847   }

   848   return result;

   849 }

   850 

   851 void ShenandoahBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {

   852   assert(!src->is_AddP(), "unexpected input");

   853   src = shenandoah_read_barrier(kit, src);

   854   BarrierSetC2::clone(kit, src, dst, size, is_array);

   855 }

   856 

   857 Node* ShenandoahBarrierSetC2::resolve(GraphKit* kit, Node* n, DecoratorSet decorators) const {

   858   bool is_write = decorators & ACCESS_WRITE;

   859   if (is_write) {

   860     return shenandoah_write_barrier(kit, n);

   861   } else {

   862   return shenandoah_read_barrier(kit, n);

   863   }

   864 }

   865 

   866 Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl, Node* mem, Node* toobig_false, Node* size_in_bytes,

   867                                            Node*& i_o, Node*& needgc_ctrl,

   868                                            Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,

   869                                            intx prefetch_lines) const {

   870   PhaseIterGVN& igvn = macro->igvn();

   871 

   872   // Allocate several words more for the Shenandoah brooks pointer.

   873   size_in_bytes = new AddXNode(size_in_bytes, igvn.MakeConX(ShenandoahBrooksPointer::byte_size()));

   874   macro->transform_later(size_in_bytes);

   875 

   876   Node* fast_oop = BarrierSetC2::obj_allocate(macro, ctrl, mem, toobig_false, size_in_bytes,

   877                                               i_o, needgc_ctrl, fast_oop_ctrl, fast_oop_rawmem,

   878                                               prefetch_lines);

   879 

   880   // Bump up object for Shenandoah brooks pointer.

   881   fast_oop = new AddPNode(macro->top(), fast_oop, igvn.MakeConX(ShenandoahBrooksPointer::byte_size()));

   882   macro->transform_later(fast_oop);

   883 

   884   // Initialize Shenandoah brooks pointer to point to the object itself.

   885   fast_oop_rawmem = macro->make_store(fast_oop_ctrl, fast_oop_rawmem, fast_oop, ShenandoahBrooksPointer::byte_offset(), fast_oop, T_OBJECT);

   886 

   887   return fast_oop;

   888 }

   889 

   890 // Support for GC barriers emitted during parsing

   891 bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {

   892   if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {

   893     return false;

   894   }

   895   CallLeafNode *call = node->as_CallLeaf();

   896   if (call->_name == NULL) {

   897     return false;

   898   }

   899 

   900   return strcmp(call->_name, "shenandoah_clone_barrier") == 0 ||

   901          strcmp(call->_name, "shenandoah_cas_obj") == 0 ||

   902          strcmp(call->_name, "shenandoah_wb_pre") == 0;

   903 }

   904 

   905 Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {

   906   return ShenandoahBarrierNode::skip_through_barrier(c);

   907 }

   908 

   909 bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {

   910   return !ShenandoahWriteBarrierNode::expand(C, igvn);

   911 }

   912 

   913 bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const {

   914   if (mode == LoopOptsShenandoahExpand) {

   915     assert(UseShenandoahGC, "only for shenandoah");

   916     ShenandoahWriteBarrierNode::pin_and_expand(phase);

   917     return true;

   918   } else if (mode == LoopOptsShenandoahPostExpand) {

   919     assert(UseShenandoahGC, "only for shenandoah");

   920     visited.Clear();

   921     ShenandoahWriteBarrierNode::optimize_after_expansion(visited, nstack, worklist, phase);

   922     return true;

   923   }

   924   GrowableArray<MemoryGraphFixer*> memory_graph_fixers;

   925   ShenandoahWriteBarrierNode::optimize_before_expansion(phase, memory_graph_fixers, false);

   926   return false;

   927 }

   928 

   929 bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {

   930   bool is_oop = type == T_OBJECT || type == T_ARRAY;

   931   if (!is_oop) {

   932     return false;

   933   }

   934 

   935   if (tightly_coupled_alloc) {

   936     if (phase == Optimization) {

   937       return false;

   938     }

   939     return !is_clone;

   940   }

   941   if (phase == Optimization) {

   942     return !ShenandoahStoreValEnqueueBarrier;

   943   }

   944   return true;

   945 }

   946 

   947 bool ShenandoahBarrierSetC2::clone_needs_postbarrier(ArrayCopyNode *ac, PhaseIterGVN& igvn) {

   948   Node* src = ac->in(ArrayCopyNode::Src);

   949   const TypeOopPtr* src_type = igvn.type(src)->is_oopptr();

   950   if (src_type->isa_instptr() != NULL) {

   951     ciInstanceKlass* ik = src_type->klass()->as_instance_klass();

   952     if ((src_type->klass_is_exact() || (!ik->is_interface() && !ik->has_subklass())) && !ik->has_injected_fields()) {

   953       if (ik->has_object_fields()) {

   954         return true;

   955       } else {

   956         if (!src_type->klass_is_exact()) {

   957           igvn.C->dependencies()->assert_leaf_type(ik);

   958         }

   959       }

   960     } else {

   961       return true;

   962     }

   963   } else if (src_type->isa_aryptr()) {

   964     BasicType src_elem  = src_type->klass()->as_array_klass()->element_type()->basic_type();

   965     if (src_elem == T_OBJECT || src_elem == T_ARRAY) {

   966       return true;

   967     }

   968   } else {

   969     return true;

   970   }

   971   return false;

   972 }

   973 

   974 void ShenandoahBarrierSetC2::clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const {

   975   assert(ac->is_clonebasic(), "no other kind of arraycopy here");

   976 

   977   if (!clone_needs_postbarrier(ac, igvn)) {

   978     BarrierSetC2::clone_barrier_at_expansion(ac, call, igvn);

   979     return;

   980   }

   981 

   982   const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;

   983   Node* c = new ProjNode(call,TypeFunc::Control);

   984   c = igvn.transform(c);

   985   Node* m = new ProjNode(call, TypeFunc::Memory);

   986   c = igvn.transform(m);

   987 

   988   Node* dest = ac->in(ArrayCopyNode::Dest);

   989   assert(dest->is_AddP(), "bad input");

   990   Node* barrier_call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),

   991                                         CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),

   992                                         "shenandoah_clone_barrier", raw_adr_type);

   993   barrier_call->init_req(TypeFunc::Control, c);

   994   barrier_call->init_req(TypeFunc::I_O    , igvn.C->top());

   995   barrier_call->init_req(TypeFunc::Memory , m);

   996   barrier_call->init_req(TypeFunc::ReturnAdr, igvn.C->top());

   997   barrier_call->init_req(TypeFunc::FramePtr, igvn.C->top());

   998   barrier_call->init_req(TypeFunc::Parms+0, dest->in(AddPNode::Base));

   999 

  1000   barrier_call = igvn.transform(barrier_call);

  1001   c = new ProjNode(barrier_call,TypeFunc::Control);

  1002   c = igvn.transform(c);

  1003   m = new ProjNode(barrier_call, TypeFunc::Memory);

  1004   m = igvn.transform(m);

  1005 

  1006   Node* out_c = ac->proj_out(TypeFunc::Control);

  1007   Node* out_m = ac->proj_out(TypeFunc::Memory);

  1008   igvn.replace_node(out_c, c);

  1009   igvn.replace_node(out_m, m);

  1010 }

  1011 

  1012 

  1013 // Support for macro expanded GC barriers

  1014 void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {

  1015   if (node->Opcode() == Op_ShenandoahWriteBarrier) {

  1016     state()->add_shenandoah_barrier((ShenandoahWriteBarrierNode*) node);

  1017   }

  1018 }

  1019 

  1020 void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {

  1021   if (node->Opcode() == Op_ShenandoahWriteBarrier) {

  1022     state()->remove_shenandoah_barrier((ShenandoahWriteBarrierNode*) node);

  1023   }

  1024 }

  1025 

  1026 void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {

  1027   if (is_shenandoah_wb_pre_call(n)) {

  1028     shenandoah_eliminate_wb_pre(n, &macro->igvn());

  1029   }

  1030 }

  1031 

  1032 void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {

  1033   assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");

  1034   Node* c = call->as_Call()->proj_out(TypeFunc::Control);

  1035   c = c->unique_ctrl_out();

  1036   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");

  1037   c = c->unique_ctrl_out();

  1038   assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");

  1039   Node* iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);

  1040   assert(iff->is_If(), "expect test");

  1041   if (!is_shenandoah_marking_if(igvn, iff)) {

  1042     c = c->unique_ctrl_out();

  1043     assert(c->is_Region() && c->req() == 3, "where's the pre barrier control flow?");

  1044     iff = c->in(1)->is_IfProj() ? c->in(1)->in(0) : c->in(2)->in(0);

  1045     assert(is_shenandoah_marking_if(igvn, iff), "expect marking test");

  1046   }

  1047   Node* cmpx = iff->in(1)->in(1);

  1048   igvn->replace_node(cmpx, igvn->makecon(TypeInt::CC_EQ));

  1049   igvn->rehash_node_delayed(call);

  1050   call->del_req(call->req()-1);

  1051 }

  1052 

  1053 void ShenandoahBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {

  1054   if (node->Opcode() == Op_AddP && ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(node)) {

  1055     igvn->add_users_to_worklist(node);

  1056   }

  1057 }

  1058 

  1059 void ShenandoahBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {

  1060   for (uint i = 0; i < useful.size(); i++) {

  1061     Node* n = useful.at(i);

  1062     if (n->Opcode() == Op_AddP && ShenandoahBarrierSetC2::has_only_shenandoah_wb_pre_uses(n)) {

  1063       for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {

  1064         C->record_for_igvn(n->fast_out(i));

  1065       }

  1066     }

  1067   }

  1068   for (int i = state()->shenandoah_barriers_count()-1; i >= 0; i--) {

  1069     ShenandoahWriteBarrierNode* n = state()->shenandoah_barrier(i);

  1070     if (!useful.member(n)) {

  1071       state()->remove_shenandoah_barrier(n);

  1072     }

  1073   }

  1074 

  1075 }

  1076 

  1077 bool ShenandoahBarrierSetC2::has_special_unique_user(const Node* node) const {

  1078   assert(node->outcnt() == 1, "match only for unique out");

  1079   Node* n = node->unique_out();

  1080   return node->Opcode() == Op_ShenandoahWriteBarrier && n->Opcode() == Op_ShenandoahWBMemProj;

  1081 }

  1082 

  1083 void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {}

  1084 

  1085 void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {

  1086   return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena);

  1087 }

  1088 

  1089 ShenandoahBarrierSetC2State* ShenandoahBarrierSetC2::state() const {

  1090   return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());

  1091 }

  1092 

  1093 // If the BarrierSetC2 state has kept macro nodes in its compilation unit state to be

  1094 // expanded later, then now is the time to do so.

  1095 bool ShenandoahBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const { return false; }

  1096 

  1097 #ifdef ASSERT

  1098 void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {

  1099   if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeExpand) {

  1100     ShenandoahBarrierNode::verify(Compile::current()->root());

  1101   } else if (phase == BarrierSetC2::BeforeCodeGen) {

  1102     // Verify G1 pre-barriers

  1103     const int marking_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset());

  1104 

  1105     ResourceArea *area = Thread::current()->resource_area();

  1106     Unique_Node_List visited(area);

  1107     Node_List worklist(area);

  1108     // We're going to walk control flow backwards starting from the Root

  1109     worklist.push(compile->root());

  1110     while (worklist.size() > 0) {

  1111       Node *x = worklist.pop();

  1112       if (x == NULL || x == compile->top()) continue;

  1113       if (visited.member(x)) {

  1114         continue;

  1115       } else {

  1116         visited.push(x);

  1117       }

  1118 

  1119       if (x->is_Region()) {

  1120         for (uint i = 1; i < x->req(); i++) {

  1121           worklist.push(x->in(i));

  1122         }

  1123       } else {

  1124         worklist.push(x->in(0));

  1125         // We are looking for the pattern:

  1126         //                            /->ThreadLocal

  1127         // If->Bool->CmpI->LoadB->AddP->ConL(marking_offset)

  1128         //              \->ConI(0)

  1129         // We want to verify that the If and the LoadB have the same control

  1130         // See GraphKit::g1_write_barrier_pre()

  1131         if (x->is_If()) {

  1132           IfNode *iff = x->as_If();

  1133           if (iff->in(1)->is_Bool() && iff->in(1)->in(1)->is_Cmp()) {

  1134             CmpNode *cmp = iff->in(1)->in(1)->as_Cmp();

  1135             if (cmp->Opcode() == Op_CmpI && cmp->in(2)->is_Con() && cmp->in(2)->bottom_type()->is_int()->get_con() == 0