921 assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw || |
921 assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw || |
922 // oop will be recorded in oop map if load crosses safepoint |
922 // oop will be recorded in oop map if load crosses safepoint |
923 rt->isa_oopptr() || is_immutable_value(adr), |
923 rt->isa_oopptr() || is_immutable_value(adr), |
924 "raw memory operations should have control edge"); |
924 "raw memory operations should have control edge"); |
925 switch (bt) { |
925 switch (bt) { |
926 case T_BOOLEAN: return new (C) LoadUBNode(ctl, mem, adr, adr_type, rt->is_int() ); |
926 case T_BOOLEAN: return new (C) LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo); |
927 case T_BYTE: return new (C) LoadBNode (ctl, mem, adr, adr_type, rt->is_int() ); |
927 case T_BYTE: return new (C) LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo); |
928 case T_INT: return new (C) LoadINode (ctl, mem, adr, adr_type, rt->is_int() ); |
928 case T_INT: return new (C) LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo); |
929 case T_CHAR: return new (C) LoadUSNode(ctl, mem, adr, adr_type, rt->is_int() ); |
929 case T_CHAR: return new (C) LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo); |
930 case T_SHORT: return new (C) LoadSNode (ctl, mem, adr, adr_type, rt->is_int() ); |
930 case T_SHORT: return new (C) LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo); |
931 case T_LONG: return new (C) LoadLNode (ctl, mem, adr, adr_type, rt->is_long() ); |
931 case T_LONG: return new (C) LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo); |
932 case T_FLOAT: return new (C) LoadFNode (ctl, mem, adr, adr_type, rt ); |
932 case T_FLOAT: return new (C) LoadFNode (ctl, mem, adr, adr_type, rt, mo); |
933 case T_DOUBLE: return new (C) LoadDNode (ctl, mem, adr, adr_type, rt ); |
933 case T_DOUBLE: return new (C) LoadDNode (ctl, mem, adr, adr_type, rt, mo); |
934 case T_ADDRESS: return new (C) LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr() ); |
934 case T_ADDRESS: return new (C) LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo); |
935 case T_OBJECT: |
935 case T_OBJECT: |
936 #ifdef _LP64 |
936 #ifdef _LP64 |
937 if (adr->bottom_type()->is_ptr_to_narrowoop()) { |
937 if (adr->bottom_type()->is_ptr_to_narrowoop()) { |
938 Node* load = gvn.transform(new (C) LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop())); |
938 Node* load = gvn.transform(new (C) LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo)); |
939 return new (C) DecodeNNode(load, load->bottom_type()->make_ptr()); |
939 return new (C) DecodeNNode(load, load->bottom_type()->make_ptr()); |
940 } else |
940 } else |
941 #endif |
941 #endif |
942 { |
942 { |
943 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop"); |
943 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop"); |
944 return new (C) LoadPNode(ctl, mem, adr, adr_type, rt->is_oopptr()); |
944 return new (C) LoadPNode(ctl, mem, adr, adr_type, rt->is_oopptr(), mo); |
945 } |
945 } |
946 } |
946 } |
947 ShouldNotReachHere(); |
947 ShouldNotReachHere(); |
948 return (LoadNode*)NULL; |
948 return (LoadNode*)NULL; |
949 } |
949 } |
950 |
950 |
951 LoadLNode* LoadLNode::make_atomic(Compile *C, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt) { |
951 LoadLNode* LoadLNode::make_atomic(Compile *C, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, MemOrd mo) { |
952 bool require_atomic = true; |
952 bool require_atomic = true; |
953 return new (C) LoadLNode(ctl, mem, adr, adr_type, rt->is_long(), require_atomic); |
953 return new (C) LoadLNode(ctl, mem, adr, adr_type, rt->is_long(), mo, require_atomic); |
954 } |
954 } |
955 |
955 |
956 |
956 |
957 |
957 |
958 |
958 |
2030 const TypePtr *adr_type = adr->bottom_type()->isa_ptr(); |
2030 const TypePtr *adr_type = adr->bottom_type()->isa_ptr(); |
2031 assert(adr_type != NULL, "expecting TypeKlassPtr"); |
2031 assert(adr_type != NULL, "expecting TypeKlassPtr"); |
2032 #ifdef _LP64 |
2032 #ifdef _LP64 |
2033 if (adr_type->is_ptr_to_narrowklass()) { |
2033 if (adr_type->is_ptr_to_narrowklass()) { |
2034 assert(UseCompressedClassPointers, "no compressed klasses"); |
2034 assert(UseCompressedClassPointers, "no compressed klasses"); |
2035 Node* load_klass = gvn.transform(new (C) LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass())); |
2035 Node* load_klass = gvn.transform(new (C) LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered)); |
2036 return new (C) DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr()); |
2036 return new (C) DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr()); |
2037 } |
2037 } |
2038 #endif |
2038 #endif |
2039 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop"); |
2039 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop"); |
2040 return new (C) LoadKlassNode(ctl, mem, adr, at, tk); |
2040 return new (C) LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered); |
2041 } |
2041 } |
2042 |
2042 |
2043 //------------------------------Value------------------------------------------ |
2043 //------------------------------Value------------------------------------------ |
2044 const Type *LoadKlassNode::Value( PhaseTransform *phase ) const { |
2044 const Type *LoadKlassNode::Value( PhaseTransform *phase ) const { |
2045 return klass_value_common(phase); |
2045 return klass_value_common(phase); |
2345 } |
2345 } |
2346 |
2346 |
2347 //============================================================================= |
2347 //============================================================================= |
2348 //---------------------------StoreNode::make----------------------------------- |
2348 //---------------------------StoreNode::make----------------------------------- |
2349 // Polymorphic factory method: |
2349 // Polymorphic factory method: |
2350 StoreNode* StoreNode::make( PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt ) { |
2350 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo) { |
|
2351 assert((mo == unordered || mo == release), "unexpected"); |
2351 Compile* C = gvn.C; |
2352 Compile* C = gvn.C; |
2352 assert( C->get_alias_index(adr_type) != Compile::AliasIdxRaw || |
2353 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw || |
2353 ctl != NULL, "raw memory operations should have control edge"); |
2354 ctl != NULL, "raw memory operations should have control edge"); |
2354 |
2355 |
2355 switch (bt) { |
2356 switch (bt) { |
2356 case T_BOOLEAN: |
2357 case T_BOOLEAN: |
2357 case T_BYTE: return new (C) StoreBNode(ctl, mem, adr, adr_type, val); |
2358 case T_BYTE: return new (C) StoreBNode(ctl, mem, adr, adr_type, val, mo); |
2358 case T_INT: return new (C) StoreINode(ctl, mem, adr, adr_type, val); |
2359 case T_INT: return new (C) StoreINode(ctl, mem, adr, adr_type, val, mo); |
2359 case T_CHAR: |
2360 case T_CHAR: |
2360 case T_SHORT: return new (C) StoreCNode(ctl, mem, adr, adr_type, val); |
2361 case T_SHORT: return new (C) StoreCNode(ctl, mem, adr, adr_type, val, mo); |
2361 case T_LONG: return new (C) StoreLNode(ctl, mem, adr, adr_type, val); |
2362 case T_LONG: return new (C) StoreLNode(ctl, mem, adr, adr_type, val, mo); |
2362 case T_FLOAT: return new (C) StoreFNode(ctl, mem, adr, adr_type, val); |
2363 case T_FLOAT: return new (C) StoreFNode(ctl, mem, adr, adr_type, val, mo); |
2363 case T_DOUBLE: return new (C) StoreDNode(ctl, mem, adr, adr_type, val); |
2364 case T_DOUBLE: return new (C) StoreDNode(ctl, mem, adr, adr_type, val, mo); |
2364 case T_METADATA: |
2365 case T_METADATA: |
2365 case T_ADDRESS: |
2366 case T_ADDRESS: |
2366 case T_OBJECT: |
2367 case T_OBJECT: |
2367 #ifdef _LP64 |
2368 #ifdef _LP64 |
2368 if (adr->bottom_type()->is_ptr_to_narrowoop()) { |
2369 if (adr->bottom_type()->is_ptr_to_narrowoop()) { |
2369 val = gvn.transform(new (C) EncodePNode(val, val->bottom_type()->make_narrowoop())); |
2370 val = gvn.transform(new (C) EncodePNode(val, val->bottom_type()->make_narrowoop())); |
2370 return new (C) StoreNNode(ctl, mem, adr, adr_type, val); |
2371 return new (C) StoreNNode(ctl, mem, adr, adr_type, val, mo); |
2371 } else if (adr->bottom_type()->is_ptr_to_narrowklass() || |
2372 } else if (adr->bottom_type()->is_ptr_to_narrowklass() || |
2372 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() && |
2373 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() && |
2373 adr->bottom_type()->isa_rawptr())) { |
2374 adr->bottom_type()->isa_rawptr())) { |
2374 val = gvn.transform(new (C) EncodePKlassNode(val, val->bottom_type()->make_narrowklass())); |
2375 val = gvn.transform(new (C) EncodePKlassNode(val, val->bottom_type()->make_narrowklass())); |
2375 return new (C) StoreNKlassNode(ctl, mem, adr, adr_type, val); |
2376 return new (C) StoreNKlassNode(ctl, mem, adr, adr_type, val, mo); |
2376 } |
2377 } |
2377 #endif |
2378 #endif |
2378 { |
2379 { |
2379 return new (C) StorePNode(ctl, mem, adr, adr_type, val); |
2380 return new (C) StorePNode(ctl, mem, adr, adr_type, val, mo); |
2380 } |
2381 } |
2381 } |
2382 } |
2382 ShouldNotReachHere(); |
2383 ShouldNotReachHere(); |
2383 return (StoreNode*)NULL; |
2384 return (StoreNode*)NULL; |
2384 } |
2385 } |
2385 |
2386 |
2386 StoreLNode* StoreLNode::make_atomic(Compile *C, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val) { |
2387 StoreLNode* StoreLNode::make_atomic(Compile *C, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, MemOrd mo) { |
2387 bool require_atomic = true; |
2388 bool require_atomic = true; |
2388 return new (C) StoreLNode(ctl, mem, adr, adr_type, val, require_atomic); |
2389 return new (C) StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic); |
2389 } |
2390 } |
2390 |
2391 |
2391 |
2392 |
2392 //--------------------------bottom_type---------------------------------------- |
2393 //--------------------------bottom_type---------------------------------------- |
2393 const Type *StoreNode::bottom_type() const { |
2394 const Type *StoreNode::bottom_type() const { |
2776 if( adr->Opcode() != Op_AddP ) Unimplemented(); |
2777 if( adr->Opcode() != Op_AddP ) Unimplemented(); |
2777 Node *base = adr->in(1); |
2778 Node *base = adr->in(1); |
2778 |
2779 |
2779 Node *zero = phase->makecon(TypeLong::ZERO); |
2780 Node *zero = phase->makecon(TypeLong::ZERO); |
2780 Node *off = phase->MakeConX(BytesPerLong); |
2781 Node *off = phase->MakeConX(BytesPerLong); |
2781 mem = new (phase->C) StoreLNode(in(0),mem,adr,atp,zero); |
2782 mem = new (phase->C) StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false); |
2782 count--; |
2783 count--; |
2783 while( count-- ) { |
2784 while( count-- ) { |
2784 mem = phase->transform(mem); |
2785 mem = phase->transform(mem); |
2785 adr = phase->transform(new (phase->C) AddPNode(base,adr,off)); |
2786 adr = phase->transform(new (phase->C) AddPNode(base,adr,off)); |
2786 mem = new (phase->C) StoreLNode(in(0),mem,adr,atp,zero); |
2787 mem = new (phase->C) StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false); |
2787 } |
2788 } |
2788 return mem; |
2789 return mem; |
2789 } |
2790 } |
2790 |
2791 |
2791 //----------------------------step_through---------------------------------- |
2792 //----------------------------step_through---------------------------------- |
2825 int unit = BytesPerLong; |
2826 int unit = BytesPerLong; |
2826 if ((offset % unit) != 0) { |
2827 if ((offset % unit) != 0) { |
2827 Node* adr = new (C) AddPNode(dest, dest, phase->MakeConX(offset)); |
2828 Node* adr = new (C) AddPNode(dest, dest, phase->MakeConX(offset)); |
2828 adr = phase->transform(adr); |
2829 adr = phase->transform(adr); |
2829 const TypePtr* atp = TypeRawPtr::BOTTOM; |
2830 const TypePtr* atp = TypeRawPtr::BOTTOM; |
2830 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT); |
2831 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered); |
2831 mem = phase->transform(mem); |
2832 mem = phase->transform(mem); |
2832 offset += BytesPerInt; |
2833 offset += BytesPerInt; |
2833 } |
2834 } |
2834 assert((offset % unit) == 0, ""); |
2835 assert((offset % unit) == 0, ""); |
2835 |
2836 |
2886 } |
2887 } |
2887 if (done_offset < end_offset) { // emit the final 32-bit store |
2888 if (done_offset < end_offset) { // emit the final 32-bit store |
2888 Node* adr = new (C) AddPNode(dest, dest, phase->MakeConX(done_offset)); |
2889 Node* adr = new (C) AddPNode(dest, dest, phase->MakeConX(done_offset)); |
2889 adr = phase->transform(adr); |
2890 adr = phase->transform(adr); |
2890 const TypePtr* atp = TypeRawPtr::BOTTOM; |
2891 const TypePtr* atp = TypeRawPtr::BOTTOM; |
2891 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT); |
2892 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered); |
2892 mem = phase->transform(mem); |
2893 mem = phase->transform(mem); |
2893 done_offset += BytesPerInt; |
2894 done_offset += BytesPerInt; |
2894 } |
2895 } |
2895 assert(done_offset == end_offset, ""); |
2896 assert(done_offset == end_offset, ""); |
2896 return mem; |
2897 return mem; |
3760 int nst = 0; |
3761 int nst = 0; |
3761 if (!split) { |
3762 if (!split) { |
3762 ++new_long; |
3763 ++new_long; |
3763 off[nst] = offset; |
3764 off[nst] = offset; |
3764 st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp, |
3765 st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp, |
3765 phase->longcon(con), T_LONG); |
3766 phase->longcon(con), T_LONG, MemNode::unordered); |
3766 } else { |
3767 } else { |
3767 // Omit either if it is a zero. |
3768 // Omit either if it is a zero. |
3768 if (con0 != 0) { |
3769 if (con0 != 0) { |
3769 ++new_int; |
3770 ++new_int; |
3770 off[nst] = offset; |
3771 off[nst] = offset; |
3771 st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp, |
3772 st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp, |
3772 phase->intcon(con0), T_INT); |
3773 phase->intcon(con0), T_INT, MemNode::unordered); |
3773 } |
3774 } |
3774 if (con1 != 0) { |
3775 if (con1 != 0) { |
3775 ++new_int; |
3776 ++new_int; |
3776 offset += BytesPerInt; |
3777 offset += BytesPerInt; |
3777 adr = make_raw_address(offset, phase); |
3778 adr = make_raw_address(offset, phase); |
3778 off[nst] = offset; |
3779 off[nst] = offset; |
3779 st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp, |
3780 st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp, |
3780 phase->intcon(con1), T_INT); |
3781 phase->intcon(con1), T_INT, MemNode::unordered); |
3781 } |
3782 } |
3782 } |
3783 } |
3783 |
3784 |
3784 // Insert second store first, then the first before the second. |
3785 // Insert second store first, then the first before the second. |
3785 // Insert each one just before any overlapping non-constant stores. |
3786 // Insert each one just before any overlapping non-constant stores. |