921 assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw || |
921 assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw || |
922 // oop will be recorded in oop map if load crosses safepoint |
922 // oop will be recorded in oop map if load crosses safepoint |
923 rt->isa_oopptr() || is_immutable_value(adr), |
923 rt->isa_oopptr() || is_immutable_value(adr), |
924 "raw memory operations should have control edge"); |
924 "raw memory operations should have control edge"); |
925 switch (bt) { |
925 switch (bt) { |
926 case T_BOOLEAN: return new (C) LoadUBNode(ctl, mem, adr, adr_type, rt->is_int() ); |
926 case T_BOOLEAN: return new (C) LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo); |
927 case T_BYTE: return new (C) LoadBNode (ctl, mem, adr, adr_type, rt->is_int() ); |
927 case T_BYTE: return new (C) LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo); |
928 case T_INT: return new (C) LoadINode (ctl, mem, adr, adr_type, rt->is_int() ); |
928 case T_INT: return new (C) LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo); |
929 case T_CHAR: return new (C) LoadUSNode(ctl, mem, adr, adr_type, rt->is_int() ); |
929 case T_CHAR: return new (C) LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo); |
930 case T_SHORT: return new (C) LoadSNode (ctl, mem, adr, adr_type, rt->is_int() ); |
930 case T_SHORT: return new (C) LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo); |
931 case T_LONG: return new (C) LoadLNode (ctl, mem, adr, adr_type, rt->is_long() ); |
931 case T_LONG: return new (C) LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo); |
932 case T_FLOAT: return new (C) LoadFNode (ctl, mem, adr, adr_type, rt ); |
932 case T_FLOAT: return new (C) LoadFNode (ctl, mem, adr, adr_type, rt, mo); |
933 case T_DOUBLE: return new (C) LoadDNode (ctl, mem, adr, adr_type, rt ); |
933 case T_DOUBLE: return new (C) LoadDNode (ctl, mem, adr, adr_type, rt, mo); |
934 case T_ADDRESS: return new (C) LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr() ); |
934 case T_ADDRESS: return new (C) LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo); |
935 case T_OBJECT: |
935 case T_OBJECT: |
936 #ifdef _LP64 |
936 #ifdef _LP64 |
937 if (adr->bottom_type()->is_ptr_to_narrowoop()) { |
937 if (adr->bottom_type()->is_ptr_to_narrowoop()) { |
938 Node* load = gvn.transform(new (C) LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop())); |
938 Node* load = gvn.transform(new (C) LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo)); |
939 return new (C) DecodeNNode(load, load->bottom_type()->make_ptr()); |
939 return new (C) DecodeNNode(load, load->bottom_type()->make_ptr()); |
940 } else |
940 } else |
941 #endif |
941 #endif |
942 { |
942 { |
943 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop"); |
943 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop"); |
944 return new (C) LoadPNode(ctl, mem, adr, adr_type, rt->is_oopptr()); |
944 return new (C) LoadPNode(ctl, mem, adr, adr_type, rt->is_oopptr(), mo); |
945 } |
945 } |
946 } |
946 } |
947 ShouldNotReachHere(); |
947 ShouldNotReachHere(); |
948 return (LoadNode*)NULL; |
948 return (LoadNode*)NULL; |
949 } |
949 } |
950 |
950 |
951 LoadLNode* LoadLNode::make_atomic(Compile *C, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt) { |
951 LoadLNode* LoadLNode::make_atomic(Compile *C, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, MemOrd mo) { |
952 bool require_atomic = true; |
952 bool require_atomic = true; |
953 return new (C) LoadLNode(ctl, mem, adr, adr_type, rt->is_long(), require_atomic); |
953 return new (C) LoadLNode(ctl, mem, adr, adr_type, rt->is_long(), mo, require_atomic); |
954 } |
954 } |
955 |
955 |
956 |
956 |
957 |
957 |
958 |
958 |
1000 // through any kind of MemBar but normal loads shouldn't skip |
1000 // through any kind of MemBar but normal loads shouldn't skip |
1001 // through MemBarAcquire since the could allow them to move out of |
1001 // through MemBarAcquire since the could allow them to move out of |
1002 // a synchronized region. |
1002 // a synchronized region. |
1003 while (current->is_Proj()) { |
1003 while (current->is_Proj()) { |
1004 int opc = current->in(0)->Opcode(); |
1004 int opc = current->in(0)->Opcode(); |
1005 if ((final && (opc == Op_MemBarAcquire || opc == Op_MemBarAcquireLock)) || |
1005 if ((final && (opc == Op_MemBarAcquire || |
1006 opc == Op_MemBarRelease || opc == Op_MemBarCPUOrder || |
1006 opc == Op_MemBarAcquireLock || |
1007 opc == Op_MemBarReleaseLock) { |
1007 opc == Op_LoadFence)) || |
|
1008 opc == Op_MemBarRelease || |
|
1009 opc == Op_StoreFence || |
|
1010 opc == Op_MemBarReleaseLock || |
|
1011 opc == Op_MemBarCPUOrder) { |
1008 Node* mem = current->in(0)->in(TypeFunc::Memory); |
1012 Node* mem = current->in(0)->in(TypeFunc::Memory); |
1009 if (mem->is_MergeMem()) { |
1013 if (mem->is_MergeMem()) { |
1010 MergeMemNode* merge = mem->as_MergeMem(); |
1014 MergeMemNode* merge = mem->as_MergeMem(); |
1011 Node* new_st = merge->memory_at(alias_idx); |
1015 Node* new_st = merge->memory_at(alias_idx); |
1012 if (new_st == merge->base_memory()) { |
1016 if (new_st == merge->base_memory()) { |
2030 const TypePtr *adr_type = adr->bottom_type()->isa_ptr(); |
2034 const TypePtr *adr_type = adr->bottom_type()->isa_ptr(); |
2031 assert(adr_type != NULL, "expecting TypeKlassPtr"); |
2035 assert(adr_type != NULL, "expecting TypeKlassPtr"); |
2032 #ifdef _LP64 |
2036 #ifdef _LP64 |
2033 if (adr_type->is_ptr_to_narrowklass()) { |
2037 if (adr_type->is_ptr_to_narrowklass()) { |
2034 assert(UseCompressedClassPointers, "no compressed klasses"); |
2038 assert(UseCompressedClassPointers, "no compressed klasses"); |
2035 Node* load_klass = gvn.transform(new (C) LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass())); |
2039 Node* load_klass = gvn.transform(new (C) LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered)); |
2036 return new (C) DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr()); |
2040 return new (C) DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr()); |
2037 } |
2041 } |
2038 #endif |
2042 #endif |
2039 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop"); |
2043 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop"); |
2040 return new (C) LoadKlassNode(ctl, mem, adr, at, tk); |
2044 return new (C) LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered); |
2041 } |
2045 } |
2042 |
2046 |
2043 //------------------------------Value------------------------------------------ |
2047 //------------------------------Value------------------------------------------ |
2044 const Type *LoadKlassNode::Value( PhaseTransform *phase ) const { |
2048 const Type *LoadKlassNode::Value( PhaseTransform *phase ) const { |
2045 return klass_value_common(phase); |
2049 return klass_value_common(phase); |
2350 } |
2354 } |
2351 |
2355 |
2352 //============================================================================= |
2356 //============================================================================= |
2353 //---------------------------StoreNode::make----------------------------------- |
2357 //---------------------------StoreNode::make----------------------------------- |
2354 // Polymorphic factory method: |
2358 // Polymorphic factory method: |
2355 StoreNode* StoreNode::make( PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt ) { |
2359 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo) { |
|
2360 assert((mo == unordered || mo == release), "unexpected"); |
2356 Compile* C = gvn.C; |
2361 Compile* C = gvn.C; |
2357 assert( C->get_alias_index(adr_type) != Compile::AliasIdxRaw || |
2362 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw || |
2358 ctl != NULL, "raw memory operations should have control edge"); |
2363 ctl != NULL, "raw memory operations should have control edge"); |
2359 |
2364 |
2360 switch (bt) { |
2365 switch (bt) { |
2361 case T_BOOLEAN: |
2366 case T_BOOLEAN: |
2362 case T_BYTE: return new (C) StoreBNode(ctl, mem, adr, adr_type, val); |
2367 case T_BYTE: return new (C) StoreBNode(ctl, mem, adr, adr_type, val, mo); |
2363 case T_INT: return new (C) StoreINode(ctl, mem, adr, adr_type, val); |
2368 case T_INT: return new (C) StoreINode(ctl, mem, adr, adr_type, val, mo); |
2364 case T_CHAR: |
2369 case T_CHAR: |
2365 case T_SHORT: return new (C) StoreCNode(ctl, mem, adr, adr_type, val); |
2370 case T_SHORT: return new (C) StoreCNode(ctl, mem, adr, adr_type, val, mo); |
2366 case T_LONG: return new (C) StoreLNode(ctl, mem, adr, adr_type, val); |
2371 case T_LONG: return new (C) StoreLNode(ctl, mem, adr, adr_type, val, mo); |
2367 case T_FLOAT: return new (C) StoreFNode(ctl, mem, adr, adr_type, val); |
2372 case T_FLOAT: return new (C) StoreFNode(ctl, mem, adr, adr_type, val, mo); |
2368 case T_DOUBLE: return new (C) StoreDNode(ctl, mem, adr, adr_type, val); |
2373 case T_DOUBLE: return new (C) StoreDNode(ctl, mem, adr, adr_type, val, mo); |
2369 case T_METADATA: |
2374 case T_METADATA: |
2370 case T_ADDRESS: |
2375 case T_ADDRESS: |
2371 case T_OBJECT: |
2376 case T_OBJECT: |
2372 #ifdef _LP64 |
2377 #ifdef _LP64 |
2373 if (adr->bottom_type()->is_ptr_to_narrowoop()) { |
2378 if (adr->bottom_type()->is_ptr_to_narrowoop()) { |
2374 val = gvn.transform(new (C) EncodePNode(val, val->bottom_type()->make_narrowoop())); |
2379 val = gvn.transform(new (C) EncodePNode(val, val->bottom_type()->make_narrowoop())); |
2375 return new (C) StoreNNode(ctl, mem, adr, adr_type, val); |
2380 return new (C) StoreNNode(ctl, mem, adr, adr_type, val, mo); |
2376 } else if (adr->bottom_type()->is_ptr_to_narrowklass() || |
2381 } else if (adr->bottom_type()->is_ptr_to_narrowklass() || |
2377 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() && |
2382 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() && |
2378 adr->bottom_type()->isa_rawptr())) { |
2383 adr->bottom_type()->isa_rawptr())) { |
2379 val = gvn.transform(new (C) EncodePKlassNode(val, val->bottom_type()->make_narrowklass())); |
2384 val = gvn.transform(new (C) EncodePKlassNode(val, val->bottom_type()->make_narrowklass())); |
2380 return new (C) StoreNKlassNode(ctl, mem, adr, adr_type, val); |
2385 return new (C) StoreNKlassNode(ctl, mem, adr, adr_type, val, mo); |
2381 } |
2386 } |
2382 #endif |
2387 #endif |
2383 { |
2388 { |
2384 return new (C) StorePNode(ctl, mem, adr, adr_type, val); |
2389 return new (C) StorePNode(ctl, mem, adr, adr_type, val, mo); |
2385 } |
2390 } |
2386 } |
2391 } |
2387 ShouldNotReachHere(); |
2392 ShouldNotReachHere(); |
2388 return (StoreNode*)NULL; |
2393 return (StoreNode*)NULL; |
2389 } |
2394 } |
2390 |
2395 |
2391 StoreLNode* StoreLNode::make_atomic(Compile *C, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val) { |
2396 StoreLNode* StoreLNode::make_atomic(Compile *C, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, MemOrd mo) { |
2392 bool require_atomic = true; |
2397 bool require_atomic = true; |
2393 return new (C) StoreLNode(ctl, mem, adr, adr_type, val, require_atomic); |
2398 return new (C) StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic); |
2394 } |
2399 } |
2395 |
2400 |
2396 |
2401 |
2397 //--------------------------bottom_type---------------------------------------- |
2402 //--------------------------bottom_type---------------------------------------- |
2398 const Type *StoreNode::bottom_type() const { |
2403 const Type *StoreNode::bottom_type() const { |
2781 if( adr->Opcode() != Op_AddP ) Unimplemented(); |
2786 if( adr->Opcode() != Op_AddP ) Unimplemented(); |
2782 Node *base = adr->in(1); |
2787 Node *base = adr->in(1); |
2783 |
2788 |
2784 Node *zero = phase->makecon(TypeLong::ZERO); |
2789 Node *zero = phase->makecon(TypeLong::ZERO); |
2785 Node *off = phase->MakeConX(BytesPerLong); |
2790 Node *off = phase->MakeConX(BytesPerLong); |
2786 mem = new (phase->C) StoreLNode(in(0),mem,adr,atp,zero); |
2791 mem = new (phase->C) StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false); |
2787 count--; |
2792 count--; |
2788 while( count-- ) { |
2793 while( count-- ) { |
2789 mem = phase->transform(mem); |
2794 mem = phase->transform(mem); |
2790 adr = phase->transform(new (phase->C) AddPNode(base,adr,off)); |
2795 adr = phase->transform(new (phase->C) AddPNode(base,adr,off)); |
2791 mem = new (phase->C) StoreLNode(in(0),mem,adr,atp,zero); |
2796 mem = new (phase->C) StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false); |
2792 } |
2797 } |
2793 return mem; |
2798 return mem; |
2794 } |
2799 } |
2795 |
2800 |
2796 //----------------------------step_through---------------------------------- |
2801 //----------------------------step_through---------------------------------- |
2830 int unit = BytesPerLong; |
2835 int unit = BytesPerLong; |
2831 if ((offset % unit) != 0) { |
2836 if ((offset % unit) != 0) { |
2832 Node* adr = new (C) AddPNode(dest, dest, phase->MakeConX(offset)); |
2837 Node* adr = new (C) AddPNode(dest, dest, phase->MakeConX(offset)); |
2833 adr = phase->transform(adr); |
2838 adr = phase->transform(adr); |
2834 const TypePtr* atp = TypeRawPtr::BOTTOM; |
2839 const TypePtr* atp = TypeRawPtr::BOTTOM; |
2835 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT); |
2840 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered); |
2836 mem = phase->transform(mem); |
2841 mem = phase->transform(mem); |
2837 offset += BytesPerInt; |
2842 offset += BytesPerInt; |
2838 } |
2843 } |
2839 assert((offset % unit) == 0, ""); |
2844 assert((offset % unit) == 0, ""); |
2840 |
2845 |
2891 } |
2896 } |
2892 if (done_offset < end_offset) { // emit the final 32-bit store |
2897 if (done_offset < end_offset) { // emit the final 32-bit store |
2893 Node* adr = new (C) AddPNode(dest, dest, phase->MakeConX(done_offset)); |
2898 Node* adr = new (C) AddPNode(dest, dest, phase->MakeConX(done_offset)); |
2894 adr = phase->transform(adr); |
2899 adr = phase->transform(adr); |
2895 const TypePtr* atp = TypeRawPtr::BOTTOM; |
2900 const TypePtr* atp = TypeRawPtr::BOTTOM; |
2896 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT); |
2901 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered); |
2897 mem = phase->transform(mem); |
2902 mem = phase->transform(mem); |
2898 done_offset += BytesPerInt; |
2903 done_offset += BytesPerInt; |
2899 } |
2904 } |
2900 assert(done_offset == end_offset, ""); |
2905 assert(done_offset == end_offset, ""); |
2901 return mem; |
2906 return mem; |
2975 } |
2980 } |
2976 |
2981 |
2977 //------------------------------make------------------------------------------- |
2982 //------------------------------make------------------------------------------- |
2978 MemBarNode* MemBarNode::make(Compile* C, int opcode, int atp, Node* pn) { |
2983 MemBarNode* MemBarNode::make(Compile* C, int opcode, int atp, Node* pn) { |
2979 switch (opcode) { |
2984 switch (opcode) { |
2980 case Op_MemBarAcquire: return new(C) MemBarAcquireNode(C, atp, pn); |
2985 case Op_MemBarAcquire: return new(C) MemBarAcquireNode(C, atp, pn); |
2981 case Op_MemBarRelease: return new(C) MemBarReleaseNode(C, atp, pn); |
2986 case Op_LoadFence: return new(C) LoadFenceNode(C, atp, pn); |
2982 case Op_MemBarAcquireLock: return new(C) MemBarAcquireLockNode(C, atp, pn); |
2987 case Op_MemBarRelease: return new(C) MemBarReleaseNode(C, atp, pn); |
2983 case Op_MemBarReleaseLock: return new(C) MemBarReleaseLockNode(C, atp, pn); |
2988 case Op_StoreFence: return new(C) StoreFenceNode(C, atp, pn); |
2984 case Op_MemBarVolatile: return new(C) MemBarVolatileNode(C, atp, pn); |
2989 case Op_MemBarAcquireLock: return new(C) MemBarAcquireLockNode(C, atp, pn); |
2985 case Op_MemBarCPUOrder: return new(C) MemBarCPUOrderNode(C, atp, pn); |
2990 case Op_MemBarReleaseLock: return new(C) MemBarReleaseLockNode(C, atp, pn); |
2986 case Op_Initialize: return new(C) InitializeNode(C, atp, pn); |
2991 case Op_MemBarVolatile: return new(C) MemBarVolatileNode(C, atp, pn); |
2987 case Op_MemBarStoreStore: return new(C) MemBarStoreStoreNode(C, atp, pn); |
2992 case Op_MemBarCPUOrder: return new(C) MemBarCPUOrderNode(C, atp, pn); |
2988 default: ShouldNotReachHere(); return NULL; |
2993 case Op_Initialize: return new(C) InitializeNode(C, atp, pn); |
|
2994 case Op_MemBarStoreStore: return new(C) MemBarStoreStoreNode(C, atp, pn); |
|
2995 default: ShouldNotReachHere(); return NULL; |
2989 } |
2996 } |
2990 } |
2997 } |
2991 |
2998 |
2992 //------------------------------Ideal------------------------------------------ |
2999 //------------------------------Ideal------------------------------------------ |
2993 // Return a node which is more "ideal" than the current node. Strip out |
3000 // Return a node which is more "ideal" than the current node. Strip out |
3765 int nst = 0; |
3772 int nst = 0; |
3766 if (!split) { |
3773 if (!split) { |
3767 ++new_long; |
3774 ++new_long; |
3768 off[nst] = offset; |
3775 off[nst] = offset; |
3769 st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp, |
3776 st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp, |
3770 phase->longcon(con), T_LONG); |
3777 phase->longcon(con), T_LONG, MemNode::unordered); |
3771 } else { |
3778 } else { |
3772 // Omit either if it is a zero. |
3779 // Omit either if it is a zero. |
3773 if (con0 != 0) { |
3780 if (con0 != 0) { |
3774 ++new_int; |
3781 ++new_int; |
3775 off[nst] = offset; |
3782 off[nst] = offset; |
3776 st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp, |
3783 st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp, |
3777 phase->intcon(con0), T_INT); |
3784 phase->intcon(con0), T_INT, MemNode::unordered); |
3778 } |
3785 } |
3779 if (con1 != 0) { |
3786 if (con1 != 0) { |
3780 ++new_int; |
3787 ++new_int; |
3781 offset += BytesPerInt; |
3788 offset += BytesPerInt; |
3782 adr = make_raw_address(offset, phase); |
3789 adr = make_raw_address(offset, phase); |
3783 off[nst] = offset; |
3790 off[nst] = offset; |
3784 st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp, |
3791 st[nst++] = StoreNode::make(*phase, ctl, zmem, adr, atp, |
3785 phase->intcon(con1), T_INT); |
3792 phase->intcon(con1), T_INT, MemNode::unordered); |
3786 } |
3793 } |
3787 } |
3794 } |
3788 |
3795 |
3789 // Insert second store first, then the first before the second. |
3796 // Insert second store first, then the first before the second. |
3790 // Insert each one just before any overlapping non-constant stores. |
3797 // Insert each one just before any overlapping non-constant stores. |