22 // |
22 // |
23 |
23 |
24 source_hpp %{ |
24 source_hpp %{ |
25 |
25 |
26 #include "gc/z/c2/zBarrierSetC2.hpp" |
26 #include "gc/z/c2/zBarrierSetC2.hpp" |
|
27 #include "gc/z/zThreadLocalData.hpp" |
27 |
28 |
28 %} |
29 %} |
29 |
30 |
30 source %{ |
31 source %{ |
31 |
32 |
32 #include "gc/z/zBarrierSetAssembler.hpp" |
33 static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) { |
33 |
34 ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak); |
34 static void z_load_barrier_slow_reg(MacroAssembler& _masm, Register dst, |
35 __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); |
35 Register base, int index, int scale, |
36 __ andr(tmp, tmp, ref); |
36 int disp, bool weak) { |
37 __ cbnz(tmp, *stub->entry()); |
37 const address stub = weak ? ZBarrierSet::assembler()->load_barrier_weak_slow_stub(dst) |
38 __ bind(*stub->continuation()); |
38 : ZBarrierSet::assembler()->load_barrier_slow_stub(dst); |
|
39 |
|
40 if (index == -1) { |
|
41 if (disp != 0) { |
|
42 __ lea(dst, Address(base, disp)); |
|
43 } else { |
|
44 __ mov(dst, base); |
|
45 } |
|
46 } else { |
|
47 Register index_reg = as_Register(index); |
|
48 if (disp == 0) { |
|
49 __ lea(dst, Address(base, index_reg, Address::lsl(scale))); |
|
50 } else { |
|
51 __ lea(dst, Address(base, disp)); |
|
52 __ lea(dst, Address(dst, index_reg, Address::lsl(scale))); |
|
53 } |
|
54 } |
|
55 |
|
56 __ far_call(RuntimeAddress(stub)); |
|
57 } |
39 } |
58 |
40 |
59 %} |
41 static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) { |
60 |
42 ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */); |
61 // |
43 __ b(*stub->entry()); |
62 // Execute ZGC load barrier (strong) slow path |
44 __ bind(*stub->continuation()); |
63 // |
45 } |
64 instruct loadBarrierSlowReg(iRegP dst, memory src, rFlagsReg cr, |
46 |
65 vRegD_V0 v0, vRegD_V1 v1, vRegD_V2 v2, vRegD_V3 v3, vRegD_V4 v4, |
47 %} |
66 vRegD_V5 v5, vRegD_V6 v6, vRegD_V7 v7, vRegD_V8 v8, vRegD_V9 v9, |
48 |
67 vRegD_V10 v10, vRegD_V11 v11, vRegD_V12 v12, vRegD_V13 v13, vRegD_V14 v14, |
49 // Load Pointer |
68 vRegD_V15 v15, vRegD_V16 v16, vRegD_V17 v17, vRegD_V18 v18, vRegD_V19 v19, |
50 instruct zLoadP(iRegPNoSp dst, memory mem, rFlagsReg cr) |
69 vRegD_V20 v20, vRegD_V21 v21, vRegD_V22 v22, vRegD_V23 v23, vRegD_V24 v24, |
51 %{ |
70 vRegD_V25 v25, vRegD_V26 v26, vRegD_V27 v27, vRegD_V28 v28, vRegD_V29 v29, |
52 match(Set dst (LoadP mem)); |
71 vRegD_V30 v30, vRegD_V31 v31) %{ |
53 predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierStrong)); |
72 match(Set dst (LoadBarrierSlowReg src dst)); |
54 effect(TEMP dst, KILL cr); |
73 predicate(!n->as_LoadBarrierSlowReg()->is_weak()); |
55 |
74 |
56 ins_cost(4 * INSN_COST); |
75 effect(KILL cr, |
57 |
76 KILL v0, KILL v1, KILL v2, KILL v3, KILL v4, KILL v5, KILL v6, KILL v7, |
58 format %{ "ldr $dst, $mem" %} |
77 KILL v8, KILL v9, KILL v10, KILL v11, KILL v12, KILL v13, KILL v14, |
59 |
78 KILL v15, KILL v16, KILL v17, KILL v18, KILL v19, KILL v20, KILL v21, |
60 ins_encode %{ |
79 KILL v22, KILL v23, KILL v24, KILL v25, KILL v26, KILL v27, KILL v28, |
61 const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); |
80 KILL v29, KILL v30, KILL v31); |
62 __ ldr($dst$$Register, ref_addr); |
81 |
63 if (barrier_data() != ZLoadBarrierElided) { |
82 format %{ "lea $dst, $src\n\t" |
64 z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, false /* weak */); |
83 "call #ZLoadBarrierSlowPath" %} |
65 } |
84 |
66 %} |
85 ins_encode %{ |
67 |
86 z_load_barrier_slow_reg(_masm, $dst$$Register, $src$$base$$Register, |
68 ins_pipe(iload_reg_mem); |
87 $src$$index, $src$$scale, $src$$disp, false); |
69 %} |
88 %} |
70 |
89 ins_pipe(pipe_slow); |
71 // Load Weak Pointer |
90 %} |
72 instruct zLoadWeakP(iRegPNoSp dst, memory mem, rFlagsReg cr) |
91 |
73 %{ |
92 // |
74 match(Set dst (LoadP mem)); |
93 // Execute ZGC load barrier (weak) slow path |
75 predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() == ZLoadBarrierWeak)); |
94 // |
76 effect(TEMP dst, KILL cr); |
95 instruct loadBarrierWeakSlowReg(iRegP dst, memory src, rFlagsReg cr, |
77 |
96 vRegD_V0 v0, vRegD_V1 v1, vRegD_V2 v2, vRegD_V3 v3, vRegD_V4 v4, |
78 ins_cost(4 * INSN_COST); |
97 vRegD_V5 v5, vRegD_V6 v6, vRegD_V7 v7, vRegD_V8 v8, vRegD_V9 v9, |
79 |
98 vRegD_V10 v10, vRegD_V11 v11, vRegD_V12 v12, vRegD_V13 v13, vRegD_V14 v14, |
80 format %{ "ldr $dst, $mem" %} |
99 vRegD_V15 v15, vRegD_V16 v16, vRegD_V17 v17, vRegD_V18 v18, vRegD_V19 v19, |
81 |
100 vRegD_V20 v20, vRegD_V21 v21, vRegD_V22 v22, vRegD_V23 v23, vRegD_V24 v24, |
82 ins_encode %{ |
101 vRegD_V25 v25, vRegD_V26 v26, vRegD_V27 v27, vRegD_V28 v28, vRegD_V29 v29, |
83 const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp); |
102 vRegD_V30 v30, vRegD_V31 v31) %{ |
84 __ ldr($dst$$Register, ref_addr); |
103 match(Set dst (LoadBarrierSlowReg src dst)); |
85 z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, true /* weak */); |
104 predicate(n->as_LoadBarrierSlowReg()->is_weak()); |
86 %} |
105 |
87 |
106 effect(KILL cr, |
88 ins_pipe(iload_reg_mem); |
107 KILL v0, KILL v1, KILL v2, KILL v3, KILL v4, KILL v5, KILL v6, KILL v7, |
89 %} |
108 KILL v8, KILL v9, KILL v10, KILL v11, KILL v12, KILL v13, KILL v14, |
90 |
109 KILL v15, KILL v16, KILL v17, KILL v18, KILL v19, KILL v20, KILL v21, |
91 // Load Pointer Volatile |
110 KILL v22, KILL v23, KILL v24, KILL v25, KILL v26, KILL v27, KILL v28, |
92 instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr) |
111 KILL v29, KILL v30, KILL v31); |
93 %{ |
112 |
94 match(Set dst (LoadP mem)); |
113 format %{ "lea $dst, $src\n\t" |
95 predicate(UseZGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() == ZLoadBarrierStrong); |
114 "call #ZLoadBarrierSlowPath" %} |
96 effect(TEMP dst, KILL cr); |
115 |
97 |
116 ins_encode %{ |
98 ins_cost(VOLATILE_REF_COST); |
117 z_load_barrier_slow_reg(_masm, $dst$$Register, $src$$base$$Register, |
99 |
118 $src$$index, $src$$scale, $src$$disp, true); |
100 format %{ "ldar $dst, $mem\t" %} |
119 %} |
101 |
120 ins_pipe(pipe_slow); |
102 ins_encode %{ |
121 %} |
103 __ ldar($dst$$Register, $mem$$Register); |
122 |
104 if (barrier_data() != ZLoadBarrierElided) { |
123 |
105 z_load_barrier(_masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, false /* weak */); |
124 // Specialized versions of compareAndExchangeP that adds a keepalive that is consumed |
106 } |
125 // but doesn't affect output. |
107 %} |
126 |
108 |
127 instruct z_compareAndExchangeP(iRegPNoSp res, indirect mem, |
109 ins_pipe(pipe_serial); |
128 iRegP oldval, iRegP newval, iRegP keepalive, |
110 %} |
129 rFlagsReg cr) %{ |
111 |
130 match(Set res (ZCompareAndExchangeP (Binary mem keepalive) (Binary oldval newval))); |
112 instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
131 ins_cost(2 * VOLATILE_REF_COST); |
113 match(Set res (CompareAndSwapP mem (Binary oldval newval))); |
|
114 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); |
|
115 predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); |
|
116 effect(KILL cr, TEMP_DEF res); |
|
117 |
|
118 ins_cost(2 * VOLATILE_REF_COST); |
|
119 |
|
120 format %{ "cmpxchg $mem, $oldval, $newval\n\t" |
|
121 "cset $res, EQ" %} |
|
122 |
|
123 ins_encode %{ |
|
124 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); |
|
125 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, |
|
126 false /* acquire */, true /* release */, false /* weak */, rscratch2); |
|
127 __ cset($res$$Register, Assembler::EQ); |
|
128 if (barrier_data() != ZLoadBarrierElided) { |
|
129 Label good; |
|
130 __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); |
|
131 __ andr(rscratch1, rscratch1, rscratch2); |
|
132 __ cbz(rscratch1, good); |
|
133 z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */); |
|
134 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, |
|
135 false /* acquire */, true /* release */, false /* weak */, rscratch2); |
|
136 __ cset($res$$Register, Assembler::EQ); |
|
137 __ bind(good); |
|
138 } |
|
139 %} |
|
140 |
|
141 ins_pipe(pipe_slow); |
|
142 %} |
|
143 |
|
144 instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
|
145 match(Set res (CompareAndSwapP mem (Binary oldval newval))); |
|
146 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); |
|
147 predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); |
|
148 effect(KILL cr, TEMP_DEF res); |
|
149 |
|
150 ins_cost(2 * VOLATILE_REF_COST); |
|
151 |
|
152 format %{ "cmpxchg $mem, $oldval, $newval\n\t" |
|
153 "cset $res, EQ" %} |
|
154 |
|
155 ins_encode %{ |
|
156 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); |
|
157 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, |
|
158 true /* acquire */, true /* release */, false /* weak */, rscratch2); |
|
159 __ cset($res$$Register, Assembler::EQ); |
|
160 if (barrier_data() != ZLoadBarrierElided) { |
|
161 Label good; |
|
162 __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); |
|
163 __ andr(rscratch1, rscratch1, rscratch2); |
|
164 __ cbz(rscratch1, good); |
|
165 z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ ); |
|
166 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, |
|
167 true /* acquire */, true /* release */, false /* weak */, rscratch2); |
|
168 __ cset($res$$Register, Assembler::EQ); |
|
169 __ bind(good); |
|
170 } |
|
171 %} |
|
172 |
|
173 ins_pipe(pipe_slow); |
|
174 %} |
|
175 |
|
176 instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
|
177 match(Set res (CompareAndExchangeP mem (Binary oldval newval))); |
|
178 predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); |
132 effect(TEMP_DEF res, KILL cr); |
179 effect(TEMP_DEF res, KILL cr); |
133 format %{ |
180 |
134 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" |
181 ins_cost(2 * VOLATILE_REF_COST); |
135 %} |
182 |
136 ins_encode %{ |
183 format %{ "cmpxchg $res = $mem, $oldval, $newval" %} |
137 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, |
184 |
138 Assembler::xword, /*acquire*/ false, /*release*/ true, |
185 ins_encode %{ |
139 /*weak*/ false, $res$$Register); |
186 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); |
140 %} |
187 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, |
141 ins_pipe(pipe_slow); |
188 false /* acquire */, true /* release */, false /* weak */, $res$$Register); |
142 %} |
189 if (barrier_data() != ZLoadBarrierElided) { |
143 |
190 Label good; |
144 instruct z_compareAndSwapP(iRegINoSp res, |
191 __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); |
145 indirect mem, |
192 __ andr(rscratch1, rscratch1, $res$$Register); |
146 iRegP oldval, iRegP newval, iRegP keepalive, |
193 __ cbz(rscratch1, good); |
147 rFlagsReg cr) %{ |
194 z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); |
148 |
195 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, |
149 match(Set res (ZCompareAndSwapP (Binary mem keepalive) (Binary oldval newval))); |
196 false /* acquire */, true /* release */, false /* weak */, $res$$Register); |
150 match(Set res (ZWeakCompareAndSwapP (Binary mem keepalive) (Binary oldval newval))); |
197 __ bind(good); |
151 |
198 } |
152 ins_cost(2 * VOLATILE_REF_COST); |
199 %} |
153 |
200 |
154 effect(KILL cr); |
201 ins_pipe(pipe_slow); |
155 |
202 %} |
156 format %{ |
203 |
157 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval" |
204 instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
158 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)" |
205 match(Set res (CompareAndExchangeP mem (Binary oldval newval))); |
159 %} |
206 predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); |
160 |
207 effect(TEMP_DEF res, KILL cr); |
161 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval), |
208 |
162 aarch64_enc_cset_eq(res)); |
209 ins_cost(2 * VOLATILE_REF_COST); |
163 |
210 |
164 ins_pipe(pipe_slow); |
211 format %{ "cmpxchg $res = $mem, $oldval, $newval" %} |
165 %} |
212 |
166 |
213 ins_encode %{ |
167 |
214 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding"); |
168 instruct z_get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev, |
215 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, |
169 iRegP keepalive) %{ |
216 true /* acquire */, true /* release */, false /* weak */, $res$$Register); |
170 match(Set prev (ZGetAndSetP mem (Binary newv keepalive))); |
217 if (barrier_data() != ZLoadBarrierElided) { |
171 |
218 Label good; |
172 ins_cost(2 * VOLATILE_REF_COST); |
219 __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset())); |
|
220 __ andr(rscratch1, rscratch1, $res$$Register); |
|
221 __ cbz(rscratch1, good); |
|
222 z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */); |
|
223 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, |
|
224 true /* acquire */, true /* release */, false /* weak */, $res$$Register); |
|
225 __ bind(good); |
|
226 } |
|
227 %} |
|
228 |
|
229 ins_pipe(pipe_slow); |
|
230 %} |
|
231 |
|
232 instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ |
|
233 match(Set prev (GetAndSetP mem newv)); |
|
234 predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong); |
|
235 effect(TEMP_DEF prev, KILL cr); |
|
236 |
|
237 ins_cost(2 * VOLATILE_REF_COST); |
|
238 |
173 format %{ "atomic_xchg $prev, $newv, [$mem]" %} |
239 format %{ "atomic_xchg $prev, $newv, [$mem]" %} |
174 ins_encode %{ |
240 |
175 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base)); |
241 ins_encode %{ |
176 %} |
242 __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register); |
|
243 if (barrier_data() != ZLoadBarrierElided) { |
|
244 z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); |
|
245 } |
|
246 %} |
|
247 |
177 ins_pipe(pipe_serial); |
248 ins_pipe(pipe_serial); |
178 %} |
249 %} |
|
250 |
|
251 instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{ |
|
252 match(Set prev (GetAndSetP mem newv)); |
|
253 predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)); |
|
254 effect(TEMP_DEF prev, KILL cr); |
|
255 |
|
256 ins_cost(VOLATILE_REF_COST); |
|
257 |
|
258 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} |
|
259 |
|
260 ins_encode %{ |
|
261 __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register); |
|
262 if (barrier_data() != ZLoadBarrierElided) { |
|
263 z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, false /* weak */); |
|
264 } |
|
265 %} |
|
266 ins_pipe(pipe_serial); |
|
267 %} |