45 address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL; |
45 address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL; |
46 |
46 |
47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, |
47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, |
48 Register src, Register dst, Register count) { |
48 Register src, Register dst, Register count) { |
49 |
49 |
50 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; |
|
51 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; |
|
52 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); |
|
53 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; |
50 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; |
54 |
51 |
55 if (type == T_OBJECT || type == T_ARRAY) { |
52 if (type == T_OBJECT || type == T_ARRAY) { |
56 #ifdef _LP64 |
53 |
57 if (!checkcast) { |
54 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) { |
58 if (!obj_int) { |
55 #ifdef _LP64 |
59 // Save count for barrier |
56 Register thread = r15_thread; |
60 __ movptr(r11, count); |
57 #else |
61 } else if (disjoint) { |
58 Register thread = rax; |
62 // Save dst in r11 in the disjoint case |
59 if (thread == src || thread == dst || thread == count) { |
63 __ movq(r11, dst); |
60 thread = rbx; |
64 } |
61 } |
65 } |
62 if (thread == src || thread == dst || thread == count) { |
66 #else |
63 thread = rcx; |
67 if (disjoint) { |
64 } |
68 __ mov(rdx, dst); // save 'to' |
65 if (thread == src || thread == dst || thread == count) { |
69 } |
66 thread = rdx; |
70 #endif |
67 } |
71 |
|
72 if (ShenandoahSATBBarrier && !dest_uninitialized) { |
|
73 Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread); |
|
74 assert_different_registers(dst, count, thread); // we don't care about src here? |
|
75 #ifndef _LP64 |
|
76 __ push(thread); |
68 __ push(thread); |
77 __ get_thread(thread); |
69 __ get_thread(thread); |
78 #endif |
70 #endif |
|
71 assert_different_registers(src, dst, count, thread); |
79 |
72 |
80 Label done; |
73 Label done; |
81 // Short-circuit if count == 0. |
74 // Short-circuit if count == 0. |
82 __ testptr(count, count); |
75 __ testptr(count, count); |
83 __ jcc(Assembler::zero, done); |
76 __ jcc(Assembler::zero, done); |
84 |
77 |
85 // Avoid runtime call when not marking. |
78 // Avoid runtime call when not marking. |
86 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); |
79 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); |
87 __ testb(gc_state, ShenandoahHeap::MARKING); |
80 int flags = ShenandoahHeap::HAS_FORWARDED; |
|
81 if (!dest_uninitialized) { |
|
82 flags |= ShenandoahHeap::MARKING; |
|
83 } |
|
84 __ testb(gc_state, flags); |
88 __ jcc(Assembler::zero, done); |
85 __ jcc(Assembler::zero, done); |
89 |
86 |
90 __ pusha(); // push registers |
87 __ pusha(); // push registers |
91 #ifdef _LP64 |
88 #ifdef _LP64 |
92 if (count == c_rarg0) { |
89 assert(src == rdi, "expected"); |
93 if (dst == c_rarg1) { |
90 assert(dst == rsi, "expected"); |
94 // exactly backwards!! |
91 assert(count == rdx, "expected"); |
95 __ xchgptr(c_rarg1, c_rarg0); |
92 if (UseCompressedOops) { |
|
93 if (dest_uninitialized) { |
|
94 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count); |
96 } else { |
95 } else { |
97 __ movptr(c_rarg1, count); |
96 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count); |
98 __ movptr(c_rarg0, dst); |
|
99 } |
97 } |
100 } else { |
98 } else |
101 __ movptr(c_rarg0, dst); |
99 #endif |
102 __ movptr(c_rarg1, count); |
100 { |
|
101 if (dest_uninitialized) { |
|
102 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count); |
|
103 } else { |
|
104 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count); |
|
105 } |
103 } |
106 } |
104 if (UseCompressedOops) { |
|
105 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2); |
|
106 } else { |
|
107 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2); |
|
108 } |
|
109 #else |
|
110 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), |
|
111 dst, count); |
|
112 #endif |
|
113 __ popa(); |
107 __ popa(); |
114 __ bind(done); |
108 __ bind(done); |
115 NOT_LP64(__ pop(thread);) |
109 NOT_LP64(__ pop(thread);) |
116 } |
110 } |
117 } |
111 } |
118 |
112 |
119 } |
|
120 |
|
121 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, |
|
122 Register src, Register dst, Register count) { |
|
123 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; |
|
124 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; |
|
125 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); |
|
126 Register tmp = rax; |
|
127 |
|
128 if (type == T_OBJECT || type == T_ARRAY) { |
|
129 #ifdef _LP64 |
|
130 if (!checkcast) { |
|
131 if (!obj_int) { |
|
132 // Save count for barrier |
|
133 count = r11; |
|
134 } else if (disjoint && obj_int) { |
|
135 // Use the saved dst in the disjoint case |
|
136 dst = r11; |
|
137 } |
|
138 } else { |
|
139 tmp = rscratch1; |
|
140 } |
|
141 #else |
|
142 if (disjoint) { |
|
143 __ mov(dst, rdx); // restore 'to' |
|
144 } |
|
145 #endif |
|
146 |
|
147 Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread); |
|
148 assert_different_registers(dst, thread); // do we care about src at all here? |
|
149 |
|
150 #ifndef _LP64 |
|
151 __ push(thread); |
|
152 __ get_thread(thread); |
|
153 #endif |
|
154 |
|
155 // Short-circuit if count == 0. |
|
156 Label done; |
|
157 __ testptr(count, count); |
|
158 __ jcc(Assembler::zero, done); |
|
159 |
|
160 // Skip runtime call if no forwarded objects. |
|
161 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); |
|
162 __ testb(gc_state, ShenandoahHeap::UPDATEREFS); |
|
163 __ jcc(Assembler::zero, done); |
|
164 |
|
165 __ pusha(); // push registers (overkill) |
|
166 #ifdef _LP64 |
|
167 if (c_rarg0 == count) { // On win64 c_rarg0 == rcx |
|
168 assert_different_registers(c_rarg1, dst); |
|
169 __ mov(c_rarg1, count); |
|
170 __ mov(c_rarg0, dst); |
|
171 } else { |
|
172 assert_different_registers(c_rarg0, count); |
|
173 __ mov(c_rarg0, dst); |
|
174 __ mov(c_rarg1, count); |
|
175 } |
|
176 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2); |
|
177 #else |
|
178 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), |
|
179 dst, count); |
|
180 #endif |
|
181 __ popa(); |
|
182 |
|
183 __ bind(done); |
|
184 NOT_LP64(__ pop(thread);) |
|
185 } |
|
186 } |
113 } |
187 |
114 |
188 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm, |
115 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm, |
189 Register obj, |
116 Register obj, |
190 Register pre_val, |
117 Register pre_val, |