2438 ATOMIC_XCHG(xchgal, swpal, ldaxr, stlxr, Assembler::xword) |
2438 ATOMIC_XCHG(xchgal, swpal, ldaxr, stlxr, Assembler::xword) |
2439 ATOMIC_XCHG(xchgalw, swpal, ldaxrw, stlxrw, Assembler::word) |
2439 ATOMIC_XCHG(xchgalw, swpal, ldaxrw, stlxrw, Assembler::word) |
2440 |
2440 |
2441 #undef ATOMIC_XCHG |
2441 #undef ATOMIC_XCHG |
2442 |
2442 |
2443 void MacroAssembler::incr_allocated_bytes(Register thread, |
|
2444 Register var_size_in_bytes, |
|
2445 int con_size_in_bytes, |
|
2446 Register t1) { |
|
2447 if (!thread->is_valid()) { |
|
2448 thread = rthread; |
|
2449 } |
|
2450 assert(t1->is_valid(), "need temp reg"); |
|
2451 |
|
2452 ldr(t1, Address(thread, in_bytes(JavaThread::allocated_bytes_offset()))); |
|
2453 if (var_size_in_bytes->is_valid()) { |
|
2454 add(t1, t1, var_size_in_bytes); |
|
2455 } else { |
|
2456 add(t1, t1, con_size_in_bytes); |
|
2457 } |
|
2458 str(t1, Address(thread, in_bytes(JavaThread::allocated_bytes_offset()))); |
|
2459 } |
|
2460 |
|
2461 #ifndef PRODUCT |
2443 #ifndef PRODUCT |
2462 extern "C" void findpc(intptr_t x); |
2444 extern "C" void findpc(intptr_t x); |
2463 #endif |
2445 #endif |
2464 |
2446 |
2465 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) |
2447 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) |
4083 Register var_size_in_bytes, |
4065 Register var_size_in_bytes, |
4084 int con_size_in_bytes, |
4066 int con_size_in_bytes, |
4085 Register t1, |
4067 Register t1, |
4086 Register t2, |
4068 Register t2, |
4087 Label& slow_case) { |
4069 Label& slow_case) { |
4088 assert_different_registers(obj, t2); |
4070 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
4089 assert_different_registers(obj, var_size_in_bytes); |
4071 bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case); |
4090 Register end = t2; |
4072 } |
4091 |
4073 |
4092 // verify_tlab(); |
4074 // Defines obj, preserves var_size_in_bytes |
4093 |
4075 void MacroAssembler::eden_allocate(Register obj, |
4094 ldr(obj, Address(rthread, JavaThread::tlab_top_offset())); |
4076 Register var_size_in_bytes, |
4095 if (var_size_in_bytes == noreg) { |
4077 int con_size_in_bytes, |
4096 lea(end, Address(obj, con_size_in_bytes)); |
4078 Register t1, |
4097 } else { |
4079 Label& slow_case) { |
4098 lea(end, Address(obj, var_size_in_bytes)); |
4080 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); |
4099 } |
4081 bs->eden_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case); |
4100 ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset())); |
|
4101 cmp(end, rscratch1); |
|
4102 br(Assembler::HI, slow_case); |
|
4103 |
|
4104 // update the tlab top pointer |
|
4105 str(end, Address(rthread, JavaThread::tlab_top_offset())); |
|
4106 |
|
4107 // recover var_size_in_bytes if necessary |
|
4108 if (var_size_in_bytes == end) { |
|
4109 sub(var_size_in_bytes, var_size_in_bytes, obj); |
|
4110 } |
|
4111 // verify_tlab(); |
|
4112 } |
4082 } |
4113 |
4083 |
4114 // Zero words; len is in bytes |
4084 // Zero words; len is in bytes |
4115 // Destroys all registers except addr |
4085 // Destroys all registers except addr |
4116 // len must be a nonzero multiple of wordSize |
4086 // len must be a nonzero multiple of wordSize |
4169 for (int i = -unroll; i < 0; i++) |
4139 for (int i = -unroll; i < 0; i++) |
4170 Assembler::str(zr, Address(t1, i * wordSize)); |
4140 Assembler::str(zr, Address(t1, i * wordSize)); |
4171 bind(entry); |
4141 bind(entry); |
4172 add(t1, t1, unroll * wordSize); |
4142 add(t1, t1, unroll * wordSize); |
4173 cbnz(len, loop); |
4143 cbnz(len, loop); |
4174 } |
|
4175 |
|
4176 // Defines obj, preserves var_size_in_bytes |
|
4177 void MacroAssembler::eden_allocate(Register obj, |
|
4178 Register var_size_in_bytes, |
|
4179 int con_size_in_bytes, |
|
4180 Register t1, |
|
4181 Label& slow_case) { |
|
4182 assert_different_registers(obj, var_size_in_bytes, t1); |
|
4183 if (!Universe::heap()->supports_inline_contig_alloc()) { |
|
4184 b(slow_case); |
|
4185 } else { |
|
4186 Register end = t1; |
|
4187 Register heap_end = rscratch2; |
|
4188 Label retry; |
|
4189 bind(retry); |
|
4190 { |
|
4191 unsigned long offset; |
|
4192 adrp(rscratch1, ExternalAddress((address) Universe::heap()->end_addr()), offset); |
|
4193 ldr(heap_end, Address(rscratch1, offset)); |
|
4194 } |
|
4195 |
|
4196 ExternalAddress heap_top((address) Universe::heap()->top_addr()); |
|
4197 |
|
4198 // Get the current top of the heap |
|
4199 { |
|
4200 unsigned long offset; |
|
4201 adrp(rscratch1, heap_top, offset); |
|
4202 // Use add() here after ARDP, rather than lea(). |
|
4203 // lea() does not generate anything if its offset is zero. |
|
4204 // However, relocs expect to find either an ADD or a load/store |
|
4205 // insn after an ADRP. add() always generates an ADD insn, even |
|
4206 // for add(Rn, Rn, 0). |
|
4207 add(rscratch1, rscratch1, offset); |
|
4208 ldaxr(obj, rscratch1); |
|
4209 } |
|
4210 |
|
4211 // Adjust it my the size of our new object |
|
4212 if (var_size_in_bytes == noreg) { |
|
4213 lea(end, Address(obj, con_size_in_bytes)); |
|
4214 } else { |
|
4215 lea(end, Address(obj, var_size_in_bytes)); |
|
4216 } |
|
4217 |
|
4218 // if end < obj then we wrapped around high memory |
|
4219 cmp(end, obj); |
|
4220 br(Assembler::LO, slow_case); |
|
4221 |
|
4222 cmp(end, heap_end); |
|
4223 br(Assembler::HI, slow_case); |
|
4224 |
|
4225 // If heap_top hasn't been changed by some other thread, update it. |
|
4226 stlxr(rscratch2, end, rscratch1); |
|
4227 cbnzw(rscratch2, retry); |
|
4228 } |
|
4229 } |
4144 } |
4230 |
4145 |
4231 void MacroAssembler::verify_tlab() { |
4146 void MacroAssembler::verify_tlab() { |
4232 #ifdef ASSERT |
4147 #ifdef ASSERT |
4233 if (UseTLAB && VerifyOops) { |
4148 if (UseTLAB && VerifyOops) { |