22 */ |
22 */ |
23 |
23 |
24 #include "precompiled.hpp" |
24 #include "precompiled.hpp" |
25 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" |
25 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" |
26 #include "gc/shenandoah/shenandoahForwarding.hpp" |
26 #include "gc/shenandoah/shenandoahForwarding.hpp" |
27 #include "gc/shenandoah/shenandoahHeap.hpp" |
27 #include "gc/shenandoah/shenandoahHeap.inline.hpp" |
28 #include "gc/shenandoah/shenandoahHeapRegion.hpp" |
28 #include "gc/shenandoah/shenandoahHeapRegion.hpp" |
29 #include "gc/shenandoah/shenandoahHeuristics.hpp" |
29 #include "gc/shenandoah/shenandoahHeuristics.hpp" |
30 #include "gc/shenandoah/shenandoahRuntime.hpp" |
30 #include "gc/shenandoah/shenandoahRuntime.hpp" |
31 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" |
31 #include "gc/shenandoah/shenandoahThreadLocalData.hpp" |
32 #include "interpreter/interpreter.hpp" |
32 #include "interpreter/interpreter.hpp" |
399 #ifndef _LP64 |
399 #ifndef _LP64 |
400 __ pop(thread); |
400 __ pop(thread); |
401 #endif |
401 #endif |
402 } |
402 } |
403 |
403 |
|
404 void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler* masm, Register dst) { |
|
405 if (!ShenandoahLoadRefBarrier) { |
|
406 return; |
|
407 } |
|
408 |
|
409 Label done; |
|
410 Label not_null; |
|
411 Label slow_path; |
|
412 |
|
413 // null check |
|
414 __ testptr(dst, dst); |
|
415 __ jcc(Assembler::notZero, not_null); |
|
416 __ jmp(done); |
|
417 __ bind(not_null); |
|
418 |
|
419 |
|
420 #ifdef _LP64 |
|
421 Register thread = r15_thread; |
|
422 #else |
|
423 Register thread = rcx; |
|
424 if (thread == dst) { |
|
425 thread = rbx; |
|
426 } |
|
427 __ push(thread); |
|
428 __ get_thread(thread); |
|
429 #endif |
|
430 assert_different_registers(dst, thread); |
|
431 |
|
432 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); |
|
433 __ testb(gc_state, ShenandoahHeap::EVACUATION); |
|
434 #ifndef _LP64 |
|
435 __ pop(thread); |
|
436 #endif |
|
437 __ jccb(Assembler::notZero, slow_path); |
|
438 __ jmp(done); |
|
439 __ bind(slow_path); |
|
440 |
|
441 if (dst != rax) { |
|
442 __ xchgptr(dst, rax); // Move obj into rax and save rax into obj. |
|
443 } |
|
444 __ push(rcx); |
|
445 __ push(rdx); |
|
446 __ push(rdi); |
|
447 __ push(rsi); |
|
448 #ifdef _LP64 |
|
449 __ push(r8); |
|
450 __ push(r9); |
|
451 __ push(r10); |
|
452 __ push(r11); |
|
453 __ push(r12); |
|
454 __ push(r13); |
|
455 __ push(r14); |
|
456 __ push(r15); |
|
457 #endif |
|
458 |
|
459 __ movptr(rdi, rax); |
|
460 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native), rdi); |
|
461 |
|
462 #ifdef _LP64 |
|
463 __ pop(r15); |
|
464 __ pop(r14); |
|
465 __ pop(r13); |
|
466 __ pop(r12); |
|
467 __ pop(r11); |
|
468 __ pop(r10); |
|
469 __ pop(r9); |
|
470 __ pop(r8); |
|
471 #endif |
|
472 __ pop(rsi); |
|
473 __ pop(rdi); |
|
474 __ pop(rdx); |
|
475 __ pop(rcx); |
|
476 |
|
477 if (dst != rax) { |
|
478 __ xchgptr(rax, dst); // Swap back obj with rax. |
|
479 } |
|
480 |
|
481 __ bind(done); |
|
482 } |
|
483 |
404 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { |
484 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) { |
405 if (ShenandoahStoreValEnqueueBarrier) { |
485 if (ShenandoahStoreValEnqueueBarrier) { |
406 storeval_barrier_impl(masm, dst, tmp); |
486 storeval_barrier_impl(masm, dst, tmp); |
407 } |
487 } |
408 } |
488 } |
455 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, |
535 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, |
456 Register dst, Address src, Register tmp1, Register tmp_thread) { |
536 Register dst, Address src, Register tmp1, Register tmp_thread) { |
457 bool on_oop = type == T_OBJECT || type == T_ARRAY; |
537 bool on_oop = type == T_OBJECT || type == T_ARRAY; |
458 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; |
538 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; |
459 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; |
539 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; |
|
540 bool not_in_heap = (decorators & IN_NATIVE) != 0; |
460 bool on_reference = on_weak || on_phantom; |
541 bool on_reference = on_weak || on_phantom; |
461 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); |
542 bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0; |
|
543 |
|
544 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); |
462 if (on_oop) { |
545 if (on_oop) { |
463 load_reference_barrier(masm, dst); |
546 if (not_in_heap) { |
464 |
547 if (ShenandoahHeap::heap()->is_traversal_mode()) { |
465 if (ShenandoahKeepAliveBarrier && on_reference) { |
548 load_reference_barrier(masm, dst); |
|
549 keep_alive = true; |
|
550 } else { |
|
551 load_reference_barrier_native(masm, dst); |
|
552 } |
|
553 } else { |
|
554 load_reference_barrier(masm, dst); |
|
555 } |
|
556 |
|
557 if (ShenandoahKeepAliveBarrier && on_reference && keep_alive) { |
466 const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); |
558 const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread); |
467 assert_different_registers(dst, tmp1, tmp_thread); |
559 assert_different_registers(dst, tmp1, tmp_thread); |
468 NOT_LP64(__ get_thread(thread)); |
560 NOT_LP64(__ get_thread(thread)); |
469 // Generate the SATB pre-barrier code to log the value of |
561 // Generate the SATB pre-barrier code to log the value of |
470 // the referent field in an SATB buffer. |
562 // the referent field in an SATB buffer. |