342 __ ret(0); |
342 __ ret(0); |
343 } |
343 } |
344 |
344 |
345 #endif // COMPILER1 |
345 #endif // COMPILER1 |
346 |
346 |
|
347 #ifdef COMPILER2 |
|
348 |
|
349 OptoReg::Name ZBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) { |
|
350 if (!OptoReg::is_reg(opto_reg)) { |
|
351 return OptoReg::Bad; |
|
352 } |
|
353 |
|
354 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg); |
|
355 if (vm_reg->is_XMMRegister()) { |
|
356 opto_reg &= ~15; |
|
357 switch (node->ideal_reg()) { |
|
358 case Op_VecX: |
|
359 opto_reg |= 2; |
|
360 break; |
|
361 case Op_VecY: |
|
362 opto_reg |= 4; |
|
363 break; |
|
364 case Op_VecZ: |
|
365 opto_reg |= 8; |
|
366 break; |
|
367 default: |
|
368 opto_reg |= 1; |
|
369 break; |
|
370 } |
|
371 } |
|
372 |
|
373 return opto_reg; |
|
374 } |
|
375 |
|
376 // We use the vec_spill_helper from the x86.ad file to avoid reinventing this wheel |
|
377 extern int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load, |
|
378 int stack_offset, int reg, uint ireg, outputStream* st); |
|
379 |
347 #undef __ |
380 #undef __ |
348 #define __ cgen->assembler()-> |
381 #define __ _masm-> |
349 |
382 |
350 // Generates a register specific stub for calling |
383 class ZSaveLiveRegisters { |
351 // ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or |
384 private: |
352 // ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). |
385 struct XMMRegisterData { |
353 // |
386 XMMRegister _reg; |
354 // The raddr register serves as both input and output for this stub. When the stub is |
387 int _size; |
355 // called the raddr register contains the object field address (oop*) where the bad oop |
388 |
356 // was loaded from, which caused the slow path to be taken. On return from the stub the |
389 // Used by GrowableArray::find() |
357 // raddr register contains the good/healed oop returned from |
390 bool operator == (const XMMRegisterData& other) { |
358 // ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded() or |
391 return _reg == other._reg; |
359 // ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(). |
392 } |
360 static address generate_load_barrier_stub(StubCodeGenerator* cgen, Register raddr, DecoratorSet decorators) { |
393 }; |
361 // Don't generate stub for invalid registers |
394 |
362 if (raddr == rsp || raddr == r15) { |
395 MacroAssembler* const _masm; |
363 return NULL; |
396 GrowableArray<Register> _gp_registers; |
364 } |
397 GrowableArray<XMMRegisterData> _xmm_registers; |
365 |
398 int _spill_size; |
366 // Create stub name |
399 int _spill_offset; |
367 char name[64]; |
400 |
368 const bool weak = (decorators & ON_WEAK_OOP_REF) != 0; |
401 static int xmm_compare_register_size(XMMRegisterData* left, XMMRegisterData* right) { |
369 os::snprintf(name, sizeof(name), "zgc_load_barrier%s_stub_%s", weak ? "_weak" : "", raddr->name()); |
402 if (left->_size == right->_size) { |
370 |
403 return 0; |
371 __ align(CodeEntryAlignment); |
404 } |
372 StubCodeMark mark(cgen, "StubRoutines", os::strdup(name, mtCode)); |
405 |
373 address start = __ pc(); |
406 return (left->_size < right->_size) ? -1 : 1; |
374 |
407 } |
375 // Save live registers |
408 |
376 if (raddr != rax) { |
409 static int xmm_slot_size(OptoReg::Name opto_reg) { |
377 __ push(rax); |
410 // The low order 4 bytes denote what size of the XMM register is live |
378 } |
411 return (opto_reg & 15) << 3; |
379 if (raddr != rcx) { |
412 } |
380 __ push(rcx); |
413 |
381 } |
414 static uint xmm_ideal_reg_for_size(int reg_size) { |
382 if (raddr != rdx) { |
415 switch (reg_size) { |
383 __ push(rdx); |
416 case 8: |
384 } |
417 return Op_VecD; |
385 if (raddr != rsi) { |
418 case 16: |
386 __ push(rsi); |
419 return Op_VecX; |
387 } |
420 case 32: |
388 if (raddr != rdi) { |
421 return Op_VecY; |
389 __ push(rdi); |
422 case 64: |
390 } |
423 return Op_VecZ; |
391 if (raddr != r8) { |
424 default: |
392 __ push(r8); |
425 fatal("Invalid register size %d", reg_size); |
393 } |
426 return 0; |
394 if (raddr != r9) { |
427 } |
395 __ push(r9); |
428 } |
396 } |
429 |
397 if (raddr != r10) { |
430 bool xmm_needs_vzeroupper() const { |
398 __ push(r10); |
431 return _xmm_registers.is_nonempty() && _xmm_registers.at(0)._size > 16; |
399 } |
432 } |
400 if (raddr != r11) { |
433 |
401 __ push(r11); |
434 void xmm_register_save(const XMMRegisterData& reg_data) { |
402 } |
435 const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg()); |
403 |
436 const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size); |
404 // Setup arguments |
437 _spill_offset -= reg_data._size; |
405 if (raddr != c_rarg1) { |
438 vec_spill_helper(__ code(), false /* do_size */, false /* is_load */, _spill_offset, opto_reg, ideal_reg, tty); |
406 __ movq(c_rarg1, raddr); |
439 } |
407 } |
440 |
408 __ movq(c_rarg0, Address(raddr, 0)); |
441 void xmm_register_restore(const XMMRegisterData& reg_data) { |
409 |
442 const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg()); |
410 // Call barrier function |
443 const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size); |
411 __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1); |
444 vec_spill_helper(__ code(), false /* do_size */, true /* is_load */, _spill_offset, opto_reg, ideal_reg, tty); |
412 |
445 _spill_offset += reg_data._size; |
413 // Move result returned in rax to raddr, if needed |
446 } |
414 if (raddr != rax) { |
447 |
415 __ movq(raddr, rax); |
448 void gp_register_save(Register reg) { |
416 } |
449 _spill_offset -= 8; |
417 |
450 __ movq(Address(rsp, _spill_offset), reg); |
418 // Restore saved registers |
451 } |
419 if (raddr != r11) { |
452 |
420 __ pop(r11); |
453 void gp_register_restore(Register reg) { |
421 } |
454 __ movq(reg, Address(rsp, _spill_offset)); |
422 if (raddr != r10) { |
455 _spill_offset += 8; |
423 __ pop(r10); |
456 } |
424 } |
457 |
425 if (raddr != r9) { |
458 void initialize(ZLoadBarrierStubC2* stub) { |
426 __ pop(r9); |
459 // Create mask of caller saved registers that need to |
427 } |
460 // be saved/restored if live |
428 if (raddr != r8) { |
461 RegMask caller_saved; |
429 __ pop(r8); |
462 caller_saved.Insert(OptoReg::as_OptoReg(rax->as_VMReg())); |
430 } |
463 caller_saved.Insert(OptoReg::as_OptoReg(rcx->as_VMReg())); |
431 if (raddr != rdi) { |
464 caller_saved.Insert(OptoReg::as_OptoReg(rdx->as_VMReg())); |
432 __ pop(rdi); |
465 caller_saved.Insert(OptoReg::as_OptoReg(rsi->as_VMReg())); |
433 } |
466 caller_saved.Insert(OptoReg::as_OptoReg(rdi->as_VMReg())); |
434 if (raddr != rsi) { |
467 caller_saved.Insert(OptoReg::as_OptoReg(r8->as_VMReg())); |
435 __ pop(rsi); |
468 caller_saved.Insert(OptoReg::as_OptoReg(r9->as_VMReg())); |
436 } |
469 caller_saved.Insert(OptoReg::as_OptoReg(r10->as_VMReg())); |
437 if (raddr != rdx) { |
470 caller_saved.Insert(OptoReg::as_OptoReg(r11->as_VMReg())); |
438 __ pop(rdx); |
471 caller_saved.Remove(OptoReg::as_OptoReg(stub->ref()->as_VMReg())); |
439 } |
472 |
440 if (raddr != rcx) { |
473 // Create mask of live registers |
441 __ pop(rcx); |
474 RegMask live = stub->live(); |
442 } |
475 if (stub->tmp() != noreg) { |
443 if (raddr != rax) { |
476 live.Insert(OptoReg::as_OptoReg(stub->tmp()->as_VMReg())); |
444 __ pop(rax); |
477 } |
445 } |
478 |
446 |
479 int gp_spill_size = 0; |
447 __ ret(0); |
480 int xmm_spill_size = 0; |
448 |
481 |
449 return start; |
482 // Record registers that needs to be saved/restored |
450 } |
483 while (live.is_NotEmpty()) { |
|
484 const OptoReg::Name opto_reg = live.find_first_elem(); |
|
485 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg); |
|
486 |
|
487 live.Remove(opto_reg); |
|
488 |
|
489 if (vm_reg->is_Register()) { |
|
490 if (caller_saved.Member(opto_reg)) { |
|
491 _gp_registers.append(vm_reg->as_Register()); |
|
492 gp_spill_size += 8; |
|
493 } |
|
494 } else if (vm_reg->is_XMMRegister()) { |
|
495 // We encode in the low order 4 bits of the opto_reg, how large part of the register is live |
|
496 const VMReg vm_reg_base = OptoReg::as_VMReg(opto_reg & ~15); |
|
497 const int reg_size = xmm_slot_size(opto_reg); |
|
498 const XMMRegisterData reg_data = { vm_reg_base->as_XMMRegister(), reg_size }; |
|
499 const int reg_index = _xmm_registers.find(reg_data); |
|
500 if (reg_index == -1) { |
|
501 // Not previously appended |
|
502 _xmm_registers.append(reg_data); |
|
503 xmm_spill_size += reg_size; |
|
504 } else { |
|
505 // Previously appended, update size |
|
506 const int reg_size_prev = _xmm_registers.at(reg_index)._size; |
|
507 if (reg_size > reg_size_prev) { |
|
508 _xmm_registers.at_put(reg_index, reg_data); |
|
509 xmm_spill_size += reg_size - reg_size_prev; |
|
510 } |
|
511 } |
|
512 } else { |
|
513 fatal("Unexpected register type"); |
|
514 } |
|
515 } |
|
516 |
|
517 // Sort by size, largest first |
|
518 _xmm_registers.sort(xmm_compare_register_size); |
|
519 |
|
520 // Stack pointer must be 16 bytes aligned for the call |
|
521 _spill_offset = _spill_size = align_up(xmm_spill_size + gp_spill_size, 16); |
|
522 } |
|
523 |
|
524 public: |
|
525 ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) : |
|
526 _masm(masm), |
|
527 _gp_registers(), |
|
528 _xmm_registers(), |
|
529 _spill_size(0), |
|
530 _spill_offset(0) { |
|
531 |
|
532 // |
|
533 // Stack layout after registers have been spilled: |
|
534 // |
|
535 // | ... | original rsp, 16 bytes aligned |
|
536 // ------------------ |
|
537 // | zmm0 high | |
|
538 // | ... | |
|
539 // | zmm0 low | 16 bytes aligned |
|
540 // | ... | |
|
541 // | ymm1 high | |
|
542 // | ... | |
|
543 // | ymm1 low | 16 bytes aligned |
|
544 // | ... | |
|
545 // | xmmN high | |
|
546 // | ... | |
|
547 // | xmmN low | 8 bytes aligned |
|
548 // | reg0 | 8 bytes aligned |
|
549 // | reg1 | |
|
550 // | ... | |
|
551 // | regN | new rsp, if 16 bytes aligned |
|
552 // | <padding> | else new rsp, 16 bytes aligned |
|
553 // ------------------ |
|
554 // |
|
555 |
|
556 // Figure out what registers to save/restore |
|
557 initialize(stub); |
|
558 |
|
559 // Allocate stack space |
|
560 if (_spill_size > 0) { |
|
561 __ subptr(rsp, _spill_size); |
|
562 } |
|
563 |
|
564 // Save XMM/YMM/ZMM registers |
|
565 for (int i = 0; i < _xmm_registers.length(); i++) { |
|
566 xmm_register_save(_xmm_registers.at(i)); |
|
567 } |
|
568 |
|
569 if (xmm_needs_vzeroupper()) { |
|
570 __ vzeroupper(); |
|
571 } |
|
572 |
|
573 // Save general purpose registers |
|
574 for (int i = 0; i < _gp_registers.length(); i++) { |
|
575 gp_register_save(_gp_registers.at(i)); |
|
576 } |
|
577 } |
|
578 |
|
579 ~ZSaveLiveRegisters() { |
|
580 // Restore general purpose registers |
|
581 for (int i = _gp_registers.length() - 1; i >= 0; i--) { |
|
582 gp_register_restore(_gp_registers.at(i)); |
|
583 } |
|
584 |
|
585 __ vzeroupper(); |
|
586 |
|
587 // Restore XMM/YMM/ZMM registers |
|
588 for (int i = _xmm_registers.length() - 1; i >= 0; i--) { |
|
589 xmm_register_restore(_xmm_registers.at(i)); |
|
590 } |
|
591 |
|
592 // Free stack space |
|
593 if (_spill_size > 0) { |
|
594 __ addptr(rsp, _spill_size); |
|
595 } |
|
596 } |
|
597 }; |
|
598 |
|
599 class ZSetupArguments { |
|
600 private: |
|
601 MacroAssembler* const _masm; |
|
602 const Register _ref; |
|
603 const Address _ref_addr; |
|
604 |
|
605 public: |
|
606 ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub) : |
|
607 _masm(masm), |
|
608 _ref(stub->ref()), |
|
609 _ref_addr(stub->ref_addr()) { |
|
610 |
|
611 // Setup arguments |
|
612 if (_ref_addr.base() == noreg) { |
|
613 // No self healing |
|
614 if (_ref != c_rarg0) { |
|
615 __ movq(c_rarg0, _ref); |
|
616 } |
|
617 __ xorq(c_rarg1, c_rarg1); |
|
618 } else { |
|
619 // Self healing |
|
620 if (_ref == c_rarg0) { |
|
621 __ lea(c_rarg1, _ref_addr); |
|
622 } else if (_ref != c_rarg1) { |
|
623 __ lea(c_rarg1, _ref_addr); |
|
624 __ movq(c_rarg0, _ref); |
|
625 } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) { |
|
626 __ movq(c_rarg0, _ref); |
|
627 __ lea(c_rarg1, _ref_addr); |
|
628 } else { |
|
629 __ xchgq(c_rarg0, c_rarg1); |
|
630 if (_ref_addr.base() == c_rarg0) { |
|
631 __ lea(c_rarg1, Address(c_rarg1, _ref_addr.index(), _ref_addr.scale(), _ref_addr.disp())); |
|
632 } else if (_ref_addr.index() == c_rarg0) { |
|
633 __ lea(c_rarg1, Address(_ref_addr.base(), c_rarg1, _ref_addr.scale(), _ref_addr.disp())); |
|
634 } else { |
|
635 ShouldNotReachHere(); |
|
636 } |
|
637 } |
|
638 } |
|
639 } |
|
640 |
|
641 ~ZSetupArguments() { |
|
642 // Transfer result |
|
643 if (_ref != rax) { |
|
644 __ movq(_ref, rax); |
|
645 } |
|
646 } |
|
647 }; |
451 |
648 |
452 #undef __ |
649 #undef __ |
453 |
650 #define __ masm-> |
454 static void barrier_stubs_init_inner(const char* label, const DecoratorSet decorators, address* stub) { |
651 |
455 const int nregs = RegisterImpl::number_of_registers; |
652 void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const { |
456 const int code_size = nregs * 128; // Rough estimate of code size |
653 BLOCK_COMMENT("ZLoadBarrierStubC2"); |
457 |
654 |
458 ResourceMark rm; |
655 // Stub entry |
459 |
656 __ bind(*stub->entry()); |
460 CodeBuffer buf(BufferBlob::create(label, code_size)); |
657 |
461 StubCodeGenerator cgen(&buf); |
658 { |
462 |
659 ZSaveLiveRegisters save_live_registers(masm, stub); |
463 for (int i = 0; i < nregs; i++) { |
660 ZSetupArguments setup_arguments(masm, stub); |
464 const Register reg = as_Register(i); |
661 __ call(RuntimeAddress(stub->slow_path())); |
465 stub[i] = generate_load_barrier_stub(&cgen, reg, decorators); |
662 } |
466 } |
663 |
467 } |
664 // Stub exit |
468 |
665 __ jmp(*stub->continuation()); |
469 void ZBarrierSetAssembler::barrier_stubs_init() { |
666 } |
470 barrier_stubs_init_inner("zgc_load_barrier_stubs", ON_STRONG_OOP_REF, _load_barrier_slow_stub); |
667 |
471 barrier_stubs_init_inner("zgc_load_barrier_weak_stubs", ON_WEAK_OOP_REF, _load_barrier_weak_slow_stub); |
668 #undef __ |
472 } |
669 |
473 |
670 #endif // COMPILER2 |
474 address ZBarrierSetAssembler::load_barrier_slow_stub(Register reg) { |
|
475 return _load_barrier_slow_stub[reg->encoding()]; |
|
476 } |
|
477 |
|
478 address ZBarrierSetAssembler::load_barrier_weak_slow_stub(Register reg) { |
|
479 return _load_barrier_weak_slow_stub[reg->encoding()]; |
|
480 } |
|