2511 guarantee(DISP == 0, "mode not permitted for volatile"); \ |
2511 guarantee(DISP == 0, "mode not permitted for volatile"); \ |
2512 guarantee(SCALE == 0, "mode not permitted for volatile"); \ |
2512 guarantee(SCALE == 0, "mode not permitted for volatile"); \ |
2513 __ INSN(REG, as_Register(BASE)); \ |
2513 __ INSN(REG, as_Register(BASE)); \ |
2514 } |
2514 } |
2515 |
2515 |
2516 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr); |
2516 |
2517 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr); |
2517 static Address mem2address(int opcode, Register base, int index, int size, int disp) |
2518 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, |
|
2519 MacroAssembler::SIMD_RegVariant T, const Address &adr); |
|
2520 |
|
2521 // Used for all non-volatile memory accesses. The use of |
|
2522 // $mem->opcode() to discover whether this pattern uses sign-extended |
|
2523 // offsets is something of a kludge. |
|
2524 static void loadStore(MacroAssembler masm, mem_insn insn, |
|
2525 Register reg, int opcode, |
|
2526 Register base, int index, int size, int disp) |
|
2527 { |
2518 { |
2528 Address::extend scale; |
2519 Address::extend scale; |
2529 |
2520 |
2530 // Hooboy, this is fugly. We need a way to communicate to the |
2521 // Hooboy, this is fugly. We need a way to communicate to the |
2531 // encoder that the index needs to be sign extended, so we have to |
2522 // encoder that the index needs to be sign extended, so we have to |
2540 default: |
2531 default: |
2541 scale = Address::lsl(size); |
2532 scale = Address::lsl(size); |
2542 } |
2533 } |
2543 |
2534 |
2544 if (index == -1) { |
2535 if (index == -1) { |
2545 (masm.*insn)(reg, Address(base, disp)); |
2536 return Address(base, disp); |
2546 } else { |
2537 } else { |
2547 assert(disp == 0, "unsupported address mode: disp = %d", disp); |
2538 assert(disp == 0, "unsupported address mode: disp = %d", disp); |
2548 (masm.*insn)(reg, Address(base, as_Register(index), scale)); |
2539 return Address(base, as_Register(index), scale); |
2549 } |
2540 } |
2550 } |
2541 } |
2551 |
2542 |
|
2543 |
|
2544 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr); |
|
2545 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr); |
|
2546 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr); |
|
2547 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt, |
|
2548 MacroAssembler::SIMD_RegVariant T, const Address &adr); |
|
2549 |
|
2550 // Used for all non-volatile memory accesses. The use of |
|
2551 // $mem->opcode() to discover whether this pattern uses sign-extended |
|
2552 // offsets is something of a kludge. |
|
2553 static void loadStore(MacroAssembler masm, mem_insn insn, |
|
2554 Register reg, int opcode, |
|
2555 Register base, int index, int size, int disp) |
|
2556 { |
|
2557 Address addr = mem2address(opcode, base, index, size, disp); |
|
2558 (masm.*insn)(reg, addr); |
|
2559 } |
|
2560 |
2552 static void loadStore(MacroAssembler masm, mem_float_insn insn, |
2561 static void loadStore(MacroAssembler masm, mem_float_insn insn, |
2553 FloatRegister reg, int opcode, |
2562 FloatRegister reg, int opcode, |
2554 Register base, int index, int size, int disp) |
2563 Register base, int index, int size, int disp) |
2555 { |
2564 { |
2556 Address::extend scale; |
2565 Address::extend scale; |
2557 |
2566 |
2558 switch (opcode) { |
2567 switch (opcode) { |
2559 case INDINDEXSCALEDI2L: |
2568 case INDINDEXSCALEDI2L: |
2571 (masm.*insn)(reg, Address(base, as_Register(index), scale)); |
2580 (masm.*insn)(reg, Address(base, as_Register(index), scale)); |
2572 } |
2581 } |
2573 } |
2582 } |
2574 |
2583 |
2575 static void loadStore(MacroAssembler masm, mem_vector_insn insn, |
2584 static void loadStore(MacroAssembler masm, mem_vector_insn insn, |
2576 FloatRegister reg, MacroAssembler::SIMD_RegVariant T, |
2585 FloatRegister reg, MacroAssembler::SIMD_RegVariant T, |
2577 int opcode, Register base, int index, int size, int disp) |
2586 int opcode, Register base, int index, int size, int disp) |
2578 { |
2587 { |
2579 if (index == -1) { |
2588 if (index == -1) { |
2580 (masm.*insn)(reg, T, Address(base, disp)); |
2589 (masm.*insn)(reg, T, Address(base, disp)); |
2581 } else { |
2590 } else { |
2582 assert(disp == 0, "unsupported address mode"); |
2591 assert(disp == 0, "unsupported address mode"); |
3789 }; |
3798 }; |
3790 |
3799 |
3791 static const int hi[Op_RegL + 1] = { // enum name |
3800 static const int hi[Op_RegL + 1] = { // enum name |
3792 0, // Op_Node |
3801 0, // Op_Node |
3793 0, // Op_Set |
3802 0, // Op_Set |
3794 OptoReg::Bad, // Op_RegN |
3803 OptoReg::Bad, // Op_RegN |
3795 OptoReg::Bad, // Op_RegI |
3804 OptoReg::Bad, // Op_RegI |
3796 R0_H_num, // Op_RegP |
3805 R0_H_num, // Op_RegP |
3797 OptoReg::Bad, // Op_RegF |
3806 OptoReg::Bad, // Op_RegF |
3798 V0_H_num, // Op_RegD |
3807 V0_H_num, // Op_RegD |
3799 R0_H_num // Op_RegL |
3808 R0_H_num // Op_RegL |
6921 |
6930 |
6922 // Load Pointer |
6931 // Load Pointer |
6923 instruct loadP(iRegPNoSp dst, memory mem) |
6932 instruct loadP(iRegPNoSp dst, memory mem) |
6924 %{ |
6933 %{ |
6925 match(Set dst (LoadP mem)); |
6934 match(Set dst (LoadP mem)); |
6926 predicate(!needs_acquiring_load(n)); |
6935 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0)); |
6927 |
6936 |
6928 ins_cost(4 * INSN_COST); |
6937 ins_cost(4 * INSN_COST); |
6929 format %{ "ldr $dst, $mem\t# ptr" %} |
6938 format %{ "ldr $dst, $mem\t# ptr" %} |
6930 |
6939 |
6931 ins_encode(aarch64_enc_ldr(dst, mem)); |
6940 ins_encode(aarch64_enc_ldr(dst, mem)); |
7614 |
7623 |
7615 // Load Pointer |
7624 // Load Pointer |
7616 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem) |
7625 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem) |
7617 %{ |
7626 %{ |
7618 match(Set dst (LoadP mem)); |
7627 match(Set dst (LoadP mem)); |
|
7628 predicate(n->as_Load()->barrier_data() == 0); |
7619 |
7629 |
7620 ins_cost(VOLATILE_REF_COST); |
7630 ins_cost(VOLATILE_REF_COST); |
7621 format %{ "ldar $dst, $mem\t# ptr" %} |
7631 format %{ "ldar $dst, $mem\t# ptr" %} |
7622 |
7632 |
7623 ins_encode(aarch64_enc_ldar(dst, mem)); |
7633 ins_encode(aarch64_enc_ldar(dst, mem)); |
8550 %} |
8560 %} |
8551 |
8561 |
8552 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
8562 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
8553 |
8563 |
8554 match(Set res (CompareAndSwapP mem (Binary oldval newval))); |
8564 match(Set res (CompareAndSwapP mem (Binary oldval newval))); |
|
8565 predicate(n->as_LoadStore()->barrier_data() == 0); |
8555 ins_cost(2 * VOLATILE_REF_COST); |
8566 ins_cost(2 * VOLATILE_REF_COST); |
8556 |
8567 |
8557 effect(KILL cr); |
8568 effect(KILL cr); |
8558 |
8569 |
8559 format %{ |
8570 format %{ |
8663 ins_pipe(pipe_slow); |
8674 ins_pipe(pipe_slow); |
8664 %} |
8675 %} |
8665 |
8676 |
8666 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
8677 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
8667 |
8678 |
8668 predicate(needs_acquiring_load_exclusive(n)); |
8679 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); |
8669 match(Set res (CompareAndSwapP mem (Binary oldval newval))); |
8680 match(Set res (CompareAndSwapP mem (Binary oldval newval))); |
8670 ins_cost(VOLATILE_REF_COST); |
8681 ins_cost(VOLATILE_REF_COST); |
8671 |
8682 |
8672 effect(KILL cr); |
8683 effect(KILL cr); |
8673 |
8684 |
8794 %} |
8805 %} |
8795 ins_pipe(pipe_slow); |
8806 ins_pipe(pipe_slow); |
8796 %} |
8807 %} |
8797 |
8808 |
8798 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
8809 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
|
8810 predicate(n->as_LoadStore()->barrier_data() == 0); |
8799 match(Set res (CompareAndExchangeP mem (Binary oldval newval))); |
8811 match(Set res (CompareAndExchangeP mem (Binary oldval newval))); |
8800 ins_cost(2 * VOLATILE_REF_COST); |
8812 ins_cost(2 * VOLATILE_REF_COST); |
8801 effect(TEMP_DEF res, KILL cr); |
8813 effect(TEMP_DEF res, KILL cr); |
8802 format %{ |
8814 format %{ |
8803 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" |
8815 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" |
8893 %} |
8905 %} |
8894 ins_pipe(pipe_slow); |
8906 ins_pipe(pipe_slow); |
8895 %} |
8907 %} |
8896 |
8908 |
8897 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
8909 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
8898 predicate(needs_acquiring_load_exclusive(n)); |
8910 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); |
8899 match(Set res (CompareAndExchangeP mem (Binary oldval newval))); |
8911 match(Set res (CompareAndExchangeP mem (Binary oldval newval))); |
8900 ins_cost(VOLATILE_REF_COST); |
8912 ins_cost(VOLATILE_REF_COST); |
8901 effect(TEMP_DEF res, KILL cr); |
8913 effect(TEMP_DEF res, KILL cr); |
8902 format %{ |
8914 format %{ |
8903 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" |
8915 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" |
8994 %} |
9006 %} |
8995 ins_pipe(pipe_slow); |
9007 ins_pipe(pipe_slow); |
8996 %} |
9008 %} |
8997 |
9009 |
8998 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
9010 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
|
9011 predicate(n->as_LoadStore()->barrier_data() == 0); |
8999 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); |
9012 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); |
9000 ins_cost(2 * VOLATILE_REF_COST); |
9013 ins_cost(2 * VOLATILE_REF_COST); |
9001 effect(KILL cr); |
9014 effect(KILL cr); |
9002 format %{ |
9015 format %{ |
9003 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" |
9016 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" |
9101 %} |
9114 %} |
9102 ins_pipe(pipe_slow); |
9115 ins_pipe(pipe_slow); |
9103 %} |
9116 %} |
9104 |
9117 |
9105 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
9118 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{ |
9106 predicate(needs_acquiring_load_exclusive(n)); |
|
9107 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); |
9119 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); |
|
9120 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); |
9108 ins_cost(VOLATILE_REF_COST); |
9121 ins_cost(VOLATILE_REF_COST); |
9109 effect(KILL cr); |
9122 effect(KILL cr); |
9110 format %{ |
9123 format %{ |
9111 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" |
9124 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval" |
9112 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" |
9125 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)" |
9152 %} |
9165 %} |
9153 ins_pipe(pipe_serial); |
9166 ins_pipe(pipe_serial); |
9154 %} |
9167 %} |
9155 |
9168 |
9156 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{ |
9169 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{ |
|
9170 predicate(n->as_LoadStore()->barrier_data() == 0); |
9157 match(Set prev (GetAndSetP mem newv)); |
9171 match(Set prev (GetAndSetP mem newv)); |
9158 ins_cost(2 * VOLATILE_REF_COST); |
9172 ins_cost(2 * VOLATILE_REF_COST); |
9159 format %{ "atomic_xchg $prev, $newv, [$mem]" %} |
9173 format %{ "atomic_xchg $prev, $newv, [$mem]" %} |
9160 ins_encode %{ |
9174 ins_encode %{ |
9161 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base)); |
9175 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base)); |
9195 %} |
9209 %} |
9196 ins_pipe(pipe_serial); |
9210 ins_pipe(pipe_serial); |
9197 %} |
9211 %} |
9198 |
9212 |
9199 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{ |
9213 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{ |
9200 predicate(needs_acquiring_load_exclusive(n)); |
9214 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0)); |
9201 match(Set prev (GetAndSetP mem newv)); |
9215 match(Set prev (GetAndSetP mem newv)); |
9202 ins_cost(VOLATILE_REF_COST); |
9216 ins_cost(VOLATILE_REF_COST); |
9203 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} |
9217 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %} |
9204 ins_encode %{ |
9218 ins_encode %{ |
9205 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base)); |
9219 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base)); |