source_hpp %{

#include "gc/z/c2/zBarrierSetC2.hpp"

#include "gc/z/zThreadLocalData.hpp"

%}

static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, bool weak) {

  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, weak);

  __ testptr(ref, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset()));

  __ jcc(Assembler::notZero, *stub->entry());

  __ bind(*stub->continuation());

}

static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {

  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, false /* weak */);

  __ jmp(*stub->entry());

  __ bind(*stub->continuation());

// Load Pointer

instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr)

%{

  predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierStrong);

  match(Set dst (LoadP mem));

  effect(KILL cr, TEMP dst);

  ins_cost(125);

  format %{ "movq     $dst, $mem" %}

    __ movptr($dst$$Register, $mem$$Address);

    if (barrier_data() != ZLoadBarrierElided) {

      z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, false /* weak */);

    }

  ins_pipe(ialu_reg_mem);

// Load Weak Pointer

instruct zLoadWeakP(rRegP dst, memory mem, rFlagsReg cr)

%{

  predicate(UseZGC && n->as_Load()->barrier_data() == ZLoadBarrierWeak);

  match(Set dst (LoadP mem));

  effect(KILL cr, TEMP dst);

  ins_cost(125);

  format %{ "movq     $dst, $mem" %}

    __ movptr($dst$$Register, $mem$$Address);

    z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, true /* weak */);

  ins_pipe(ialu_reg_mem);

instruct zCompareAndExchangeP(memory mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{

  match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));

  predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);

  effect(KILL cr, TEMP tmp);

  format %{ "lock\n\t"

            "cmpxchgq $newval, $mem" %}

    if (barrier_data() != ZLoadBarrierElided) {

      __ movptr($tmp$$Register, $oldval$$Register);

    }

    __ lock();

    __ cmpxchgptr($newval$$Register, $mem$$Address);

    if (barrier_data() != ZLoadBarrierElided) {

      Label good;

      __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset()));

      __ jcc(Assembler::zero, good);

      z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register);

      __ movptr($oldval$$Register, $tmp$$Register);

      __ lock();

      __ cmpxchgptr($newval$$Register, $mem$$Address);

      __ bind(good);

    }

  ins_pipe(pipe_cmpxchg);

instruct zCompareAndSwapP(rRegI res, memory mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{

  match(Set res (CompareAndSwapP mem (Binary oldval newval)));

  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));

  predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);

  effect(KILL cr, KILL oldval, TEMP tmp);

  format %{ "lock\n\t"

            "cmpxchgq $newval, $mem\n\t"

            "sete     $res\n\t"

            "movzbl   $res, $res" %}

    if (barrier_data() != ZLoadBarrierElided) {

      __ movptr($tmp$$Register, $oldval$$Register);

    }

    __ lock();

    __ cmpxchgptr($newval$$Register, $mem$$Address);

    if (barrier_data() != ZLoadBarrierElided) {

      Label good;

      __ testptr($oldval$$Register, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset()));

      __ jcc(Assembler::zero, good);

      z_load_barrier_slow_path(_masm, this, $mem$$Address, $oldval$$Register, $tmp$$Register);

      __ movptr($oldval$$Register, $tmp$$Register);

      __ lock();

      __ cmpxchgptr($newval$$Register, $mem$$Address);

      __ bind(good);

      __ cmpptr($tmp$$Register, $oldval$$Register);

    }

    __ setb(Assembler::equal, $res$$Register);

    __ movzbl($res$$Register, $res$$Register);

  ins_pipe(pipe_cmpxchg);

instruct zXChgP(memory mem, rRegP newval, rFlagsReg cr) %{

  match(Set newval (GetAndSetP mem newval));

  predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);

  effect(KILL cr);

  format %{ "xchgq    $newval, $mem" %}

  ins_encode %{

    __ xchgptr($newval$$Register, $mem$$Address);

    if (barrier_data() != ZLoadBarrierElided) {

      z_load_barrier(_masm, this, Address(noreg, 0), $newval$$Register, noreg /* tmp */, false /* weak */);

    }

  %}

  ins_pipe(pipe_cmpxchg);

%}