/*

 * Copyright (c) 2015, 2019, Red Hat, Inc. All rights reserved.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "gc/shenandoah/c2/shenandoahSupport.hpp"

#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"

#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"

#include "gc/shenandoah/shenandoahBrooksPointer.hpp"

#include "gc/shenandoah/shenandoahHeap.hpp"

#include "gc/shenandoah/shenandoahHeapRegion.hpp"

#include "gc/shenandoah/shenandoahRuntime.hpp"

#include "gc/shenandoah/shenandoahThreadLocalData.hpp"

#include "opto/arraycopynode.hpp"

#include "opto/block.hpp"

#include "opto/callnode.hpp"

#include "opto/castnode.hpp"

#include "opto/movenode.hpp"

#include "opto/phaseX.hpp"

#include "opto/rootnode.hpp"

#include "opto/runtime.hpp"

#include "opto/subnode.hpp"

        return false;

      }

      C->clear_major_progress();

    }

  }

  return true;

}

  if (!UseShenandoahGC) {

    return false;

  }

  assert(iff->is_If(), "bad input");

  if (iff->Opcode() != Op_If) {

    return false;

  }

  Node* bol = iff->in(1);

  if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) {

    return false;

  }

  Node* cmp = bol->in(1);

  if (cmp->Opcode() != Op_CmpI) {

    return false;

  }

  Node* in1 = cmp->in(1);

  Node* in2 = cmp->in(2);

  if (in2->find_int_con(-1) != 0) {

    return false;

  }

  if (in1->Opcode() != Op_AndI) {

    return false;

  }

  in2 = in1->in(2);

  if (in2->find_int_con(-1) != mask) {

    return false;

  }

  in1 = in1->in(1);

  return is_gc_state_load(in1);

}

  return is_heap_state_test(iff, ShenandoahHeap::HAS_FORWARDED);

}

  if (!UseShenandoahGC) {

    return false;

  }

  if (n->Opcode() != Op_LoadB && n->Opcode() != Op_LoadUB) {

    return false;

  }

  Node* addp = n->in(MemNode::Address);

  if (!addp->is_AddP()) {

    return false;

  }

  Node* base = addp->in(AddPNode::Address);

  Node* off = addp->in(AddPNode::Offset);

  if (base->Opcode() != Op_ThreadLocal) {

    return false;

  }

  if (off->find_intptr_t_con(-1) != in_bytes(ShenandoahThreadLocalData::gc_state_offset())) {

    return false;

  }

  return true;

}

  assert(phase->is_dominator(stop, start), "bad inputs");

  ResourceMark rm;

  Unique_Node_List wq;

  wq.push(start);

  for (uint next = 0; next < wq.size(); next++) {

    Node *m = wq.at(next);

    if (m == stop) {

      continue;

    }

    if (m->is_SafePoint() && !m->is_CallLeaf()) {

      return true;

    }

    if (m->is_Region()) {

      for (uint i = 1; i < m->req(); i++) {

        wq.push(m->in(i));

      }

    } else {

      wq.push(m->in(0));

    }

  }

  return false;

}

  assert(is_gc_state_load(n), "inconsistent");

  Node* addp = n->in(MemNode::Address);

  Node* dominator = NULL;

  for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {

    Node* u = addp->fast_out(i);

    assert(is_gc_state_load(u), "inconsistent");

    if (u != n && phase->is_dominator(u->in(0), n->in(0))) {

      if (dominator == NULL) {

        dominator = u;

      } else {

        if (phase->dom_depth(u->in(0)) < phase->dom_depth(dominator->in(0))) {

          dominator = u;

        }

      }

    }

  }

  if (dominator == NULL || has_safepoint_between(n->in(0), dominator->in(0), phase)) {

    return false;

  }

  phase->igvn().replace_node(n, dominator);

  return true;

}

#ifdef ASSERT

  assert(phis.size() == 0, "");

  while (true) {

    if (in->bottom_type() == TypePtr::NULL_PTR) {

      if (trace) {tty->print_cr("NULL");}

    } else if (!in->bottom_type()->make_ptr()->make_oopptr()) {

      if (trace) {tty->print_cr("Non oop");}

    } else if (t == ShenandoahLoad && ShenandoahOptimizeStableFinals &&

               in->bottom_type()->make_ptr()->isa_aryptr() &&

               in->bottom_type()->make_ptr()->is_aryptr()->is_stable()) {

      if (trace) {tty->print_cr("Stable array load");}

    } else {

      if (in->is_ConstraintCast()) {

        in = in->in(1);

        continue;

      } else if (in->is_AddP()) {

        assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access");

        in = in->in(AddPNode::Address);

        continue;

      } else if (in->is_Con()) {

        if (t == ShenandoahOopStore) {

          uint i = 0;

          for (; i < phis.size(); i++) {

            Node* n = phis.node_at(i);

            if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {

              break;

            }

          }

          if (i == phis.size()) {

            return false;

          }

        }

        barriers_used.push(in);

        if (trace) {tty->print("Found barrier"); in->dump();}

      } else if (in->Opcode() == Op_ShenandoahEnqueueBarrier) {

        if (t != ShenandoahOopStore) {

          in = in->in(1);

          continue;

        }

        if (trace) {tty->print("Found enqueue barrier"); in->dump();}

        phis.push(in, in->req());

        in = in->in(1);

        continue;

      } else if (in->is_Proj() && in->in(0)->is_Allocate()) {

      } else if (in->is_Phi()) {

        if (!visited.test_set(in->_idx)) {

          if (trace) {tty->print("Pushed phi:"); in->dump();}

          phis.push(in, 2);

          in = in->in(1);

          continue;

        }

        if (trace) {tty->print("Already seen phi:"); in->dump();}

      } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) {

        if (!visited.test_set(in->_idx)) {

          if (trace) {tty->print("Pushed cmovep:"); in->dump();}

          phis.push(in, CMoveNode::IfTrue);

          in = in->in(CMoveNode::IfFalse);

          continue;

        }

        if (trace) {tty->print("Already seen cmovep:"); in->dump();}

      } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) {

        in = in->in(1);

        continue;

      } else {

        return false;

      }

    }

    bool cont = false;

    while (phis.is_nonempty()) {

      uint idx = phis.index();

      Node* phi = phis.node();

      if (idx >= phi->req()) {

        if (trace) {tty->print("Popped phi:"); phi->dump();}

        phis.pop();

        continue;

      }

      if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();}

      in = phi->in(idx);

      phis.set_index(idx+1);

      cont = true;

      break;

    }

    if (!cont) {

      break;

    }

  }

  return true;

}

  if (n1 != NULL) {

    n1->dump(+10);

  }

  if (n2 != NULL) {

    n2->dump(+10);

  }

  fatal("%s", msg);

}

  ResourceMark rm;

  Unique_Node_List wq;

  GrowableArray<Node*> barriers;

  Unique_Node_List barriers_used;

  Node_Stack phis(0);

  VectorSet visited(Thread::current()->resource_area());

  const bool trace = false;

  const bool verify_no_useless_barrier = false;

  wq.push(root);

  for (uint next = 0; next < wq.size(); next++) {

    Node *n = wq.at(next);

    if (n->is_Load()) {

      const bool trace = false;

      if (trace) {tty->print("Verifying"); n->dump();}

      if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) {

        if (trace) {tty->print_cr("Load range/klass");}

      } else {

        const TypePtr* adr_type = n->as_Load()->adr_type();

        if (adr_type->isa_oopptr() && adr_type->is_oopptr()->offset() == oopDesc::mark_offset_in_bytes()) {

          if (trace) {tty->print_cr("Mark load");}

        } else if (adr_type->isa_instptr() &&

                   adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) &&

                   adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) {

          if (trace) {tty->print_cr("Reference.get()");}

        } else {

          bool verify = true;

          if (adr_type->isa_instptr()) {

            const TypeInstPtr* tinst = adr_type->is_instptr();

            ciKlass* k = tinst->klass();

            assert(k->is_instance_klass(), "");

            ciInstanceKlass* ik = (ciInstanceKlass*)k;

            int offset = adr_type->offset();

            if ((ik->debug_final_field_at(offset) && ShenandoahOptimizeInstanceFinals) ||

                (ik->debug_stable_field_at(offset) && ShenandoahOptimizeStableFinals)) {

              if (trace) {tty->print_cr("Final/stable");}

              verify = false;

            } else if (k == ciEnv::current()->Class_klass() &&

                       tinst->const_oop() != NULL &&

                       tinst->offset() >= (ik->size_helper() * wordSize)) {

              ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();

              ciField* field = k->get_field_by_offset(tinst->offset(), true);

              if ((ShenandoahOptimizeStaticFinals && field->is_final()) ||

                  (ShenandoahOptimizeStableFinals && field->is_stable())) {

                verify = false;

              }

            }

          }

            report_verify_failure("Shenandoah verification: Load should have barriers", n);

          }

        }

      }

    } else if (n->is_Store()) {

      const bool trace = false;

      if (trace) {tty->print("Verifying"); n->dump();}

      if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) {

        Node* adr = n->in(MemNode::Address);

        bool verify = true;

        if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) {

          adr = adr->in(AddPNode::Address);

          if (adr->is_AddP()) {

            assert(adr->in(AddPNode::Base)->is_top(), "");

            adr = adr->in(AddPNode::Address);

            if (adr->Opcode() == Op_LoadP &&

                adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() &&

                adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&

                adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) {

              if (trace) {tty->print_cr("SATB prebarrier");}

              verify = false;

            }

          }

        }

          report_verify_failure("Shenandoah verification: Store should have barriers", n);

        }

      }

        report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);

      }

    } else if (n->Opcode() == Op_CmpP) {

      const bool trace = false;

      Node* in1 = n->in(1);

      Node* in2 = n->in(2);

      if (in1->bottom_type()->isa_oopptr()) {

        if (trace) {tty->print("Verifying"); n->dump();}

        bool mark_inputs = false;

        if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR ||

            (in1->is_Con() || in2->is_Con())) {

          if (trace) {tty->print_cr("Comparison against a constant");}

          mark_inputs = true;

        } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) ||

                   (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) {

          if (trace) {tty->print_cr("Comparison with newly alloc'ed object");}

          mark_inputs = true;

        } else {

          assert(in2->bottom_type()->isa_oopptr(), "");

            report_verify_failure("Shenandoah verification: Cmp should have barriers", n);

          }

        }

        if (verify_no_useless_barrier &&

            mark_inputs &&

          phis.clear();

          visited.Reset();

        }

      }

    } else if (n->is_LoadStore()) {

      if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&

        report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);

      }

        report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);

      }

    } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {

      CallNode* call = n->as_Call();

      static struct {

        const char* name;

        struct {

          int pos;

          verify_type t;

        } args[6];

      } calls[] = {

        "aescrypt_encryptBlock",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "aescrypt_decryptBlock",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "multiplyToLen",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },   { TypeFunc::Parms+4, ShenandoahStore },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "squareToLen",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },   { -1,  ShenandoahNone},

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "montgomery_multiply",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },

          { TypeFunc::Parms+6, ShenandoahStore }, { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "montgomery_square",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahLoad },   { TypeFunc::Parms+5, ShenandoahStore },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "mulAdd",

        { { TypeFunc::Parms, ShenandoahStore },  { TypeFunc::Parms+1, ShenandoahLoad },   { -1,  ShenandoahNone},

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "vectorizedMismatch",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahLoad },   { -1,  ShenandoahNone},

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "updateBytesCRC32",

        { { TypeFunc::Parms+1, ShenandoahLoad }, { -1,  ShenandoahNone},                  { -1,  ShenandoahNone},

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "updateBytesAdler32",

        { { TypeFunc::Parms+1, ShenandoahLoad }, { -1,  ShenandoahNone},                  { -1,  ShenandoahNone},

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "updateBytesCRC32C",

        { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad},    { -1,  ShenandoahNone},

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "counterMode_AESCrypt",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },

          { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } },

        "cipherBlockChaining_encryptAESCrypt",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },

          { TypeFunc::Parms+3, ShenandoahLoad },  { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "cipherBlockChaining_decryptAESCrypt",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },

          { TypeFunc::Parms+3, ShenandoahLoad },  { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "shenandoah_clone_barrier",

        { { TypeFunc::Parms, ShenandoahLoad },   { -1,  ShenandoahNone},                  { -1,  ShenandoahNone},

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "ghash_processBlocks",

        { { TypeFunc::Parms, ShenandoahStore },  { TypeFunc::Parms+1, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "sha1_implCompress",

        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "sha256_implCompress",

        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "sha512_implCompress",

        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "sha1_implCompressMB",

        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "sha256_implCompressMB",

        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "sha512_implCompressMB",

        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "encodeBlock",

        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

      };

      if (call->is_call_to_arraycopystub()) {

        Node* dest = NULL;

        const TypeTuple* args = n->as_Call()->_tf->domain();

        for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {

          if (args->field_at(i)->isa_ptr()) {

            j++;

            if (j == 2) {

              dest = n->in(i);

              break;

            }

          }

        }