/*

 * Copyright (c) 2015, 2019, Red Hat, Inc. All rights reserved.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "gc/shenandoah/c2/shenandoahSupport.hpp"

#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"

#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"

#include "gc/shenandoah/shenandoahForwarding.hpp"

#include "gc/shenandoah/shenandoahHeap.hpp"

#include "gc/shenandoah/shenandoahHeapRegion.hpp"

#include "gc/shenandoah/shenandoahRuntime.hpp"

#include "gc/shenandoah/shenandoahThreadLocalData.hpp"

#include "opto/arraycopynode.hpp"

#include "opto/block.hpp"

#include "opto/callnode.hpp"

#include "opto/castnode.hpp"

#include "opto/movenode.hpp"

#include "opto/phaseX.hpp"

#include "opto/rootnode.hpp"

#include "opto/runtime.hpp"

#include "opto/subnode.hpp"

bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {

  ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();

  if ((state->enqueue_barriers_count() +

       state->load_reference_barriers_count()) > 0) {

    bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion;

    C->clear_major_progress();

    PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand);

    if (C->failing()) return false;

    PhaseIdealLoop::verify(igvn);

    DEBUG_ONLY(verify_raw_mem(C->root());)

    if (attempt_more_loopopts) {

      C->set_major_progress();

      if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) {

        return false;

      }

      C->clear_major_progress();

    }

  }

  return true;

}

bool ShenandoahBarrierC2Support::is_heap_state_test(Node* iff, int mask) {

  if (!UseShenandoahGC) {

    return false;

  }

  assert(iff->is_If(), "bad input");

  if (iff->Opcode() != Op_If) {

    return false;

  }

  Node* bol = iff->in(1);

  if (!bol->is_Bool() || bol->as_Bool()->_test._test != BoolTest::ne) {

    return false;

  }

  Node* cmp = bol->in(1);

  if (cmp->Opcode() != Op_CmpI) {

    return false;

  }

  Node* in1 = cmp->in(1);

  Node* in2 = cmp->in(2);

  if (in2->find_int_con(-1) != 0) {

    return false;

  }

  if (in1->Opcode() != Op_AndI) {

    return false;

  }

  in2 = in1->in(2);

  if (in2->find_int_con(-1) != mask) {

    return false;

  }

  in1 = in1->in(1);

  return is_gc_state_load(in1);

}

bool ShenandoahBarrierC2Support::is_heap_stable_test(Node* iff) {

  return is_heap_state_test(iff, ShenandoahHeap::HAS_FORWARDED);

}

bool ShenandoahBarrierC2Support::is_gc_state_load(Node *n) {

  if (!UseShenandoahGC) {

    return false;

  }

  if (n->Opcode() != Op_LoadB && n->Opcode() != Op_LoadUB) {

    return false;

  }

  Node* addp = n->in(MemNode::Address);

  if (!addp->is_AddP()) {

    return false;

  }

  Node* base = addp->in(AddPNode::Address);

  Node* off = addp->in(AddPNode::Offset);

  if (base->Opcode() != Op_ThreadLocal) {

    return false;

  }

  if (off->find_intptr_t_con(-1) != in_bytes(ShenandoahThreadLocalData::gc_state_offset())) {

    return false;

  }

  return true;

}

bool ShenandoahBarrierC2Support::has_safepoint_between(Node* start, Node* stop, PhaseIdealLoop *phase) {

  assert(phase->is_dominator(stop, start), "bad inputs");

  ResourceMark rm;

  Unique_Node_List wq;

  wq.push(start);

  for (uint next = 0; next < wq.size(); next++) {

    Node *m = wq.at(next);

    if (m == stop) {

      continue;

    }

    if (m->is_SafePoint() && !m->is_CallLeaf()) {

      return true;

    }

    if (m->is_Region()) {

      for (uint i = 1; i < m->req(); i++) {

        wq.push(m->in(i));

      }

    } else {

      wq.push(m->in(0));

    }

  }

  return false;

}

bool ShenandoahBarrierC2Support::try_common_gc_state_load(Node *n, PhaseIdealLoop *phase) {

  assert(is_gc_state_load(n), "inconsistent");

  Node* addp = n->in(MemNode::Address);

  Node* dominator = NULL;

  for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {

    Node* u = addp->fast_out(i);

    assert(is_gc_state_load(u), "inconsistent");

    if (u != n && phase->is_dominator(u->in(0), n->in(0))) {

      if (dominator == NULL) {

        dominator = u;

      } else {

        if (phase->dom_depth(u->in(0)) < phase->dom_depth(dominator->in(0))) {

          dominator = u;

        }

      }

    }

  }

  if (dominator == NULL || has_safepoint_between(n->in(0), dominator->in(0), phase)) {

    return false;

  }

  phase->igvn().replace_node(n, dominator);

  return true;

}

#ifdef ASSERT

bool ShenandoahBarrierC2Support::verify_helper(Node* in, Node_Stack& phis, VectorSet& visited, verify_type t, bool trace, Unique_Node_List& barriers_used) {

  assert(phis.size() == 0, "");

  while (true) {

    if (in->bottom_type() == TypePtr::NULL_PTR) {

      if (trace) {tty->print_cr("NULL");}

    } else if (!in->bottom_type()->make_ptr()->make_oopptr()) {

      if (trace) {tty->print_cr("Non oop");}

    } else {

      if (in->is_ConstraintCast()) {

        in = in->in(1);

        continue;

      } else if (in->is_AddP()) {

        assert(!in->in(AddPNode::Address)->is_top(), "no raw memory access");

        in = in->in(AddPNode::Address);

        continue;

      } else if (in->is_Con()) {

        if (trace) {

          tty->print("Found constant");

          in->dump();

        }

      } else if (in->Opcode() == Op_Parm) {

        if (trace) {

          tty->print("Found argument");

        }

      } else if (in->Opcode() == Op_CreateEx) {

        if (trace) {

          tty->print("Found create-exception");

        }

      } else if (in->Opcode() == Op_LoadP && in->adr_type() == TypeRawPtr::BOTTOM) {

        if (trace) {

          tty->print("Found raw LoadP (OSR argument?)");

        }

      } else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) {

        if (t == ShenandoahOopStore) {

          uint i = 0;

          for (; i < phis.size(); i++) {

            Node* n = phis.node_at(i);

            if (n->Opcode() == Op_ShenandoahEnqueueBarrier) {

              break;

            }

          }

          if (i == phis.size()) {

            return false;

          }

        }

        barriers_used.push(in);

        if (trace) {tty->print("Found barrier"); in->dump();}

      } else if (in->Opcode() == Op_ShenandoahEnqueueBarrier) {

        if (t != ShenandoahOopStore) {

          in = in->in(1);

          continue;

        }

        if (trace) {tty->print("Found enqueue barrier"); in->dump();}

        phis.push(in, in->req());

        in = in->in(1);

        continue;

      } else if (in->is_Proj() && in->in(0)->is_Allocate()) {

        if (trace) {

          tty->print("Found alloc");

          in->in(0)->dump();

        }

      } else if (in->is_Proj() && (in->in(0)->Opcode() == Op_CallStaticJava || in->in(0)->Opcode() == Op_CallDynamicJava)) {

        if (trace) {

          tty->print("Found Java call");

        }

      } else if (in->is_Phi()) {

        if (!visited.test_set(in->_idx)) {

          if (trace) {tty->print("Pushed phi:"); in->dump();}

          phis.push(in, 2);

          in = in->in(1);

          continue;

        }

        if (trace) {tty->print("Already seen phi:"); in->dump();}

      } else if (in->Opcode() == Op_CMoveP || in->Opcode() == Op_CMoveN) {

        if (!visited.test_set(in->_idx)) {

          if (trace) {tty->print("Pushed cmovep:"); in->dump();}

          phis.push(in, CMoveNode::IfTrue);

          in = in->in(CMoveNode::IfFalse);

          continue;

        }

        if (trace) {tty->print("Already seen cmovep:"); in->dump();}

      } else if (in->Opcode() == Op_EncodeP || in->Opcode() == Op_DecodeN) {

        in = in->in(1);

        continue;

      } else {

        return false;

      }

    }

    bool cont = false;

    while (phis.is_nonempty()) {

      uint idx = phis.index();

      Node* phi = phis.node();

      if (idx >= phi->req()) {

        if (trace) {tty->print("Popped phi:"); phi->dump();}

        phis.pop();

        continue;

      }

      if (trace) {tty->print("Next entry(%d) for phi:", idx); phi->dump();}

      in = phi->in(idx);

      phis.set_index(idx+1);

      cont = true;

      break;

    }

    if (!cont) {

      break;

    }

  }

  return true;

}

void ShenandoahBarrierC2Support::report_verify_failure(const char* msg, Node* n1, Node* n2) {

  if (n1 != NULL) {

    n1->dump(+10);

  }

  if (n2 != NULL) {

    n2->dump(+10);

  }

  fatal("%s", msg);

}

void ShenandoahBarrierC2Support::verify(RootNode* root) {

  ResourceMark rm;

  Unique_Node_List wq;

  GrowableArray<Node*> barriers;

  Unique_Node_List barriers_used;

  Node_Stack phis(0);

  VectorSet visited(Thread::current()->resource_area());

  const bool trace = false;

  const bool verify_no_useless_barrier = false;

  wq.push(root);

  for (uint next = 0; next < wq.size(); next++) {

    Node *n = wq.at(next);

    if (n->is_Load()) {

      const bool trace = false;

      if (trace) {tty->print("Verifying"); n->dump();}

      if (n->Opcode() == Op_LoadRange || n->Opcode() == Op_LoadKlass || n->Opcode() == Op_LoadNKlass) {

        if (trace) {tty->print_cr("Load range/klass");}

      } else {

        const TypePtr* adr_type = n->as_Load()->adr_type();

        if (adr_type->isa_oopptr() && adr_type->is_oopptr()->offset() == oopDesc::mark_offset_in_bytes()) {

          if (trace) {tty->print_cr("Mark load");}

        } else if (adr_type->isa_instptr() &&

                   adr_type->is_instptr()->klass()->is_subtype_of(Compile::current()->env()->Reference_klass()) &&

                   adr_type->is_instptr()->offset() == java_lang_ref_Reference::referent_offset) {

          if (trace) {tty->print_cr("Reference.get()");}

        } else if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahLoad, trace, barriers_used)) {

          report_verify_failure("Shenandoah verification: Load should have barriers", n);

        }

      }

    } else if (n->is_Store()) {

      const bool trace = false;

      if (trace) {tty->print("Verifying"); n->dump();}

      if (n->in(MemNode::ValueIn)->bottom_type()->make_oopptr()) {

        Node* adr = n->in(MemNode::Address);

        bool verify = true;

        if (adr->is_AddP() && adr->in(AddPNode::Base)->is_top()) {

          adr = adr->in(AddPNode::Address);

          if (adr->is_AddP()) {

            assert(adr->in(AddPNode::Base)->is_top(), "");

            adr = adr->in(AddPNode::Address);

            if (adr->Opcode() == Op_LoadP &&

                adr->in(MemNode::Address)->in(AddPNode::Base)->is_top() &&

                adr->in(MemNode::Address)->in(AddPNode::Address)->Opcode() == Op_ThreadLocal &&

                adr->in(MemNode::Address)->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())) {

              if (trace) {tty->print_cr("SATB prebarrier");}

              verify = false;

            }

          }

        }

        if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {

          report_verify_failure("Shenandoah verification: Store should have barriers", n);

        }

      }

      if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {

        report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);

      }

    } else if (n->Opcode() == Op_CmpP) {

      const bool trace = false;

      Node* in1 = n->in(1);

      Node* in2 = n->in(2);

      if (in1->bottom_type()->isa_oopptr()) {

        if (trace) {tty->print("Verifying"); n->dump();}

        bool mark_inputs = false;

        if (in1->bottom_type() == TypePtr::NULL_PTR || in2->bottom_type() == TypePtr::NULL_PTR ||

            (in1->is_Con() || in2->is_Con())) {

          if (trace) {tty->print_cr("Comparison against a constant");}

          mark_inputs = true;

        } else if ((in1->is_CheckCastPP() && in1->in(1)->is_Proj() && in1->in(1)->in(0)->is_Allocate()) ||

                   (in2->is_CheckCastPP() && in2->in(1)->is_Proj() && in2->in(1)->in(0)->is_Allocate())) {

          if (trace) {tty->print_cr("Comparison with newly alloc'ed object");}

          mark_inputs = true;

        } else {

          assert(in2->bottom_type()->isa_oopptr(), "");

          if (!verify_helper(in1, phis, visited, ShenandoahStore, trace, barriers_used) ||

              !verify_helper(in2, phis, visited, ShenandoahStore, trace, barriers_used)) {

            report_verify_failure("Shenandoah verification: Cmp should have barriers", n);

          }

        }

        if (verify_no_useless_barrier &&

            mark_inputs &&

            (!verify_helper(in1, phis, visited, ShenandoahValue, trace, barriers_used) ||

             !verify_helper(in2, phis, visited, ShenandoahValue, trace, barriers_used))) {

          phis.clear();

          visited.Reset();

        }

      }

    } else if (n->is_LoadStore()) {

      if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&

          !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahStoreValEnqueueBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {

        report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);

      }

      if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {

        report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);

      }

    } else if (n->Opcode() == Op_CallLeafNoFP || n->Opcode() == Op_CallLeaf) {

      CallNode* call = n->as_Call();

      static struct {

        const char* name;

        struct {

          int pos;

          verify_type t;

        } args[6];

      } calls[] = {

        "aescrypt_encryptBlock",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "aescrypt_decryptBlock",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "multiplyToLen",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },   { TypeFunc::Parms+4, ShenandoahStore },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "squareToLen",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },   { -1,  ShenandoahNone},

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "montgomery_multiply",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },

          { TypeFunc::Parms+6, ShenandoahStore }, { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "montgomery_square",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahLoad },   { TypeFunc::Parms+5, ShenandoahStore },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "mulAdd",

        { { TypeFunc::Parms, ShenandoahStore },  { TypeFunc::Parms+1, ShenandoahLoad },   { -1,  ShenandoahNone},

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "vectorizedMismatch",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahLoad },   { -1,  ShenandoahNone},

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "updateBytesCRC32",

        { { TypeFunc::Parms+1, ShenandoahLoad }, { -1,  ShenandoahNone},                  { -1,  ShenandoahNone},

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "updateBytesAdler32",

        { { TypeFunc::Parms+1, ShenandoahLoad }, { -1,  ShenandoahNone},                  { -1,  ShenandoahNone},

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "updateBytesCRC32C",

        { { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+3, ShenandoahLoad},    { -1,  ShenandoahNone},

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "counterMode_AESCrypt",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },

          { TypeFunc::Parms+3, ShenandoahStore }, { TypeFunc::Parms+5, ShenandoahStore }, { TypeFunc::Parms+6, ShenandoahStore } },

        "cipherBlockChaining_encryptAESCrypt",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },

          { TypeFunc::Parms+3, ShenandoahLoad },  { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "cipherBlockChaining_decryptAESCrypt",

        { { TypeFunc::Parms, ShenandoahLoad },   { TypeFunc::Parms+1, ShenandoahStore },  { TypeFunc::Parms+2, ShenandoahLoad },

          { TypeFunc::Parms+3, ShenandoahLoad },  { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "shenandoah_clone_barrier",

        { { TypeFunc::Parms, ShenandoahLoad },   { -1,  ShenandoahNone},                  { -1,  ShenandoahNone},

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "ghash_processBlocks",

        { { TypeFunc::Parms, ShenandoahStore },  { TypeFunc::Parms+1, ShenandoahLoad },   { TypeFunc::Parms+2, ShenandoahLoad },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "sha1_implCompress",

        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "sha256_implCompress",

        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "sha512_implCompress",

        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "sha1_implCompressMB",

        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "sha256_implCompressMB",

        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "sha512_implCompressMB",

        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+1, ShenandoahStore },   { -1, ShenandoahNone },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

        "encodeBlock",

        { { TypeFunc::Parms, ShenandoahLoad },  { TypeFunc::Parms+3, ShenandoahStore },   { -1, ShenandoahNone },

          { -1,  ShenandoahNone},                 { -1,  ShenandoahNone},                 { -1,  ShenandoahNone} },

      };

      if (call->is_call_to_arraycopystub()) {

        Node* dest = NULL;

        const TypeTuple* args = n->as_Call()->_tf->domain();

        for (uint i = TypeFunc::Parms, j = 0; i < args->cnt(); i++) {

          if (args->field_at(i)->isa_ptr()) {

            j++;

            if (j == 2) {

              dest = n->in(i);

              break;

            }

          }

        }

        if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahLoad, trace, barriers_used) ||

            !verify_helper(dest, phis, visited, ShenandoahStore, trace, barriers_used)) {

          report_verify_failure("Shenandoah verification: ArrayCopy should have barriers", n);

        }

      } else if (strlen(call->_name) > 5 &&

                 !strcmp(call->_name + strlen(call->_name) - 5, "_fill")) {

        if (!verify_helper(n->in(TypeFunc::Parms), phis, visited, ShenandoahStore, trace, barriers_used)) {

          report_verify_failure("Shenandoah verification: _fill should have barriers", n);

        }

      } else if (!strcmp(call->_name, "shenandoah_wb_pre")) {

        // skip

      } else {

        const int calls_len = sizeof(calls) / sizeof(calls[0]);

        int i = 0;

        for (; i < calls_len; i++) {

          if (!strcmp(calls[i].name, call->_name)) {

            break;

          }

        }

        if (i != calls_len) {

          const uint args_len = sizeof(calls[0].args) / sizeof(calls[0].args[0]);

          for (uint j = 0; j < args_len; j++) {

            int pos = calls[i].args[j].pos;

            if (pos == -1) {

              break;

            }

            if (!verify_helper(call->in(pos), phis, visited, calls[i].args[j].t, trace, barriers_used)) {

              report_verify_failure("Shenandoah verification: intrinsic calls should have barriers", n);

            }

          }