/*

 * Copyright (c) 1998, 2018, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#ifndef SHARE_VM_OPTO_LOOPNODE_HPP

#define SHARE_VM_OPTO_LOOPNODE_HPP

#include "opto/cfgnode.hpp"

#include "opto/multnode.hpp"

#include "opto/phaseX.hpp"

#include "opto/subnode.hpp"

#include "opto/type.hpp"

class CmpNode;

class CountedLoopEndNode;

class CountedLoopNode;

class IdealLoopTree;

class LoopNode;

class Node;

class OuterStripMinedLoopEndNode;

class PathFrequency;

class PhaseIdealLoop;

class CountedLoopReserveKit;

class VectorSet;

class Invariance;

struct small_cache;

//

//                  I D E A L I Z E D   L O O P S

//

// Idealized loops are the set of loops I perform more interesting

// transformations on, beyond simple hoisting.

//------------------------------LoopNode---------------------------------------

// Simple loop header.  Fall in path on left, loop-back path on right.

class LoopNode : public RegionNode {

  // Size is bigger to hold the flags.  However, the flags do not change

  // the semantics so it does not appear in the hash & cmp functions.

  virtual uint size_of() const { return sizeof(*this); }

protected:

  uint _loop_flags;

  // Names for flag bitfields

  enum { Normal=0, Pre=1, Main=2, Post=3, PreMainPostFlagsMask=3,

         MainHasNoPreLoop=4,

         HasExactTripCount=8,

         InnerLoop=16,

         PartialPeelLoop=32,

         PartialPeelFailed=64,

         HasReductions=128,

         WasSlpAnalyzed=256,

         PassedSlpAnalysis=512,

         DoUnrollOnly=1024,

         VectorizedLoop=2048,

         HasAtomicPostLoop=4096,

         HasRangeChecks=8192,

         IsMultiversioned=16384,

         StripMined=32768,

         SubwordLoop=65536,

         ProfileTripFailed=131072};

  char _unswitch_count;

  enum { _unswitch_max=3 };

  char _postloop_flags;

  enum { LoopNotRCEChecked = 0, LoopRCEChecked = 1, RCEPostLoop = 2 };

  // Expected trip count from profile data

  float _profile_trip_cnt;

public:

  // Names for edge indices

  enum { Self=0, EntryControl, LoopBackControl };

  bool is_inner_loop() const { return _loop_flags & InnerLoop; }

  void set_inner_loop() { _loop_flags |= InnerLoop; }

  bool range_checks_present() const { return _loop_flags & HasRangeChecks; }

  bool is_multiversioned() const { return _loop_flags & IsMultiversioned; }

  bool is_vectorized_loop() const { return _loop_flags & VectorizedLoop; }

  bool is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; }

  void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; }

  bool partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; }

  bool is_strip_mined() const { return _loop_flags & StripMined; }

  bool is_profile_trip_failed() const { return _loop_flags & ProfileTripFailed; }

  bool is_subword_loop() const { return _loop_flags & SubwordLoop; }

  void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; }

  void mark_has_reductions() { _loop_flags |= HasReductions; }

  void mark_was_slp() { _loop_flags |= WasSlpAnalyzed; }

  void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; }

  void mark_do_unroll_only() { _loop_flags |= DoUnrollOnly; }

  void mark_loop_vectorized() { _loop_flags |= VectorizedLoop; }

  void mark_has_atomic_post_loop() { _loop_flags |= HasAtomicPostLoop; }

  void mark_has_range_checks() { _loop_flags |=  HasRangeChecks; }

  void mark_is_multiversioned() { _loop_flags |= IsMultiversioned; }

  void mark_strip_mined() { _loop_flags |= StripMined; }

  void clear_strip_mined() { _loop_flags &= ~StripMined; }

  void mark_profile_trip_failed() { _loop_flags |= ProfileTripFailed; }

  void mark_subword_loop() { _loop_flags |= SubwordLoop; }

  int unswitch_max() { return _unswitch_max; }

  int unswitch_count() { return _unswitch_count; }

  int has_been_range_checked() const { return _postloop_flags & LoopRCEChecked; }

  void set_has_been_range_checked() { _postloop_flags |= LoopRCEChecked; }

  int is_rce_post_loop() const { return _postloop_flags & RCEPostLoop; }

  void set_is_rce_post_loop() { _postloop_flags |= RCEPostLoop; }

  void set_unswitch_count(int val) {

    assert (val <= unswitch_max(), "too many unswitches");

    _unswitch_count = val;

  }

  void set_profile_trip_cnt(float ptc) { _profile_trip_cnt = ptc; }

  float profile_trip_cnt()             { return _profile_trip_cnt; }

  LoopNode(Node *entry, Node *backedge)

    : RegionNode(3), _loop_flags(0), _unswitch_count(0),

      _postloop_flags(0), _profile_trip_cnt(COUNT_UNKNOWN)  {

    init_class_id(Class_Loop);

    init_req(EntryControl, entry);

    init_req(LoopBackControl, backedge);

  }

  virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);

  virtual int Opcode() const;

  bool can_be_counted_loop(PhaseTransform* phase) const {

    return req() == 3 && in(0) != NULL &&

      in(1) != NULL && phase->type(in(1)) != Type::TOP &&

      in(2) != NULL && phase->type(in(2)) != Type::TOP;

  }

  bool is_valid_counted_loop() const;

#ifndef PRODUCT

  virtual void dump_spec(outputStream *st) const;

#endif

  void verify_strip_mined(int expect_skeleton) const;

  virtual LoopNode* skip_strip_mined(int expect_skeleton = 1) { return this; }

  virtual IfTrueNode* outer_loop_tail() const { ShouldNotReachHere(); return NULL; }

  virtual OuterStripMinedLoopEndNode* outer_loop_end() const { ShouldNotReachHere(); return NULL; }

  virtual IfFalseNode* outer_loop_exit() const { ShouldNotReachHere(); return NULL; }

  virtual SafePointNode* outer_safepoint() const { ShouldNotReachHere(); return NULL; }

};

//------------------------------Counted Loops----------------------------------

// Counted loops are all trip-counted loops, with exactly 1 trip-counter exit

// path (and maybe some other exit paths).  The trip-counter exit is always

// last in the loop.  The trip-counter have to stride by a constant;

// the exit value is also loop invariant.

// CountedLoopNodes and CountedLoopEndNodes come in matched pairs.  The

// CountedLoopNode has the incoming loop control and the loop-back-control

// which is always the IfTrue before the matching CountedLoopEndNode.  The

// CountedLoopEndNode has an incoming control (possibly not the

// CountedLoopNode if there is control flow in the loop), the post-increment

// trip-counter value, and the limit.  The trip-counter value is always of

// the form (Op old-trip-counter stride).  The old-trip-counter is produced

// by a Phi connected to the CountedLoopNode.  The stride is constant.

// The Op is any commutable opcode, including Add, Mul, Xor.  The

// CountedLoopEndNode also takes in the loop-invariant limit value.

// From a CountedLoopNode I can reach the matching CountedLoopEndNode via the

// loop-back control.  From CountedLoopEndNodes I can reach CountedLoopNodes

// via the old-trip-counter from the Op node.

//------------------------------CountedLoopNode--------------------------------

// CountedLoopNodes head simple counted loops.  CountedLoopNodes have as

// inputs the incoming loop-start control and the loop-back control, so they

// act like RegionNodes.  They also take in the initial trip counter, the

// loop-invariant stride and the loop-invariant limit value.  CountedLoopNodes

// produce a loop-body control and the trip counter value.  Since

// CountedLoopNodes behave like RegionNodes I still have a standard CFG model.

class CountedLoopNode : public LoopNode {

  // Size is bigger to hold _main_idx.  However, _main_idx does not change

  // the semantics so it does not appear in the hash & cmp functions.

  virtual uint size_of() const { return sizeof(*this); }

  // For Pre- and Post-loops during debugging ONLY, this holds the index of

  // the Main CountedLoop.  Used to assert that we understand the graph shape.

  node_idx_t _main_idx;

  // Known trip count calculated by compute_exact_trip_count()

  uint  _trip_count;

  // Log2 of original loop bodies in unrolled loop

  int _unrolled_count_log2;

  // Node count prior to last unrolling - used to decide if

  // unroll,optimize,unroll,optimize,... is making progress

  int _node_count_before_unroll;

  // If slp analysis is performed we record the maximum

  // vector mapped unroll factor here

  int _slp_maximum_unroll_factor;

public:

  CountedLoopNode( Node *entry, Node *backedge )

    : LoopNode(entry, backedge), _main_idx(0), _trip_count(max_juint),

      _unrolled_count_log2(0), _node_count_before_unroll(0),

      _slp_maximum_unroll_factor(0) {

    init_class_id(Class_CountedLoop);

    // Initialize _trip_count to the largest possible value.

    // Will be reset (lower) if the loop's trip count is known.

  }

  virtual int Opcode() const;

  virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);

  Node *init_control() const { return in(EntryControl); }

  Node *back_control() const { return in(LoopBackControl); }

  CountedLoopEndNode *loopexit_or_null() const;

  CountedLoopEndNode *loopexit() const;

  Node *init_trip() const;

  Node *stride() const;

  int   stride_con() const;

  bool  stride_is_con() const;

  Node *limit() const;

  Node *incr() const;

  Node *phi() const;

  // Match increment with optional truncation

  static Node* match_incr_with_optional_truncation(Node* expr, Node** trunc1, Node** trunc2, const TypeInt** trunc_type);

  // A 'main' loop has a pre-loop and a post-loop.  The 'main' loop

  // can run short a few iterations and may start a few iterations in.

  // It will be RCE'd and unrolled and aligned.

  // A following 'post' loop will run any remaining iterations.  Used

  // during Range Check Elimination, the 'post' loop will do any final

  // iterations with full checks.  Also used by Loop Unrolling, where

  // the 'post' loop will do any epilog iterations needed.  Basically,

  // a 'post' loop can not profitably be further unrolled or RCE'd.

  // A preceding 'pre' loop will run at least 1 iteration (to do peeling),

  // it may do under-flow checks for RCE and may do alignment iterations

  // so the following main loop 'knows' that it is striding down cache

  // lines.

  // A 'main' loop that is ONLY unrolled or peeled, never RCE'd or

  // Aligned, may be missing it's pre-loop.

  bool is_normal_loop   () const { return (_loop_flags&PreMainPostFlagsMask) == Normal; }

  bool is_pre_loop      () const { return (_loop_flags&PreMainPostFlagsMask) == Pre;    }

  bool is_main_loop     () const { return (_loop_flags&PreMainPostFlagsMask) == Main;   }

  bool is_post_loop     () const { return (_loop_flags&PreMainPostFlagsMask) == Post;   }

  bool is_reduction_loop() const { return (_loop_flags&HasReductions) == HasReductions; }

  bool was_slp_analyzed () const { return (_loop_flags&WasSlpAnalyzed) == WasSlpAnalyzed; }

  bool has_passed_slp   () const { return (_loop_flags&PassedSlpAnalysis) == PassedSlpAnalysis; }

  bool do_unroll_only      () const { return (_loop_flags&DoUnrollOnly) == DoUnrollOnly; }

  bool is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; }

  bool has_atomic_post_loop  () const { return (_loop_flags & HasAtomicPostLoop) == HasAtomicPostLoop; }

  void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; }

  int main_idx() const { return _main_idx; }

  void set_pre_loop  (CountedLoopNode *main) { assert(is_normal_loop(),""); _loop_flags |= Pre ; _main_idx = main->_idx; }

  void set_main_loop (                     ) { assert(is_normal_loop(),""); _loop_flags |= Main;                         }

  void set_post_loop (CountedLoopNode *main) { assert(is_normal_loop(),""); _loop_flags |= Post; _main_idx = main->_idx; }

  void set_normal_loop(                    ) { _loop_flags &= ~PreMainPostFlagsMask; }

  void set_trip_count(uint tc) { _trip_count = tc; }

  uint trip_count()            { return _trip_count; }

  bool has_exact_trip_count() const { return (_loop_flags & HasExactTripCount) != 0; }

  void set_exact_trip_count(uint tc) {

    _trip_count = tc;

    _loop_flags |= HasExactTripCount;

  }

  void set_nonexact_trip_count() {

    _loop_flags &= ~HasExactTripCount;

  }

  void set_notpassed_slp() {

    _loop_flags &= ~PassedSlpAnalysis;

  }

  void double_unrolled_count() { _unrolled_count_log2++; }

  int  unrolled_count()        { return 1 << MIN2(_unrolled_count_log2, BitsPerInt-3); }

  void set_node_count_before_unroll(int ct)  { _node_count_before_unroll = ct; }

  int  node_count_before_unroll()            { return _node_count_before_unroll; }

  void set_slp_max_unroll(int unroll_factor) { _slp_maximum_unroll_factor = unroll_factor; }

  int  slp_max_unroll() const                { return _slp_maximum_unroll_factor; }

  virtual LoopNode* skip_strip_mined(int expect_opaq = 1);

  OuterStripMinedLoopNode* outer_loop() const;

  virtual IfTrueNode* outer_loop_tail() const;

  virtual OuterStripMinedLoopEndNode* outer_loop_end() const;

  virtual IfFalseNode* outer_loop_exit() const;

  virtual SafePointNode* outer_safepoint() const;

  // If this is a main loop in a pre/main/post loop nest, walk over

  // the predicates that were inserted by

  // duplicate_predicates()/add_range_check_predicate()

  static Node* skip_predicates_from_entry(Node* ctrl);

  Node* skip_predicates();

#ifndef PRODUCT

  virtual void dump_spec(outputStream *st) const;

#endif

};

//------------------------------CountedLoopEndNode-----------------------------

// CountedLoopEndNodes end simple trip counted loops.  They act much like

// IfNodes.

class CountedLoopEndNode : public IfNode {

public:

  enum { TestControl, TestValue };

  CountedLoopEndNode( Node *control, Node *test, float prob, float cnt )

    : IfNode( control, test, prob, cnt) {

    init_class_id(Class_CountedLoopEnd);

  }

  virtual int Opcode() const;

  Node *cmp_node() const            { return (in(TestValue)->req() >=2) ? in(TestValue)->in(1) : NULL; }

  Node *incr() const                { Node *tmp = cmp_node(); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }

  Node *limit() const               { Node *tmp = cmp_node(); return (tmp && tmp->req()==3) ? tmp->in(2) : NULL; }

  Node *stride() const              { Node *tmp = incr    (); return (tmp && tmp->req()==3) ? tmp->in(2) : NULL; }

  Node *init_trip() const           { Node *tmp = phi     (); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }

  int stride_con() const;

  bool stride_is_con() const        { Node *tmp = stride  (); return (tmp != NULL && tmp->is_Con()); }

  BoolTest::mask test_trip() const  { return in(TestValue)->as_Bool()->_test._test; }

  PhiNode *phi() const {

    Node *tmp = incr();

    if (tmp && tmp->req() == 3) {

      Node* phi = tmp->in(1);

      if (phi->is_Phi()) {

        return phi->as_Phi();

      }

    }

    return NULL;

  }

  CountedLoopNode *loopnode() const {

    // The CountedLoopNode that goes with this CountedLoopEndNode may

    // have been optimized out by the IGVN so be cautious with the

    // pattern matching on the graph

    PhiNode* iv_phi = phi();

    if (iv_phi == NULL) {

      return NULL;

    }

    Node *ln = iv_phi->in(0);

    if (ln->is_CountedLoop() && ln->as_CountedLoop()->loopexit_or_null() == this) {

      return (CountedLoopNode*)ln;

    }

    return NULL;

  }

#ifndef PRODUCT

  virtual void dump_spec(outputStream *st) const;

#endif

};

inline CountedLoopEndNode *CountedLoopNode::loopexit_or_null() const {

  Node *bc = back_control();

  if( bc == NULL ) return NULL;

  Node *le = bc->in(0);

  if( le->Opcode() != Op_CountedLoopEnd )

    return NULL;

  return (CountedLoopEndNode*)le;

}

inline CountedLoopEndNode *CountedLoopNode::loopexit() const {

  CountedLoopEndNode* cle = loopexit_or_null();

  assert(cle != NULL, "loopexit is NULL");

  return cle;

}

inline Node *CountedLoopNode::init_trip() const { return loopexit_or_null() ? loopexit()->init_trip() : NULL; }

inline Node *CountedLoopNode::stride() const { return loopexit_or_null() ? loopexit()->stride() : NULL; }

inline int CountedLoopNode::stride_con() const { return loopexit_or_null() ? loopexit()->stride_con() : 0; }

inline bool CountedLoopNode::stride_is_con() const { return loopexit_or_null() && loopexit()->stride_is_con(); }

inline Node *CountedLoopNode::limit() const { return loopexit_or_null() ? loopexit()->limit() : NULL; }

inline Node *CountedLoopNode::incr() const { return loopexit_or_null() ? loopexit()->incr() : NULL; }

inline Node *CountedLoopNode::phi() const { return loopexit_or_null() ? loopexit()->phi() : NULL; }

//------------------------------LoopLimitNode-----------------------------

// Counted Loop limit node which represents exact final iterator value:

// trip_count = (limit - init_trip + stride - 1)/stride

// final_value= trip_count * stride + init_trip.

// Use HW instructions to calculate it when it can overflow in integer.

// Note, final_value should fit into integer since counted loop has

// limit check: limit <= max_int-stride.

class LoopLimitNode : public Node {

  enum { Init=1, Limit=2, Stride=3 };

 public:

  LoopLimitNode( Compile* C, Node *init, Node *limit, Node *stride ) : Node(0,init,limit,stride) {

    // Put it on the Macro nodes list to optimize during macro nodes expansion.

    init_flags(Flag_is_macro);

    C->add_macro_node(this);

  }

  virtual int Opcode() const;

  virtual const Type *bottom_type() const { return TypeInt::INT; }

  virtual uint ideal_reg() const { return Op_RegI; }

  virtual const Type* Value(PhaseGVN* phase) const;

  virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);

  virtual Node* Identity(PhaseGVN* phase);

};

// Support for strip mining

class OuterStripMinedLoopNode : public LoopNode {

private:

  CountedLoopNode* inner_loop() const;

public:

  OuterStripMinedLoopNode(Compile* C, Node *entry, Node *backedge)

    : LoopNode(entry, backedge) {

    init_class_id(Class_OuterStripMinedLoop);

    init_flags(Flag_is_macro);

    C->add_macro_node(this);

  }

  virtual int Opcode() const;

  virtual IfTrueNode* outer_loop_tail() const;

  virtual OuterStripMinedLoopEndNode* outer_loop_end() const;

  virtual IfFalseNode* outer_loop_exit() const;

  virtual SafePointNode* outer_safepoint() const;

  void adjust_strip_mined_loop(PhaseIterGVN* igvn);

};

class OuterStripMinedLoopEndNode : public IfNode {

public:

  OuterStripMinedLoopEndNode(Node *control, Node *test, float prob, float cnt)

    : IfNode(control, test, prob, cnt) {

    init_class_id(Class_OuterStripMinedLoopEnd);

  }

  virtual int Opcode() const;

  virtual const Type* Value(PhaseGVN* phase) const;

  virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);

};

// -----------------------------IdealLoopTree----------------------------------

class IdealLoopTree : public ResourceObj {

public:

  IdealLoopTree *_parent;       // Parent in loop tree

  IdealLoopTree *_next;         // Next sibling in loop tree

  IdealLoopTree *_child;        // First child in loop tree

  // The head-tail backedge defines the loop.

  // If tail is NULL then this loop has multiple backedges as part of the

  // same loop.  During cleanup I'll peel off the multiple backedges; merge

  // them at the loop bottom and flow 1 real backedge into the loop.

  Node *_head;                  // Head of loop

  Node *_tail;                  // Tail of loop

  inline Node *tail();          // Handle lazy update of _tail field

  PhaseIdealLoop* _phase;

  int _local_loop_unroll_limit;

  int _local_loop_unroll_factor;

  Node_List _body;              // Loop body for inner loops

  uint8_t _nest;                // Nesting depth

  uint8_t _irreducible:1,       // True if irreducible

          _has_call:1,          // True if has call safepoint

          _has_sfpt:1,          // True if has non-call safepoint

          _rce_candidate:1;     // True if candidate for range check elimination

  Node_List* _safepts;          // List of safepoints in this loop

  Node_List* _required_safept;  // A inner loop cannot delete these safepts;

  bool  _allow_optimizations;   // Allow loop optimizations

  IdealLoopTree( PhaseIdealLoop* phase, Node *head, Node *tail )

    : _parent(0), _next(0), _child(0),

      _head(head), _tail(tail),

      _phase(phase),

      _safepts(NULL),

      _required_safept(NULL),

  { }

  // Is 'l' a member of 'this'?

  bool is_member(const IdealLoopTree *l) const; // Test for nested membership

  // Set loop nesting depth.  Accumulate has_call bits.

  int set_nest( uint depth );

  // Split out multiple fall-in edges from the loop header.  Move them to a

  // private RegionNode before the loop.  This becomes the loop landing pad.

  void split_fall_in( PhaseIdealLoop *phase, int fall_in_cnt );

  // Split out the outermost loop from this shared header.

  void split_outer_loop( PhaseIdealLoop *phase );