/*

 * Copyright (c) 1998, 2015, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#ifndef SHARE_VM_OPTO_LOOPNODE_HPP

#define SHARE_VM_OPTO_LOOPNODE_HPP

#include "opto/cfgnode.hpp"

#include "opto/multnode.hpp"

#include "opto/phaseX.hpp"

#include "opto/subnode.hpp"

#include "opto/type.hpp"

class CmpNode;

class CountedLoopEndNode;

class CountedLoopNode;

class IdealLoopTree;

class LoopNode;

class Node;

class PhaseIdealLoop;

class CountedLoopReserveKit;

class VectorSet;

class Invariance;

struct small_cache;

//

//                  I D E A L I Z E D   L O O P S

//

// Idealized loops are the set of loops I perform more interesting

// transformations on, beyond simple hoisting.

//------------------------------LoopNode---------------------------------------

// Simple loop header.  Fall in path on left, loop-back path on right.

class LoopNode : public RegionNode {

  // Size is bigger to hold the flags.  However, the flags do not change

  // the semantics so it does not appear in the hash & cmp functions.

  virtual uint size_of() const { return sizeof(*this); }

protected:

  short _loop_flags;

  // Names for flag bitfields

  enum { Normal=0, Pre=1, Main=2, Post=3, PreMainPostFlagsMask=3,

         MainHasNoPreLoop=4,

         HasExactTripCount=8,

         InnerLoop=16,

         PartialPeelLoop=32,

         PartialPeelFailed=64,

         HasReductions=128,

         WasSlpAnalyzed=256,

         PassedSlpAnalysis=512,

         DoUnrollOnly=1024,

         VectorizedLoop=2048,

         HasAtomicPostLoop=4096 };

  char _unswitch_count;

  enum { _unswitch_max=3 };

public:

  // Names for edge indices

  enum { Self=0, EntryControl, LoopBackControl };

  int is_inner_loop() const { return _loop_flags & InnerLoop; }

  void set_inner_loop() { _loop_flags |= InnerLoop; }

  int is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; }

  void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; }

  int partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; }

  void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; }

  void mark_has_reductions() { _loop_flags |= HasReductions; }

  void mark_was_slp() { _loop_flags |= WasSlpAnalyzed; }

  void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; }

  void mark_do_unroll_only() { _loop_flags |= DoUnrollOnly; }

  void mark_loop_vectorized() { _loop_flags |= VectorizedLoop; }

  void mark_has_atomic_post_loop() { _loop_flags |= HasAtomicPostLoop; }

  int unswitch_max() { return _unswitch_max; }

  int unswitch_count() { return _unswitch_count; }

  void set_unswitch_count(int val) {

    assert (val <= unswitch_max(), "too many unswitches");

    _unswitch_count = val;

  }

  LoopNode( Node *entry, Node *backedge ) : RegionNode(3), _loop_flags(0), _unswitch_count(0) {

    init_class_id(Class_Loop);

    init_req(EntryControl, entry);

    init_req(LoopBackControl, backedge);

  }

  virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);

  virtual int Opcode() const;

  bool can_be_counted_loop(PhaseTransform* phase) const {

    return req() == 3 && in(0) != NULL &&

      in(1) != NULL && phase->type(in(1)) != Type::TOP &&

      in(2) != NULL && phase->type(in(2)) != Type::TOP;

  }

  bool is_valid_counted_loop() const;

#ifndef PRODUCT

  virtual void dump_spec(outputStream *st) const;

#endif

};

//------------------------------Counted Loops----------------------------------

// Counted loops are all trip-counted loops, with exactly 1 trip-counter exit

// path (and maybe some other exit paths).  The trip-counter exit is always

// last in the loop.  The trip-counter have to stride by a constant;

// the exit value is also loop invariant.

// CountedLoopNodes and CountedLoopEndNodes come in matched pairs.  The

// CountedLoopNode has the incoming loop control and the loop-back-control

// which is always the IfTrue before the matching CountedLoopEndNode.  The

// CountedLoopEndNode has an incoming control (possibly not the

// CountedLoopNode if there is control flow in the loop), the post-increment

// trip-counter value, and the limit.  The trip-counter value is always of

// the form (Op old-trip-counter stride).  The old-trip-counter is produced

// by a Phi connected to the CountedLoopNode.  The stride is constant.

// The Op is any commutable opcode, including Add, Mul, Xor.  The

// CountedLoopEndNode also takes in the loop-invariant limit value.

// From a CountedLoopNode I can reach the matching CountedLoopEndNode via the

// loop-back control.  From CountedLoopEndNodes I can reach CountedLoopNodes

// via the old-trip-counter from the Op node.

//------------------------------CountedLoopNode--------------------------------

// CountedLoopNodes head simple counted loops.  CountedLoopNodes have as

// inputs the incoming loop-start control and the loop-back control, so they

// act like RegionNodes.  They also take in the initial trip counter, the

// loop-invariant stride and the loop-invariant limit value.  CountedLoopNodes

// produce a loop-body control and the trip counter value.  Since

// CountedLoopNodes behave like RegionNodes I still have a standard CFG model.

class CountedLoopNode : public LoopNode {

  // Size is bigger to hold _main_idx.  However, _main_idx does not change

  // the semantics so it does not appear in the hash & cmp functions.

  virtual uint size_of() const { return sizeof(*this); }

  // For Pre- and Post-loops during debugging ONLY, this holds the index of

  // the Main CountedLoop.  Used to assert that we understand the graph shape.

  node_idx_t _main_idx;

  // Known trip count calculated by compute_exact_trip_count()

  uint  _trip_count;

  // Expected trip count from profile data

  float _profile_trip_cnt;

  // Log2 of original loop bodies in unrolled loop

  int _unrolled_count_log2;

  // Node count prior to last unrolling - used to decide if

  // unroll,optimize,unroll,optimize,... is making progress

  int _node_count_before_unroll;

  // If slp analysis is performed we record the maximum

  // vector mapped unroll factor here

  int _slp_maximum_unroll_factor;

public:

  CountedLoopNode( Node *entry, Node *backedge )

    : LoopNode(entry, backedge), _main_idx(0), _trip_count(max_juint),

      _profile_trip_cnt(COUNT_UNKNOWN), _unrolled_count_log2(0),

      _node_count_before_unroll(0), _slp_maximum_unroll_factor(0) {

    init_class_id(Class_CountedLoop);

    // Initialize _trip_count to the largest possible value.

    // Will be reset (lower) if the loop's trip count is known.

  }

  virtual int Opcode() const;

  virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);

  Node *init_control() const { return in(EntryControl); }

  Node *back_control() const { return in(LoopBackControl); }

  CountedLoopEndNode *loopexit() const;

  Node *init_trip() const;

  Node *stride() const;

  int   stride_con() const;

  bool  stride_is_con() const;

  Node *limit() const;

  Node *incr() const;

  Node *phi() const;

  // Match increment with optional truncation

  static Node* match_incr_with_optional_truncation(Node* expr, Node** trunc1, Node** trunc2, const TypeInt** trunc_type);

  // A 'main' loop has a pre-loop and a post-loop.  The 'main' loop

  // can run short a few iterations and may start a few iterations in.

  // It will be RCE'd and unrolled and aligned.

  // A following 'post' loop will run any remaining iterations.  Used

  // during Range Check Elimination, the 'post' loop will do any final

  // iterations with full checks.  Also used by Loop Unrolling, where

  // the 'post' loop will do any epilog iterations needed.  Basically,

  // a 'post' loop can not profitably be further unrolled or RCE'd.

  // A preceding 'pre' loop will run at least 1 iteration (to do peeling),

  // it may do under-flow checks for RCE and may do alignment iterations

  // so the following main loop 'knows' that it is striding down cache

  // lines.

  // A 'main' loop that is ONLY unrolled or peeled, never RCE'd or

  // Aligned, may be missing it's pre-loop.

  int is_normal_loop   () const { return (_loop_flags&PreMainPostFlagsMask) == Normal; }

  int is_pre_loop      () const { return (_loop_flags&PreMainPostFlagsMask) == Pre;    }

  int is_main_loop     () const { return (_loop_flags&PreMainPostFlagsMask) == Main;   }

  int is_post_loop     () const { return (_loop_flags&PreMainPostFlagsMask) == Post;   }

  int is_reduction_loop() const { return (_loop_flags&HasReductions) == HasReductions; }

  int was_slp_analyzed () const { return (_loop_flags&WasSlpAnalyzed) == WasSlpAnalyzed; }

  int has_passed_slp   () const { return (_loop_flags&PassedSlpAnalysis) == PassedSlpAnalysis; }

  int do_unroll_only      () const { return (_loop_flags&DoUnrollOnly) == DoUnrollOnly; }

  int is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; }

  int is_vectorized_loop    () const { return (_loop_flags & VectorizedLoop) == VectorizedLoop; }

  int has_atomic_post_loop  () const { return (_loop_flags & HasAtomicPostLoop) == HasAtomicPostLoop; }

  void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; }

  int main_idx() const { return _main_idx; }

  void set_pre_loop  (CountedLoopNode *main) { assert(is_normal_loop(),""); _loop_flags |= Pre ; _main_idx = main->_idx; }

  void set_main_loop (                     ) { assert(is_normal_loop(),""); _loop_flags |= Main;                         }

  void set_post_loop (CountedLoopNode *main) { assert(is_normal_loop(),""); _loop_flags |= Post; _main_idx = main->_idx; }

  void set_normal_loop(                    ) { _loop_flags &= ~PreMainPostFlagsMask; }

  void set_trip_count(uint tc) { _trip_count = tc; }

  uint trip_count()            { return _trip_count; }

  bool has_exact_trip_count() const { return (_loop_flags & HasExactTripCount) != 0; }

  void set_exact_trip_count(uint tc) {

    _trip_count = tc;

    _loop_flags |= HasExactTripCount;

  }

  void set_nonexact_trip_count() {

    _loop_flags &= ~HasExactTripCount;

  }

  void set_notpassed_slp() {

    _loop_flags &= ~PassedSlpAnalysis;

  }

  void set_profile_trip_cnt(float ptc) { _profile_trip_cnt = ptc; }

  float profile_trip_cnt()             { return _profile_trip_cnt; }

  void double_unrolled_count() { _unrolled_count_log2++; }

  int  unrolled_count()        { return 1 << MIN2(_unrolled_count_log2, BitsPerInt-3); }

  void set_node_count_before_unroll(int ct)  { _node_count_before_unroll = ct; }

  int  node_count_before_unroll()            { return _node_count_before_unroll; }

  void set_slp_max_unroll(int unroll_factor) { _slp_maximum_unroll_factor = unroll_factor; }

  int  slp_max_unroll() const                { return _slp_maximum_unroll_factor; }

#ifndef PRODUCT

  virtual void dump_spec(outputStream *st) const;

#endif

};

//------------------------------CountedLoopEndNode-----------------------------

// CountedLoopEndNodes end simple trip counted loops.  They act much like

// IfNodes.

class CountedLoopEndNode : public IfNode {

public:

  enum { TestControl, TestValue };

  CountedLoopEndNode( Node *control, Node *test, float prob, float cnt )

    : IfNode( control, test, prob, cnt) {

    init_class_id(Class_CountedLoopEnd);

  }

  virtual int Opcode() const;

  Node *cmp_node() const            { return (in(TestValue)->req() >=2) ? in(TestValue)->in(1) : NULL; }

  Node *incr() const                { Node *tmp = cmp_node(); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }

  Node *limit() const               { Node *tmp = cmp_node(); return (tmp && tmp->req()==3) ? tmp->in(2) : NULL; }

  Node *stride() const              { Node *tmp = incr    (); return (tmp && tmp->req()==3) ? tmp->in(2) : NULL; }

  Node *init_trip() const           { Node *tmp = phi     (); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }

  int stride_con() const;

  bool stride_is_con() const        { Node *tmp = stride  (); return (tmp != NULL && tmp->is_Con()); }

  BoolTest::mask test_trip() const  { return in(TestValue)->as_Bool()->_test._test; }

  CountedLoopNode *loopnode() const {

    // The CountedLoopNode that goes with this CountedLoopEndNode may

    // have been optimized out by the IGVN so be cautious with the

    // pattern matching on the graph

      return NULL;

    }

    if (ln->is_CountedLoop() && ln->as_CountedLoop()->loopexit() == this) {

      return (CountedLoopNode*)ln;

    }

    return NULL;

  }

#ifndef PRODUCT

  virtual void dump_spec(outputStream *st) const;

#endif

};

inline CountedLoopEndNode *CountedLoopNode::loopexit() const {

  Node *bc = back_control();

  if( bc == NULL ) return NULL;

  Node *le = bc->in(0);

  if( le->Opcode() != Op_CountedLoopEnd )

    return NULL;

  return (CountedLoopEndNode*)le;

}

inline Node *CountedLoopNode::init_trip() const { return loopexit() ? loopexit()->init_trip() : NULL; }

inline Node *CountedLoopNode::stride() const { return loopexit() ? loopexit()->stride() : NULL; }

inline int CountedLoopNode::stride_con() const { return loopexit() ? loopexit()->stride_con() : 0; }

inline bool CountedLoopNode::stride_is_con() const { return loopexit() && loopexit()->stride_is_con(); }

inline Node *CountedLoopNode::limit() const { return loopexit() ? loopexit()->limit() : NULL; }

inline Node *CountedLoopNode::incr() const { return loopexit() ? loopexit()->incr() : NULL; }

inline Node *CountedLoopNode::phi() const { return loopexit() ? loopexit()->phi() : NULL; }

//------------------------------LoopLimitNode-----------------------------

// Counted Loop limit node which represents exact final iterator value:

// trip_count = (limit - init_trip + stride - 1)/stride

// final_value= trip_count * stride + init_trip.

// Use HW instructions to calculate it when it can overflow in integer.

// Note, final_value should fit into integer since counted loop has

// limit check: limit <= max_int-stride.

class LoopLimitNode : public Node {

  enum { Init=1, Limit=2, Stride=3 };

 public:

  LoopLimitNode( Compile* C, Node *init, Node *limit, Node *stride ) : Node(0,init,limit,stride) {

    // Put it on the Macro nodes list to optimize during macro nodes expansion.

    init_flags(Flag_is_macro);

    C->add_macro_node(this);

  }

  virtual int Opcode() const;

  virtual const Type *bottom_type() const { return TypeInt::INT; }

  virtual uint ideal_reg() const { return Op_RegI; }

  virtual const Type* Value(PhaseGVN* phase) const;

  virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);

  virtual Node* Identity(PhaseGVN* phase);

};

// -----------------------------IdealLoopTree----------------------------------

class IdealLoopTree : public ResourceObj {

public:

  IdealLoopTree *_parent;       // Parent in loop tree

  IdealLoopTree *_next;         // Next sibling in loop tree

  IdealLoopTree *_child;        // First child in loop tree

  // The head-tail backedge defines the loop.

  // If tail is NULL then this loop has multiple backedges as part of the

  // same loop.  During cleanup I'll peel off the multiple backedges; merge

  // them at the loop bottom and flow 1 real backedge into the loop.

  Node *_head;                  // Head of loop

  Node *_tail;                  // Tail of loop

  inline Node *tail();          // Handle lazy update of _tail field

  PhaseIdealLoop* _phase;

  int _local_loop_unroll_limit;

  int _local_loop_unroll_factor;

  Node_List _body;              // Loop body for inner loops

  uint8_t _nest;                // Nesting depth

  uint8_t _irreducible:1,       // True if irreducible

          _has_call:1,          // True if has call safepoint

          _has_sfpt:1,          // True if has non-call safepoint

          _rce_candidate:1;     // True if candidate for range check elimination

  Node_List* _safepts;          // List of safepoints in this loop

  Node_List* _required_safept;  // A inner loop cannot delete these safepts;

  bool  _allow_optimizations;   // Allow loop optimizations

  IdealLoopTree( PhaseIdealLoop* phase, Node *head, Node *tail )

    : _parent(0), _next(0), _child(0),

      _head(head), _tail(tail),

      _phase(phase),

      _safepts(NULL),

      _required_safept(NULL),

      _allow_optimizations(true),

      _nest(0), _irreducible(0), _has_call(0), _has_sfpt(0), _rce_candidate(0),

      _local_loop_unroll_limit(0), _local_loop_unroll_factor(0)

  { }

  // Is 'l' a member of 'this'?

  bool is_member(const IdealLoopTree *l) const; // Test for nested membership

  // Set loop nesting depth.  Accumulate has_call bits.

  int set_nest( uint depth );

  // Split out multiple fall-in edges from the loop header.  Move them to a

  // private RegionNode before the loop.  This becomes the loop landing pad.

  void split_fall_in( PhaseIdealLoop *phase, int fall_in_cnt );

  // Split out the outermost loop from this shared header.

  void split_outer_loop( PhaseIdealLoop *phase );

  // Merge all the backedges from the shared header into a private Region.

  // Feed that region as the one backedge to this loop.

  void merge_many_backedges( PhaseIdealLoop *phase );

  // Split shared headers and insert loop landing pads.

  // Insert a LoopNode to replace the RegionNode.

  // Returns TRUE if loop tree is structurally changed.

  bool beautify_loops( PhaseIdealLoop *phase );

  // Perform optimization to use the loop predicates for null checks and range checks.

  // Applies to any loop level (not just the innermost one)

  bool loop_predication( PhaseIdealLoop *phase);

  // Perform iteration-splitting on inner loops.  Split iterations to

  // avoid range checks or one-shot null checks.  Returns false if the

  // current round of loop opts should stop.

  bool iteration_split( PhaseIdealLoop *phase, Node_List &old_new );

  // Driver for various flavors of iteration splitting.  Returns false

  // if the current round of loop opts should stop.

  bool iteration_split_impl( PhaseIdealLoop *phase, Node_List &old_new );

  // Given dominators, try to find loops with calls that must always be

  // executed (call dominates loop tail).  These loops do not need non-call

  // safepoints (ncsfpt).

  void check_safepts(VectorSet &visited, Node_List &stack);

  // Allpaths backwards scan from loop tail, terminating each path at first safepoint

  // encountered.

  void allpaths_check_safepts(VectorSet &visited, Node_List &stack);

  // Remove safepoints from loop. Optionally keeping one.

  void remove_safepoints(PhaseIdealLoop* phase, bool keep_one);

  // Convert to counted loops where possible

  void counted_loop( PhaseIdealLoop *phase );

  // Check for Node being a loop-breaking test

  Node *is_loop_exit(Node *iff) const;

  // Returns true if ctrl is executed on every complete iteration

  bool dominates_backedge(Node* ctrl);

  // Remove simplistic dead code from loop body

  void DCE_loop_body();

  // Look for loop-exit tests with my 50/50 guesses from the Parsing stage.

  // Replace with a 1-in-10 exit guess.

  void adjust_loop_exit_prob( PhaseIdealLoop *phase );

  // Return TRUE or FALSE if the loop should never be RCE'd or aligned.

  // Useful for unrolling loops with NO array accesses.

  bool policy_peel_only( PhaseIdealLoop *phase ) const;

  // Return TRUE or FALSE if the loop should be unswitched -- clone

  // loop with an invariant test

  bool policy_unswitching( PhaseIdealLoop *phase ) const;

  // Micro-benchmark spamming.  Remove empty loops.

  bool policy_do_remove_empty_loop( PhaseIdealLoop *phase );

  // Convert one iteration loop into normal code.

  bool policy_do_one_iteration_loop( PhaseIdealLoop *phase );

  // Return TRUE or FALSE if the loop should be peeled or not.  Peel if we can

  // make some loop-invariant test (usually a null-check) happen before the

  // loop.

  bool policy_peeling( PhaseIdealLoop *phase ) const;

  // Return TRUE or FALSE if the loop should be maximally unrolled. Stash any

  // known trip count in the counted loop node.

  bool policy_maximally_unroll( PhaseIdealLoop *phase ) const;

  // Return TRUE or FALSE if the loop should be unrolled or not.  Unroll if

  // the loop is a CountedLoop and the body is small enough.

  bool policy_unroll(PhaseIdealLoop *phase);

  // Loop analyses to map to a maximal superword unrolling for vectorization.

  void policy_unroll_slp_analysis(CountedLoopNode *cl, PhaseIdealLoop *phase, int future_unroll_ct);

  // Return TRUE or FALSE if the loop should be range-check-eliminated.

  // Gather a list of IF tests that are dominated by iteration splitting;

  // also gather the end of the first split and the start of the 2nd split.

  bool policy_range_check( PhaseIdealLoop *phase ) const;

  // Return TRUE or FALSE if the loop should be cache-line aligned.

  // Gather the expression that does the alignment.  Note that only

  // one array base can be aligned in a loop (unless the VM guarantees

  // mutual alignment).  Note that if we vectorize short memory ops

  // into longer memory ops, we may want to increase alignment.

  bool policy_align( PhaseIdealLoop *phase ) const;

  // Return TRUE if "iff" is a range check.

  bool is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invariance& invar) const;

  // Compute loop exact trip count if possible