/*

 * Copyright (c) 1998, 2009, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

class CmpNode;

class CountedLoopEndNode;

class CountedLoopNode;

class IdealLoopTree;

class LoopNode;

class Node;

class PhaseIdealLoop;

class VectorSet;

class Invariance;

struct small_cache;

//

//                  I D E A L I Z E D   L O O P S

//

// Idealized loops are the set of loops I perform more interesting

// transformations on, beyond simple hoisting.

//------------------------------LoopNode---------------------------------------

// Simple loop header.  Fall in path on left, loop-back path on right.

class LoopNode : public RegionNode {

  // Size is bigger to hold the flags.  However, the flags do not change

  // the semantics so it does not appear in the hash & cmp functions.

  virtual uint size_of() const { return sizeof(*this); }

protected:

  short _loop_flags;

  // Names for flag bitfields

  enum { pre_post_main=0, inner_loop=8, partial_peel_loop=16, partial_peel_failed=32  };

  char _unswitch_count;

  enum { _unswitch_max=3 };

public:

  // Names for edge indices

  enum { Self=0, EntryControl, LoopBackControl };

  int is_inner_loop() const { return _loop_flags & inner_loop; }

  void set_inner_loop() { _loop_flags |= inner_loop; }

  int is_partial_peel_loop() const { return _loop_flags & partial_peel_loop; }

  void set_partial_peel_loop() { _loop_flags |= partial_peel_loop; }

  int partial_peel_has_failed() const { return _loop_flags & partial_peel_failed; }

  void mark_partial_peel_failed() { _loop_flags |= partial_peel_failed; }

  int unswitch_max() { return _unswitch_max; }

  int unswitch_count() { return _unswitch_count; }

  void set_unswitch_count(int val) {

    assert (val <= unswitch_max(), "too many unswitches");

    _unswitch_count = val;

  }

  LoopNode( Node *entry, Node *backedge ) : RegionNode(3), _loop_flags(0), _unswitch_count(0) {

    init_class_id(Class_Loop);

    init_req(EntryControl, entry);

    init_req(LoopBackControl, backedge);

  }

  virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);

  virtual int Opcode() const;

  bool can_be_counted_loop(PhaseTransform* phase) const {

    return req() == 3 && in(0) != NULL &&

      in(1) != NULL && phase->type(in(1)) != Type::TOP &&

      in(2) != NULL && phase->type(in(2)) != Type::TOP;

  }

#ifndef PRODUCT

  virtual void dump_spec(outputStream *st) const;

#endif

};

//------------------------------Counted Loops----------------------------------

// Counted loops are all trip-counted loops, with exactly 1 trip-counter exit

// path (and maybe some other exit paths).  The trip-counter exit is always

// last in the loop.  The trip-counter does not have to stride by a constant,

// but it does have to stride by a loop-invariant amount; the exit value is

// also loop invariant.

// CountedLoopNodes and CountedLoopEndNodes come in matched pairs.  The

// CountedLoopNode has the incoming loop control and the loop-back-control

// which is always the IfTrue before the matching CountedLoopEndNode.  The

// CountedLoopEndNode has an incoming control (possibly not the

// CountedLoopNode if there is control flow in the loop), the post-increment

// trip-counter value, and the limit.  The trip-counter value is always of

// the form (Op old-trip-counter stride).  The old-trip-counter is produced

// by a Phi connected to the CountedLoopNode.  The stride is loop invariant.

// The Op is any commutable opcode, including Add, Mul, Xor.  The

// CountedLoopEndNode also takes in the loop-invariant limit value.

// From a CountedLoopNode I can reach the matching CountedLoopEndNode via the

// loop-back control.  From CountedLoopEndNodes I can reach CountedLoopNodes

// via the old-trip-counter from the Op node.

//------------------------------CountedLoopNode--------------------------------

// CountedLoopNodes head simple counted loops.  CountedLoopNodes have as

// inputs the incoming loop-start control and the loop-back control, so they

// act like RegionNodes.  They also take in the initial trip counter, the

// loop-invariant stride and the loop-invariant limit value.  CountedLoopNodes

// produce a loop-body control and the trip counter value.  Since

// CountedLoopNodes behave like RegionNodes I still have a standard CFG model.

class CountedLoopNode : public LoopNode {

  // Size is bigger to hold _main_idx.  However, _main_idx does not change

  // the semantics so it does not appear in the hash & cmp functions.

  virtual uint size_of() const { return sizeof(*this); }

  // For Pre- and Post-loops during debugging ONLY, this holds the index of

  // the Main CountedLoop.  Used to assert that we understand the graph shape.

  node_idx_t _main_idx;

  // Known trip count calculated by policy_maximally_unroll

  int   _trip_count;

  // Expected trip count from profile data

  float _profile_trip_cnt;

  // Log2 of original loop bodies in unrolled loop

  int _unrolled_count_log2;

  // Node count prior to last unrolling - used to decide if

  // unroll,optimize,unroll,optimize,... is making progress

  int _node_count_before_unroll;

public:

  CountedLoopNode( Node *entry, Node *backedge )

    : LoopNode(entry, backedge), _trip_count(max_jint),

      _profile_trip_cnt(COUNT_UNKNOWN), _unrolled_count_log2(0),

      _node_count_before_unroll(0) {

    init_class_id(Class_CountedLoop);

    // Initialize _trip_count to the largest possible value.

    // Will be reset (lower) if the loop's trip count is known.

  }

  virtual int Opcode() const;

  virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);

  Node *init_control() const { return in(EntryControl); }

  Node *back_control() const { return in(LoopBackControl); }

  CountedLoopEndNode *loopexit() const;

  Node *init_trip() const;

  Node *stride() const;

  int   stride_con() const;

  bool  stride_is_con() const;

  Node *limit() const;

  Node *incr() const;

  Node *phi() const;

  // Match increment with optional truncation

  static Node* match_incr_with_optional_truncation(Node* expr, Node** trunc1, Node** trunc2, const TypeInt** trunc_type);

  // A 'main' loop has a pre-loop and a post-loop.  The 'main' loop

  // can run short a few iterations and may start a few iterations in.

  // It will be RCE'd and unrolled and aligned.

  // A following 'post' loop will run any remaining iterations.  Used

  // during Range Check Elimination, the 'post' loop will do any final

  // iterations with full checks.  Also used by Loop Unrolling, where

  // the 'post' loop will do any epilog iterations needed.  Basically,

  // a 'post' loop can not profitably be further unrolled or RCE'd.

  // A preceding 'pre' loop will run at least 1 iteration (to do peeling),

  // it may do under-flow checks for RCE and may do alignment iterations

  // so the following main loop 'knows' that it is striding down cache

  // lines.

  // A 'main' loop that is ONLY unrolled or peeled, never RCE'd or

  // Aligned, may be missing it's pre-loop.

  enum { Normal=0, Pre=1, Main=2, Post=3, PrePostFlagsMask=3, Main_Has_No_Pre_Loop=4 };

  int is_normal_loop() const { return (_loop_flags&PrePostFlagsMask) == Normal; }

  int is_pre_loop   () const { return (_loop_flags&PrePostFlagsMask) == Pre;    }

  int is_main_loop  () const { return (_loop_flags&PrePostFlagsMask) == Main;   }

  int is_post_loop  () const { return (_loop_flags&PrePostFlagsMask) == Post;   }

  int is_main_no_pre_loop() const { return _loop_flags & Main_Has_No_Pre_Loop; }

  void set_main_no_pre_loop() { _loop_flags |= Main_Has_No_Pre_Loop; }

  int main_idx() const { return _main_idx; }

  void set_pre_loop  (CountedLoopNode *main) { assert(is_normal_loop(),""); _loop_flags |= Pre ; _main_idx = main->_idx; }

  void set_main_loop (                     ) { assert(is_normal_loop(),""); _loop_flags |= Main;                         }

  void set_post_loop (CountedLoopNode *main) { assert(is_normal_loop(),""); _loop_flags |= Post; _main_idx = main->_idx; }

  void set_normal_loop(                    ) { _loop_flags &= ~PrePostFlagsMask; }

  void set_trip_count(int tc) { _trip_count = tc; }

  int trip_count()            { return _trip_count; }

  void set_profile_trip_cnt(float ptc) { _profile_trip_cnt = ptc; }

  float profile_trip_cnt()             { return _profile_trip_cnt; }

  void double_unrolled_count() { _unrolled_count_log2++; }

  int  unrolled_count()        { return 1 << MIN2(_unrolled_count_log2, BitsPerInt-3); }

  void set_node_count_before_unroll(int ct) { _node_count_before_unroll = ct; }

  int  node_count_before_unroll()           { return _node_count_before_unroll; }

#ifndef PRODUCT

  virtual void dump_spec(outputStream *st) const;

#endif

};

//------------------------------CountedLoopEndNode-----------------------------

// CountedLoopEndNodes end simple trip counted loops.  They act much like

// IfNodes.

class CountedLoopEndNode : public IfNode {

public:

  enum { TestControl, TestValue };

  CountedLoopEndNode( Node *control, Node *test, float prob, float cnt )

    : IfNode( control, test, prob, cnt) {

    init_class_id(Class_CountedLoopEnd);

  }

  virtual int Opcode() const;

  Node *cmp_node() const            { return (in(TestValue)->req() >=2) ? in(TestValue)->in(1) : NULL; }

  Node *incr() const                { Node *tmp = cmp_node(); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }

  Node *limit() const               { Node *tmp = cmp_node(); return (tmp && tmp->req()==3) ? tmp->in(2) : NULL; }

  Node *stride() const              { Node *tmp = incr    (); return (tmp && tmp->req()==3) ? tmp->in(2) : NULL; }

  Node *phi() const                 { Node *tmp = incr    (); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }

  Node *init_trip() const           { Node *tmp = phi     (); return (tmp && tmp->req()==3) ? tmp->in(1) : NULL; }

  int stride_con() const;

  bool stride_is_con() const        { Node *tmp = stride  (); return (tmp != NULL && tmp->is_Con()); }

  BoolTest::mask test_trip() const  { return in(TestValue)->as_Bool()->_test._test; }

  CountedLoopNode *loopnode() const {

    Node *ln = phi()->in(0);

    assert( ln->Opcode() == Op_CountedLoop, "malformed loop" );

    return (CountedLoopNode*)ln; }

#ifndef PRODUCT

  virtual void dump_spec(outputStream *st) const;

#endif

};

inline CountedLoopEndNode *CountedLoopNode::loopexit() const {

  Node *bc = back_control();

  if( bc == NULL ) return NULL;

  Node *le = bc->in(0);

  if( le->Opcode() != Op_CountedLoopEnd )

    return NULL;

  return (CountedLoopEndNode*)le;

}

inline Node *CountedLoopNode::init_trip() const { return loopexit() ? loopexit()->init_trip() : NULL; }

inline Node *CountedLoopNode::stride() const { return loopexit() ? loopexit()->stride() : NULL; }

inline int CountedLoopNode::stride_con() const { return loopexit() ? loopexit()->stride_con() : 0; }

inline bool CountedLoopNode::stride_is_con() const { return loopexit() && loopexit()->stride_is_con(); }

inline Node *CountedLoopNode::limit() const { return loopexit() ? loopexit()->limit() : NULL; }

inline Node *CountedLoopNode::incr() const { return loopexit() ? loopexit()->incr() : NULL; }

inline Node *CountedLoopNode::phi() const { return loopexit() ? loopexit()->phi() : NULL; }

// -----------------------------IdealLoopTree----------------------------------

class IdealLoopTree : public ResourceObj {

public:

  IdealLoopTree *_parent;       // Parent in loop tree

  IdealLoopTree *_next;         // Next sibling in loop tree

  IdealLoopTree *_child;        // First child in loop tree

  // The head-tail backedge defines the loop.

  // If tail is NULL then this loop has multiple backedges as part of the

  // same loop.  During cleanup I'll peel off the multiple backedges; merge

  // them at the loop bottom and flow 1 real backedge into the loop.

  Node *_head;                  // Head of loop

  Node *_tail;                  // Tail of loop

  inline Node *tail();          // Handle lazy update of _tail field

  PhaseIdealLoop* _phase;

  Node_List _body;              // Loop body for inner loops

  uint8 _nest;                  // Nesting depth

  uint8 _irreducible:1,         // True if irreducible

        _has_call:1,            // True if has call safepoint

        _has_sfpt:1,            // True if has non-call safepoint

        _rce_candidate:1;       // True if candidate for range check elimination

  Node_List* _required_safept;  // A inner loop cannot delete these safepts;

  bool  _allow_optimizations;   // Allow loop optimizations

  IdealLoopTree( PhaseIdealLoop* phase, Node *head, Node *tail )

    : _parent(0), _next(0), _child(0),

      _head(head), _tail(tail),

      _phase(phase),

      _required_safept(NULL),

      _allow_optimizations(true),

      _nest(0), _irreducible(0), _has_call(0), _has_sfpt(0), _rce_candidate(0)

  { }

  // Is 'l' a member of 'this'?

  int is_member( const IdealLoopTree *l ) const; // Test for nested membership

  // Set loop nesting depth.  Accumulate has_call bits.

  int set_nest( uint depth );

  // Split out multiple fall-in edges from the loop header.  Move them to a

  // private RegionNode before the loop.  This becomes the loop landing pad.

  void split_fall_in( PhaseIdealLoop *phase, int fall_in_cnt );

  // Split out the outermost loop from this shared header.

  void split_outer_loop( PhaseIdealLoop *phase );

  // Merge all the backedges from the shared header into a private Region.

  // Feed that region as the one backedge to this loop.

  void merge_many_backedges( PhaseIdealLoop *phase );

  // Split shared headers and insert loop landing pads.

  // Insert a LoopNode to replace the RegionNode.

  // Returns TRUE if loop tree is structurally changed.

  bool beautify_loops( PhaseIdealLoop *phase );

  // Perform optimization to use the loop predicates for null checks and range checks.

  // Applies to any loop level (not just the innermost one)

  bool loop_predication( PhaseIdealLoop *phase);

  // Perform iteration-splitting on inner loops.  Split iterations to

  // avoid range checks or one-shot null checks.  Returns false if the

  // current round of loop opts should stop.

  bool iteration_split( PhaseIdealLoop *phase, Node_List &old_new );

  // Driver for various flavors of iteration splitting.  Returns false

  // if the current round of loop opts should stop.

  bool iteration_split_impl( PhaseIdealLoop *phase, Node_List &old_new );

  // Given dominators, try to find loops with calls that must always be

  // executed (call dominates loop tail).  These loops do not need non-call

  // safepoints (ncsfpt).

  void check_safepts(VectorSet &visited, Node_List &stack);

  // Allpaths backwards scan from loop tail, terminating each path at first safepoint

  // encountered.

  void allpaths_check_safepts(VectorSet &visited, Node_List &stack);

  // Convert to counted loops where possible

  void counted_loop( PhaseIdealLoop *phase );

  // Check for Node being a loop-breaking test

  Node *is_loop_exit(Node *iff) const;

  // Returns true if ctrl is executed on every complete iteration

  bool dominates_backedge(Node* ctrl);

  // Remove simplistic dead code from loop body

  void DCE_loop_body();

  // Look for loop-exit tests with my 50/50 guesses from the Parsing stage.

  // Replace with a 1-in-10 exit guess.

  void adjust_loop_exit_prob( PhaseIdealLoop *phase );

  // Return TRUE or FALSE if the loop should never be RCE'd or aligned.

  // Useful for unrolling loops with NO array accesses.

  bool policy_peel_only( PhaseIdealLoop *phase ) const;

  // Return TRUE or FALSE if the loop should be unswitched -- clone

  // loop with an invariant test

  bool policy_unswitching( PhaseIdealLoop *phase ) const;

  // Micro-benchmark spamming.  Remove empty loops.

  bool policy_do_remove_empty_loop( PhaseIdealLoop *phase );

  // Return TRUE or FALSE if the loop should be peeled or not.  Peel if we can

  // make some loop-invariant test (usually a null-check) happen before the

  // loop.

  bool policy_peeling( PhaseIdealLoop *phase ) const;

  // Return TRUE or FALSE if the loop should be maximally unrolled. Stash any

  // known trip count in the counted loop node.

  bool policy_maximally_unroll( PhaseIdealLoop *phase ) const;

  // Return TRUE or FALSE if the loop should be unrolled or not.  Unroll if

  // the loop is a CountedLoop and the body is small enough.

  bool policy_unroll( PhaseIdealLoop *phase ) const;

  // Return TRUE or FALSE if the loop should be range-check-eliminated.

  // Gather a list of IF tests that are dominated by iteration splitting;

  // also gather the end of the first split and the start of the 2nd split.

  bool policy_range_check( PhaseIdealLoop *phase ) const;

  // Return TRUE or FALSE if the loop should be cache-line aligned.

  // Gather the expression that does the alignment.  Note that only

  // one array base can be aligned in a loop (unless the VM guarantees

  // mutual alignment).  Note that if we vectorize short memory ops

  // into longer memory ops, we may want to increase alignment.

  bool policy_align( PhaseIdealLoop *phase ) const;

  // Return TRUE if "iff" is a range check.

  bool is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invariance& invar) const;

  // Compute loop trip count from profile data

  void compute_profile_trip_cnt( PhaseIdealLoop *phase );

  // Reassociate invariant expressions.

  void reassociate_invariants(PhaseIdealLoop *phase);

  // Reassociate invariant add and subtract expressions.

  Node* reassociate_add_sub(Node* n1, PhaseIdealLoop *phase);

  // Return nonzero index of invariant operand if invariant and variant

  // are combined with an Add or Sub. Helper for reassociate_invariants.

  int is_invariant_addition(Node* n, PhaseIdealLoop *phase);

  // Return true if n is invariant

  bool is_invariant(Node* n) const;

  // Put loop body on igvn work list

  void record_for_igvn();

  bool is_loop()    { return !_irreducible && _tail && !_tail->is_top(); }

  bool is_inner()   { return is_loop() && _child == NULL; }

  bool is_counted() { return is_loop() && _head != NULL && _head->is_CountedLoop(); }

#ifndef PRODUCT

  void dump_head( ) const;      // Dump loop head only

  void dump() const;            // Dump this loop recursively

  void verify_tree(IdealLoopTree *loop, const IdealLoopTree *parent) const;

#endif

};

// -----------------------------PhaseIdealLoop---------------------------------

// Computes the mapping from Nodes to IdealLoopTrees.  Organizes IdealLoopTrees into a

// loop tree.  Drives the loop-based transformations on the ideal graph.

class PhaseIdealLoop : public PhaseTransform {

  friend class IdealLoopTree;

  friend class SuperWord;

  // Pre-computed def-use info

  PhaseIterGVN &_igvn;

  // Head of loop tree

  IdealLoopTree *_ltree_root;

  // Array of pre-order numbers, plus post-visited bit.

  // ZERO for not pre-visited.  EVEN for pre-visited but not post-visited.

  // ODD for post-visited.  Other bits are the pre-order number.

  uint *_preorders;

  uint _max_preorder;

  const PhaseIdealLoop* _verify_me;

  bool _verify_only;

  // Allocate _preorders[] array

  void allocate_preorders() {

    _max_preorder = C->unique()+8;

    _preorders = NEW_RESOURCE_ARRAY(uint, _max_preorder);

    memset(_preorders, 0, sizeof(uint) * _max_preorder);

  }

  // Allocate _preorders[] array

  void reallocate_preorders() {

    if ( _max_preorder < C->unique() ) {

      _preorders = REALLOC_RESOURCE_ARRAY(uint, _preorders, _max_preorder, C->unique());

      _max_preorder = C->unique();

    }

    memset(_preorders, 0, sizeof(uint) * _max_preorder);

  }

  // Check to grow _preorders[] array for the case when build_loop_tree_impl()

  // adds new nodes.

  void check_grow_preorders( ) {

    if ( _max_preorder < C->unique() ) {

      uint newsize = _max_preorder<<1;  // double size of array

      _preorders = REALLOC_RESOURCE_ARRAY(uint, _preorders, _max_preorder, newsize);

      memset(&_preorders[_max_preorder],0,sizeof(uint)*(newsize-_max_preorder));

      _max_preorder = newsize;

    }

  }

  // Check for pre-visited.  Zero for NOT visited; non-zero for visited.

  int is_visited( Node *n ) const { return _preorders[n->_idx]; }

  // Pre-order numbers are written to the Nodes array as low-bit-set values.

  void set_preorder_visited( Node *n, int pre_order ) {

    assert( !is_visited( n ), "already set" );

    _preorders[n->_idx] = (pre_order<<1);

  };

  // Return pre-order number.

  int get_preorder( Node *n ) const { assert( is_visited(n), "" ); return _preorders[n->_idx]>>1; }

  // Check for being post-visited.

  // Should be previsited already (checked with assert(is_visited(n))).

  int is_postvisited( Node *n ) const { assert( is_visited(n), "" ); return _preorders[n->_idx]&1; }

  // Mark as post visited

  void set_postvisited( Node *n ) { assert( !is_postvisited( n ), "" ); _preorders[n->_idx] |= 1; }

  // Set/get control node out.  Set lower bit to distinguish from IdealLoopTree

  // Returns true if "n" is a data node, false if it's a control node.

  bool has_ctrl( Node *n ) const { return ((intptr_t)_nodes[n->_idx]) & 1; }

  // clear out dead code after build_loop_late

  Node_List _deadlist;

  // Support for faster execution of get_late_ctrl()/dom_lca()

  // when a node has many uses and dominator depth is deep.

  Node_Array _dom_lca_tags;

  void   init_dom_lca_tags();

  void   clear_dom_lca_tags();

  // Helper for debugging bad dominance relationships

  bool verify_dominance(Node* n, Node* use, Node* LCA, Node* early);

  Node* compute_lca_of_uses(Node* n, Node* early, bool verify = false);

  // Inline wrapper for frequent cases:

  // 1) only one use

  // 2) a use is the same as the current LCA passed as 'n1'

  Node *dom_lca_for_get_late_ctrl( Node *lca, Node *n, Node *tag ) {

    assert( n->is_CFG(), "" );

    // Fast-path NULL lca

    if( lca != NULL && lca != n ) {

      assert( lca->is_CFG(), "" );