# HG changeset patch # User redestad # Date 1573741475 -3600 # Node ID 95a99e617f28c8b0271b599c7e4e1902542de9ec # Parent 355f4f42dda5b1d0167e0f14d1a1cb6f0f070226 8234003: Improve IndexSet iteration Reviewed-by: neliasso, thartmann diff -r 355f4f42dda5 -r 95a99e617f28 src/hotspot/share/opto/chaitin.cpp --- a/src/hotspot/share/opto/chaitin.cpp Thu Nov 14 10:55:46 2019 +0100 +++ b/src/hotspot/share/opto/chaitin.cpp Thu Nov 14 15:24:35 2019 +0100 @@ -1169,7 +1169,7 @@ lrgs(lo)._next = _simplified; _simplified = lo; // If this guy is "at risk" then mark his current neighbors - if( lrgs(lo)._at_risk ) { + if (lrgs(lo)._at_risk && !_ifg->neighbors(lo)->is_empty()) { IndexSetIterator elements(_ifg->neighbors(lo)); uint datum; while ((datum = elements.next()) != 0) { @@ -1178,7 +1178,10 @@ } // Yank this guy from the IFG. - IndexSet *adj = _ifg->remove_node( lo ); + IndexSet *adj = _ifg->remove_node(lo); + if (adj->is_empty()) { + continue; + } // If any neighbors' degrees fall below their number of // allowed registers, then put that neighbor on the low degree @@ -1202,8 +1205,11 @@ // Pull from hi-degree list uint prev = n->_prev; uint next = n->_next; - if (prev) lrgs(prev)._next = next; - else _hi_degree = next; + if (prev) { + lrgs(prev)._next = next; + } else { + _hi_degree = next; + } lrgs(next)._prev = prev; n->_next = _lo_degree; _lo_degree = neighbor; @@ -1314,7 +1320,7 @@ // Check for "at_risk" LRG's uint risk_lrg = _lrg_map.find(lrg._risk_bias); - if( risk_lrg != 0 ) { + if (risk_lrg != 0 && !_ifg->neighbors(risk_lrg)->is_empty()) { // Walk the colored neighbors of the "at_risk" candidate // Choose a color which is both legal and already taken by a neighbor // of the "at_risk" candidate in order to improve the chances of the @@ -1330,7 +1336,7 @@ } uint copy_lrg = _lrg_map.find(lrg._copy_bias); - if( copy_lrg != 0 ) { + if (copy_lrg != 0) { // If he has a color, if(!_ifg->_yanked->test(copy_lrg)) { OptoReg::Name reg = lrgs(copy_lrg).reg(); @@ -1432,41 +1438,43 @@ // Remove neighbor colors IndexSet *s = _ifg->neighbors(lidx); + debug_only(RegMask orig_mask = lrg->mask();) - debug_only(RegMask orig_mask = lrg->mask();) - IndexSetIterator elements(s); - uint neighbor; - while ((neighbor = elements.next()) != 0) { - // Note that neighbor might be a spill_reg. In this case, exclusion - // of its color will be a no-op, since the spill_reg chunk is in outer - // space. Also, if neighbor is in a different chunk, this exclusion - // will be a no-op. (Later on, if lrg runs out of possible colors in - // its chunk, a new chunk of color may be tried, in which case - // examination of neighbors is started again, at retry_next_chunk.) - LRG &nlrg = lrgs(neighbor); - OptoReg::Name nreg = nlrg.reg(); - // Only subtract masks in the same chunk - if( nreg >= chunk && nreg < chunk + RegMask::CHUNK_SIZE ) { + if (!s->is_empty()) { + IndexSetIterator elements(s); + uint neighbor; + while ((neighbor = elements.next()) != 0) { + // Note that neighbor might be a spill_reg. In this case, exclusion + // of its color will be a no-op, since the spill_reg chunk is in outer + // space. Also, if neighbor is in a different chunk, this exclusion + // will be a no-op. (Later on, if lrg runs out of possible colors in + // its chunk, a new chunk of color may be tried, in which case + // examination of neighbors is started again, at retry_next_chunk.) + LRG &nlrg = lrgs(neighbor); + OptoReg::Name nreg = nlrg.reg(); + // Only subtract masks in the same chunk + if (nreg >= chunk && nreg < chunk + RegMask::CHUNK_SIZE) { #ifndef PRODUCT - uint size = lrg->mask().Size(); - RegMask rm = lrg->mask(); + uint size = lrg->mask().Size(); + RegMask rm = lrg->mask(); #endif - lrg->SUBTRACT(nlrg.mask()); + lrg->SUBTRACT(nlrg.mask()); #ifndef PRODUCT - if (trace_spilling() && lrg->mask().Size() != size) { - ttyLocker ttyl; - tty->print("L%d ", lidx); - rm.dump(); - tty->print(" intersected L%d ", neighbor); - nlrg.mask().dump(); - tty->print(" removed "); - rm.SUBTRACT(lrg->mask()); - rm.dump(); - tty->print(" leaving "); - lrg->mask().dump(); - tty->cr(); + if (trace_spilling() && lrg->mask().Size() != size) { + ttyLocker ttyl; + tty->print("L%d ", lidx); + rm.dump(); + tty->print(" intersected L%d ", neighbor); + nlrg.mask().dump(); + tty->print(" removed "); + rm.SUBTRACT(lrg->mask()); + rm.dump(); + tty->print(" leaving "); + lrg->mask().dump(); + tty->cr(); + } +#endif } -#endif } } //assert(is_allstack == lrg->mask().is_AllStack(), "nbrs must not change AllStackedness"); @@ -1827,7 +1835,7 @@ // Found a safepoint? JVMState *jvms = n->jvms(); - if( jvms ) { + if (jvms && !liveout.is_empty()) { // Now scan for a live derived pointer IndexSetIterator elements(&liveout); uint neighbor; diff -r 355f4f42dda5 -r 95a99e617f28 src/hotspot/share/opto/coalesce.cpp --- a/src/hotspot/share/opto/coalesce.cpp Thu Nov 14 10:55:46 2019 +0100 +++ b/src/hotspot/share/opto/coalesce.cpp Thu Nov 14 15:24:35 2019 +0100 @@ -602,29 +602,42 @@ // Some original neighbors of lr1 might have gone away // because the constrained register mask prevented them. // Remove lr1 from such neighbors. - IndexSetIterator one(n_lr1); - uint neighbor; + uint neighbor = 0; LRG &lrg1 = lrgs(lr1); - while ((neighbor = one.next()) != 0) - if( !_ulr.member(neighbor) ) - if( _phc._ifg->neighbors(neighbor)->remove(lr1) ) - lrgs(neighbor).inc_degree( -lrg1.compute_degree(lrgs(neighbor)) ); + if (!n_lr1->is_empty()) { + IndexSetIterator one(n_lr1); + while ((neighbor = one.next()) != 0) { + if (!_ulr.member(neighbor)) { + if (_phc._ifg->neighbors(neighbor)->remove(lr1)) { + lrgs(neighbor).inc_degree(-lrg1.compute_degree(lrgs(neighbor))); + } + } + } + } // lr2 is now called (coalesced into) lr1. // Remove lr2 from the IFG. - IndexSetIterator two(n_lr2); LRG &lrg2 = lrgs(lr2); - while ((neighbor = two.next()) != 0) - if( _phc._ifg->neighbors(neighbor)->remove(lr2) ) - lrgs(neighbor).inc_degree( -lrg2.compute_degree(lrgs(neighbor)) ); + if (!n_lr2->is_empty()) { + IndexSetIterator two(n_lr2); + while ((neighbor = two.next()) != 0) { + if (_phc._ifg->neighbors(neighbor)->remove(lr2)) { + lrgs(neighbor).inc_degree(-lrg2.compute_degree(lrgs(neighbor))); + } + } + } // Some neighbors of intermediate copies now interfere with the // combined live range. - IndexSetIterator three(&_ulr); - while ((neighbor = three.next()) != 0) - if( _phc._ifg->neighbors(neighbor)->insert(lr1) ) - lrgs(neighbor).inc_degree( lrg1.compute_degree(lrgs(neighbor)) ); + if (!_ulr.is_empty()) { + IndexSetIterator three(&_ulr); + while ((neighbor = three.next()) != 0) { + if (_phc._ifg->neighbors(neighbor)->insert(lr1)) { + lrgs(neighbor).inc_degree(lrg1.compute_degree(lrgs(neighbor))); + } + } + } } static void record_bias( const PhaseIFG *ifg, int lr1, int lr2 ) { diff -r 355f4f42dda5 -r 95a99e617f28 src/hotspot/share/opto/ifg.cpp --- a/src/hotspot/share/opto/ifg.cpp Thu Nov 14 10:55:46 2019 +0100 +++ b/src/hotspot/share/opto/ifg.cpp Thu Nov 14 15:24:35 2019 +0100 @@ -80,11 +80,13 @@ assert( !_is_square, "only on triangular" ); // Simple transpose - for( uint i = 0; i < _maxlrg; i++ ) { - IndexSetIterator elements(&_adjs[i]); - uint datum; - while ((datum = elements.next()) != 0) { - _adjs[datum].insert( i ); + for(uint i = 0; i < _maxlrg; i++ ) { + if (!_adjs[i].is_empty()) { + IndexSetIterator elements(&_adjs[i]); + uint datum; + while ((datum = elements.next()) != 0) { + _adjs[datum].insert(i); + } } } _is_square = true; @@ -108,16 +110,18 @@ } // Union edges of B into A -void PhaseIFG::Union( uint a, uint b ) { +void PhaseIFG::Union(uint a, uint b) { assert( _is_square, "only on square" ); IndexSet *A = &_adjs[a]; - IndexSetIterator b_elements(&_adjs[b]); - uint datum; - while ((datum = b_elements.next()) != 0) { - if(A->insert(datum)) { - _adjs[datum].insert(a); - lrgs(a).invalid_degree(); - lrgs(datum).invalid_degree(); + if (!_adjs[b].is_empty()) { + IndexSetIterator b_elements(&_adjs[b]); + uint datum; + while ((datum = b_elements.next()) != 0) { + if (A->insert(datum)) { + _adjs[datum].insert(a); + lrgs(a).invalid_degree(); + lrgs(datum).invalid_degree(); + } } } } @@ -130,12 +134,15 @@ _yanked->set(a); // I remove the LRG from all neighbors. - IndexSetIterator elements(&_adjs[a]); LRG &lrg_a = lrgs(a); - uint datum; - while ((datum = elements.next()) != 0) { - _adjs[datum].remove(a); - lrgs(datum).inc_degree( -lrg_a.compute_degree(lrgs(datum)) ); + + if (!_adjs[a].is_empty()) { + IndexSetIterator elements(&_adjs[a]); + uint datum; + while ((datum = elements.next()) != 0) { + _adjs[datum].remove(a); + lrgs(datum).inc_degree(-lrg_a.compute_degree(lrgs(datum))); + } } return neighbors(a); } @@ -146,6 +153,8 @@ assert( _yanked->test(a), "" ); _yanked->remove(a); + if (_adjs[a].is_empty()) return; + IndexSetIterator elements(&_adjs[a]); uint datum; while ((datum = elements.next()) != 0) { @@ -159,7 +168,7 @@ // mis-aligned (or for Fat-Projections, not-adjacent) then we have to // MULTIPLY the sizes. Inspect Brigg's thesis on register pairs to see why // this is so. -int LRG::compute_degree( LRG &l ) const { +int LRG::compute_degree(LRG &l) const { int tmp; int num_regs = _num_regs; int nregs = l.num_regs(); @@ -174,14 +183,15 @@ // mis-aligned (or for Fat-Projections, not-adjacent) then we have to // MULTIPLY the sizes. Inspect Brigg's thesis on register pairs to see why // this is so. -int PhaseIFG::effective_degree( uint lidx ) const { +int PhaseIFG::effective_degree(uint lidx) const { + IndexSet *s = neighbors(lidx); + if (s->is_empty()) return 0; int eff = 0; int num_regs = lrgs(lidx).num_regs(); int fat_proj = lrgs(lidx)._fat_proj; - IndexSet *s = neighbors(lidx); IndexSetIterator elements(s); uint nidx; - while((nidx = elements.next()) != 0) { + while ((nidx = elements.next()) != 0) { LRG &lrgn = lrgs(nidx); int nregs = lrgn.num_regs(); eff += (fat_proj || lrgn._fat_proj) // either is a fat-proj? @@ -196,14 +206,16 @@ void PhaseIFG::dump() const { tty->print_cr("-- Interference Graph --%s--", _is_square ? "square" : "triangular" ); - if( _is_square ) { - for( uint i = 0; i < _maxlrg; i++ ) { + if (_is_square) { + for (uint i = 0; i < _maxlrg; i++) { tty->print(_yanked->test(i) ? "XX " : " "); tty->print("L%d: { ",i); - IndexSetIterator elements(&_adjs[i]); - uint datum; - while ((datum = elements.next()) != 0) { - tty->print("L%d ", datum); + if (!_adjs[i].is_empty()) { + IndexSetIterator elements(&_adjs[i]); + uint datum; + while ((datum = elements.next()) != 0) { + tty->print("L%d ", datum); + } } tty->print_cr("}"); @@ -221,10 +233,12 @@ tty->print("L%d ",j - 1); } tty->print("| "); - IndexSetIterator elements(&_adjs[i]); - uint datum; - while ((datum = elements.next()) != 0) { - tty->print("L%d ", datum); + if (!_adjs[i].is_empty()) { + IndexSetIterator elements(&_adjs[i]); + uint datum; + while ((datum = elements.next()) != 0) { + tty->print("L%d ", datum); + } } tty->print("}\n"); } @@ -251,16 +265,18 @@ for( uint i = 0; i < _maxlrg; i++ ) { assert(!_yanked->test(i) || !neighbor_cnt(i), "Is removed completely" ); IndexSet *set = &_adjs[i]; - IndexSetIterator elements(set); - uint idx; - uint last = 0; - while ((idx = elements.next()) != 0) { - assert(idx != i, "Must have empty diagonal"); - assert(pc->_lrg_map.find_const(idx) == idx, "Must not need Find"); - assert(_adjs[idx].member(i), "IFG not square"); - assert(!_yanked->test(idx), "No yanked neighbors"); - assert(last < idx, "not sorted increasing"); - last = idx; + if (!set->is_empty()) { + IndexSetIterator elements(set); + uint idx; + uint last = 0; + while ((idx = elements.next()) != 0) { + assert(idx != i, "Must have empty diagonal"); + assert(pc->_lrg_map.find_const(idx) == idx, "Must not need Find"); + assert(_adjs[idx].member(i), "IFG not square"); + assert(!_yanked->test(idx), "No yanked neighbors"); + assert(last < idx, "not sorted increasing"); + last = idx; + } } assert(!lrgs(i)._degree_valid || effective_degree(i) == lrgs(i).degree(), "degree is valid but wrong"); } @@ -273,16 +289,18 @@ * Only interfere if acceptable register masks overlap. */ void PhaseChaitin::interfere_with_live(uint lid, IndexSet* liveout) { - LRG& lrg = lrgs(lid); - const RegMask& rm = lrg.mask(); - IndexSetIterator elements(liveout); - uint interfering_lid = elements.next(); - while (interfering_lid != 0) { - LRG& interfering_lrg = lrgs(interfering_lid); - if (rm.overlap(interfering_lrg.mask())) { - _ifg->add_edge(lid, interfering_lid); + if (!liveout->is_empty()) { + LRG& lrg = lrgs(lid); + const RegMask &rm = lrg.mask(); + IndexSetIterator elements(liveout); + uint interfering_lid = elements.next(); + while (interfering_lid != 0) { + LRG& interfering_lrg = lrgs(interfering_lid); + if (rm.overlap(interfering_lrg.mask())) { + _ifg->add_edge(lid, interfering_lid); + } + interfering_lid = elements.next(); } - interfering_lid = elements.next(); } } @@ -381,6 +399,9 @@ #ifdef ASSERT uint PhaseChaitin::count_int_pressure(IndexSet* liveout) { + if (liveout->is_empty()) { + return 0; + } IndexSetIterator elements(liveout); uint lidx = elements.next(); uint cnt = 0; @@ -397,6 +418,9 @@ } uint PhaseChaitin::count_float_pressure(IndexSet* liveout) { + if (liveout->is_empty()) { + return 0; + } IndexSetIterator elements(liveout); uint lidx = elements.next(); uint cnt = 0; @@ -494,13 +518,15 @@ * the block from the area. */ void PhaseChaitin::compute_initial_block_pressure(Block* b, IndexSet* liveout, Pressure& int_pressure, Pressure& float_pressure, double cost) { - IndexSetIterator elements(liveout); - uint lid = elements.next(); - while (lid != 0) { - LRG& lrg = lrgs(lid); - lrg._area += cost; - raise_pressure(b, lrg, int_pressure, float_pressure); - lid = elements.next(); + if (!liveout->is_empty()) { + IndexSetIterator elements(liveout); + uint lid = elements.next(); + while (lid != 0) { + LRG &lrg = lrgs(lid); + lrg._area += cost; + raise_pressure(b, lrg, int_pressure, float_pressure); + lid = elements.next(); + } } assert(int_pressure.current_pressure() == count_int_pressure(liveout), "the int pressure is incorrect"); assert(float_pressure.current_pressure() == count_float_pressure(liveout), "the float pressure is incorrect"); @@ -512,13 +538,15 @@ * and int/pointer registers. */ void PhaseChaitin::compute_entry_block_pressure(Block* b) { - IndexSet* livein = _live->livein(b); - IndexSetIterator elements(livein); - uint lid = elements.next(); - while (lid != 0) { - LRG& lrg = lrgs(lid); - raise_pressure(b, lrg, _sched_int_pressure, _sched_float_pressure); - lid = elements.next(); + IndexSet *livein = _live->livein(b); + if (!livein->is_empty()) { + IndexSetIterator elements(livein); + uint lid = elements.next(); + while (lid != 0) { + LRG &lrg = lrgs(lid); + raise_pressure(b, lrg, _sched_int_pressure, _sched_float_pressure); + lid = elements.next(); + } } // Now check phis for locally defined inputs for (uint j = 0; j < b->number_of_nodes(); j++) { @@ -546,15 +574,18 @@ * and int/pointer registers. */ void PhaseChaitin::compute_exit_block_pressure(Block* b) { + IndexSet* livein = _live->live(b); - IndexSetIterator elements(livein); _sched_int_pressure.set_current_pressure(0); _sched_float_pressure.set_current_pressure(0); - uint lid = elements.next(); - while (lid != 0) { - LRG& lrg = lrgs(lid); - raise_pressure(b, lrg, _sched_int_pressure, _sched_float_pressure); - lid = elements.next(); + if (!livein->is_empty()) { + IndexSetIterator elements(livein); + uint lid = elements.next(); + while (lid != 0) { + LRG &lrg = lrgs(lid); + raise_pressure(b, lrg, _sched_int_pressure, _sched_float_pressure); + lid = elements.next(); + } } } @@ -654,6 +685,7 @@ * all conflicting parties and avoid the interference. */ void PhaseChaitin::remove_bound_register_from_interfering_live_ranges(LRG& lrg, IndexSet* liveout, uint& must_spill) { + if (liveout->is_empty()) return; // Check for common case const RegMask& rm = lrg.mask(); int r_size = lrg.num_regs(); @@ -833,7 +865,7 @@ Node* n = block->get_node(location); uint lid = _lrg_map.live_range_id(n); - if(lid) { + if (lid) { LRG& lrg = lrgs(lid); // A DEF normally costs block frequency; rematerialized values are diff -r 355f4f42dda5 -r 95a99e617f28 src/hotspot/share/opto/indexSet.cpp --- a/src/hotspot/share/opto/indexSet.cpp Thu Nov 14 10:55:46 2019 +0100 +++ b/src/hotspot/share/opto/indexSet.cpp Thu Nov 14 15:24:35 2019 +0100 @@ -111,6 +111,9 @@ IndexSet::BitBlock *IndexSet::alloc_block_containing(uint element) { BitBlock *block = alloc_block(); uint bi = get_block_index(element); + if (bi >= _current_block_limit) { + _current_block_limit = bi + 1; + } _blocks[bi] = block; return block; } @@ -125,7 +128,7 @@ assert(block != &_empty_block, "cannot free the empty block"); block->set_next((IndexSet::BitBlock*)Compile::current()->indexSet_free_block_list()); Compile::current()->set_indexSet_free_block_list(block); - set_block(i,&_empty_block); + set_block(i, &_empty_block); } //------------------------------lrg_union-------------------------------------- @@ -168,38 +171,43 @@ // other color. (A variant of the Briggs assertion) uint reg_degree = 0; - uint element; + uint element = 0; // Load up the combined interference set with the neighbors of one - IndexSetIterator elements(one); - while ((element = elements.next()) != 0) { - LRG &lrg = ifg->lrgs(element); - if (mask.overlap(lrg.mask())) { - insert(element); - if( !lrg.mask().is_AllStack() ) { - reg_degree += lrg1.compute_degree(lrg); - if( reg_degree >= fail_degree ) return reg_degree; - } else { - // !!!!! Danger! No update to reg_degree despite having a neighbor. - // A variant of the Briggs assertion. - // Not needed if I simplify during coalesce, ala George/Appel. - assert( lrg.lo_degree(), "" ); + if (!one->is_empty()) { + IndexSetIterator elements(one); + while ((element = elements.next()) != 0) { + LRG &lrg = ifg->lrgs(element); + if (mask.overlap(lrg.mask())) { + insert(element); + if (!lrg.mask().is_AllStack()) { + reg_degree += lrg1.compute_degree(lrg); + if (reg_degree >= fail_degree) return reg_degree; + } else { + // !!!!! Danger! No update to reg_degree despite having a neighbor. + // A variant of the Briggs assertion. + // Not needed if I simplify during coalesce, ala George/Appel. + assert(lrg.lo_degree(), ""); + } } } } // Add neighbors of two as well - IndexSetIterator elements2(two); - while ((element = elements2.next()) != 0) { - LRG &lrg = ifg->lrgs(element); - if (mask.overlap(lrg.mask())) { - if (insert(element)) { - if( !lrg.mask().is_AllStack() ) { - reg_degree += lrg2.compute_degree(lrg); - if( reg_degree >= fail_degree ) return reg_degree; - } else { - // !!!!! Danger! No update to reg_degree despite having a neighbor. - // A variant of the Briggs assertion. - // Not needed if I simplify during coalesce, ala George/Appel. - assert( lrg.lo_degree(), "" ); + + if (!two->is_empty()) { + IndexSetIterator elements2(two); + while ((element = elements2.next()) != 0) { + LRG &lrg = ifg->lrgs(element); + if (mask.overlap(lrg.mask())) { + if (insert(element)) { + if (!lrg.mask().is_AllStack()) { + reg_degree += lrg2.compute_degree(lrg); + if (reg_degree >= fail_degree) return reg_degree; + } else { + // !!!!! Danger! No update to reg_degree despite having a neighbor. + // A variant of the Briggs assertion. + // Not needed if I simplify during coalesce, ala George/Appel. + assert(lrg.lo_degree(), ""); + } } } } @@ -219,6 +227,7 @@ _max_elements = set->_max_elements; #endif _count = set->_count; + _current_block_limit = set->_current_block_limit; _max_blocks = set->_max_blocks; if (_max_blocks <= preallocated_block_list_size) { _blocks = _preallocated_block_list; @@ -248,6 +257,7 @@ _max_elements = max_elements; #endif _count = 0; + _current_block_limit = 0; _max_blocks = (max_elements + bits_per_block - 1) / bits_per_block; if (_max_blocks <= preallocated_block_list_size) { @@ -272,6 +282,7 @@ _max_elements = max_elements; #endif // ASSERT _count = 0; + _current_block_limit = 0; _max_blocks = (max_elements + bits_per_block - 1) / bits_per_block; if (_max_blocks <= preallocated_block_list_size) { @@ -294,7 +305,8 @@ set->check_watch("swap", _serial_number); #endif - for (uint i = 0; i < _max_blocks; i++) { + uint max = MAX2(_current_block_limit, set->_current_block_limit); + for (uint i = 0; i < max; i++) { BitBlock *temp = _blocks[i]; set_block(i, set->_blocks[i]); set->set_block(i, temp); @@ -302,6 +314,11 @@ uint temp = _count; _count = set->_count; set->_count = temp; + + temp = _current_block_limit; + _current_block_limit = set->_current_block_limit; + set->_current_block_limit = temp; + } //---------------------------- IndexSet::dump() ----------------------------- @@ -383,78 +400,6 @@ // Create an iterator for a set. If empty blocks are detected when iterating // over the set, these blocks are replaced. -IndexSetIterator::IndexSetIterator(IndexSet *set) { -#ifdef ASSERT - if (CollectIndexSetStatistics) { - set->tally_iteration_statistics(); - } - set->check_watch("traversed", set->count()); -#endif - if (set->is_empty()) { - _current = 0; - _next_word = IndexSet::words_per_block; - _next_block = 1; - _max_blocks = 1; - - // We don't need the following values when we iterate over an empty set. - // The commented out code is left here to document that the omission - // is intentional. - // - //_value = 0; - //_words = NULL; - //_blocks = NULL; - //_set = NULL; - } else { - _current = 0; - _value = 0; - _next_block = 0; - _next_word = IndexSet::words_per_block; - - _max_blocks = set->_max_blocks; - _words = NULL; - _blocks = set->_blocks; - _set = set; - } -} - -//---------------------------- IndexSetIterator(const) ----------------------------- -// Iterate over a constant IndexSet. - -IndexSetIterator::IndexSetIterator(const IndexSet *set) { -#ifdef ASSERT - if (CollectIndexSetStatistics) { - set->tally_iteration_statistics(); - } - // We don't call check_watch from here to avoid bad recursion. - // set->check_watch("traversed const", set->count()); -#endif - if (set->is_empty()) { - _current = 0; - _next_word = IndexSet::words_per_block; - _next_block = 1; - _max_blocks = 1; - - // We don't need the following values when we iterate over an empty set. - // The commented out code is left here to document that the omission - // is intentional. - // - //_value = 0; - //_words = NULL; - //_blocks = NULL; - //_set = NULL; - } else { - _current = 0; - _value = 0; - _next_block = 0; - _next_word = IndexSet::words_per_block; - - _max_blocks = set->_max_blocks; - _words = NULL; - _blocks = set->_blocks; - _set = NULL; - } -} - //---------------------------- List16Iterator::advance_and_next() ----------------------------- // Advance to the next non-empty word in the set being iterated over. Return the next element // if there is one. If we are done, return 0. This method is called from the next() method @@ -467,10 +412,8 @@ // Found a non-empty word. _value = ((_next_block - 1) * IndexSet::bits_per_block) + (wi * IndexSet::bits_per_word); _current = _words[wi]; - - _next_word = wi+1; - - return next(); + _next_word = wi + 1; + return next_value(); } } @@ -488,8 +431,7 @@ _next_block = bi+1; _next_word = wi+1; - - return next(); + return next_value(); } } @@ -501,11 +443,6 @@ } } - // These assignments make redundant calls to next on a finished iterator - // faster. Probably not necessary. - _next_block = _max_blocks; - _next_word = IndexSet::words_per_block; - // No more words. return 0; } diff -r 355f4f42dda5 -r 95a99e617f28 src/hotspot/share/opto/indexSet.hpp --- a/src/hotspot/share/opto/indexSet.hpp Thu Nov 14 10:55:46 2019 +0100 +++ b/src/hotspot/share/opto/indexSet.hpp Thu Nov 14 15:24:35 2019 +0100 @@ -190,6 +190,9 @@ // The number of elements in the set uint _count; + // The current upper limit of blocks that has been allocated and might be in use + uint _current_block_limit; + // Our top level array of bitvector segments BitBlock **_blocks; @@ -246,12 +249,13 @@ void clear() { _count = 0; - for (uint i = 0; i < _max_blocks; i++) { + for (uint i = 0; i < _current_block_limit; i++) { BitBlock *block = _blocks[i]; if (block != &_empty_block) { free_block(i); } } + _current_block_limit = 0; } uint count() const { return _count; } @@ -380,18 +384,18 @@ // The index of the next word we will inspect uint _next_word; - // A pointer to the contents of the current block - uint32_t *_words; - // The index of the next block we will inspect uint _next_block; + // The number of blocks in the set + uint _max_blocks; + + // A pointer to the contents of the current block + uint32_t *_words; + // A pointer to the blocks in our set IndexSet::BitBlock **_blocks; - // The number of blocks in the set - uint _max_blocks; - // If the iterator was created from a non-const set, we replace // non-canonical empty blocks with the _empty_block pointer. If // _set is NULL, we do no replacement. @@ -405,22 +409,64 @@ // If an iterator is built from a constant set then empty blocks // are not canonicalized. - IndexSetIterator(IndexSet *set); - IndexSetIterator(const IndexSet *set); + IndexSetIterator(IndexSet *set) : + _current(0), + _value(0), + _next_word(IndexSet::words_per_block), + _next_block(0), + _max_blocks(set->is_empty() ? 0 : set->_current_block_limit), + _words(NULL), + _blocks(set->_blocks), + _set(set) { + #ifdef ASSERT + if (CollectIndexSetStatistics) { + set->tally_iteration_statistics(); + } + set->check_watch("traversed", set->count()); + #endif + } + + IndexSetIterator(const IndexSet *set) : + _current(0), + _value(0), + _next_word(IndexSet::words_per_block), + _next_block(0), + _max_blocks(set->is_empty() ? 0 : set->_current_block_limit), + _words(NULL), + _blocks(set->_blocks), + _set(NULL) + { + #ifdef ASSERT + if (CollectIndexSetStatistics) { + set->tally_iteration_statistics(); + } + // We don't call check_watch from here to avoid bad recursion. + // set->check_watch("traversed const", set->count()); + #endif + } + + // Return the next element of the set. + uint next_value() { + uint current = _current; + assert(current != 0, "sanity"); + uint advance = count_trailing_zeros(current); + assert(((current >> advance) & 0x1) == 1, "sanity"); + _current = (current >> advance) - 1; + _value += advance; + return _value; + } // Return the next element of the set. Return 0 when done. uint next() { - uint current = _current; - if (current != 0) { - uint advance = count_trailing_zeros(current); - assert(((current >> advance) & 0x1) == 1, "sanity"); - _current = (current >> advance) - 1; - _value += advance; - return _value; + if (_current != 0) { + return next_value(); + } else if (_next_word < IndexSet::words_per_block || _next_block < _max_blocks) { + return advance_and_next(); } else { - return advance_and_next(); + return 0; } } + }; #endif // SHARE_OPTO_INDEXSET_HPP diff -r 355f4f42dda5 -r 95a99e617f28 src/hotspot/share/opto/live.cpp --- a/src/hotspot/share/opto/live.cpp Thu Nov 14 10:55:46 2019 +0100 +++ b/src/hotspot/share/opto/live.cpp Thu Nov 14 15:24:35 2019 +0100 @@ -84,7 +84,7 @@ // Array of delta-set pointers, indexed by block pre_order-1. _deltas = NEW_RESOURCE_ARRAY(IndexSet*,_cfg.number_of_blocks()); - memset( _deltas, 0, sizeof(IndexSet*)* _cfg.number_of_blocks()); + memset(_deltas, 0, sizeof(IndexSet*)* _cfg.number_of_blocks()); _free_IndexSet = NULL; @@ -108,8 +108,8 @@ uint r = _names.at(n->_idx); assert(!def_outside->member(r), "Use of external LRG overlaps the same LRG defined in this block"); - def->insert( r ); - use->remove( r ); + def->insert(r); + use->remove(r); uint cnt = n->req(); for (uint k = 1; k < cnt; k++) { Node *nk = n->in(k); @@ -152,7 +152,7 @@ while (_worklist->size()) { Block* block = _worklist->pop(); IndexSet *delta = getset(block); - assert( delta->count(), "missing delta set" ); + assert(delta->count(), "missing delta set"); // Add new-live-in to predecessors live-out sets for (uint l = 1; l < block->num_preds(); l++) { @@ -191,36 +191,34 @@ // Get an IndexSet for a block. Return existing one, if any. Make a new // empty one if a prior one does not exist. -IndexSet *PhaseLive::getset( Block *p ) { +IndexSet *PhaseLive::getset(Block *p) { IndexSet *delta = _deltas[p->_pre_order-1]; - if( !delta ) // Not on worklist? + if (!delta) { // Not on worklist? // Get a free set; flag as being on worklist delta = _deltas[p->_pre_order-1] = getfreeset(); + } return delta; // Return set of new live-out items } // Pull from free list, or allocate. Internal allocation on the returned set // is always from thread local storage. -IndexSet *PhaseLive::getfreeset( ) { +IndexSet *PhaseLive::getfreeset() { IndexSet *f = _free_IndexSet; - if( !f ) { + if (!f) { f = new IndexSet; -// f->set_arena(Thread::current()->resource_area()); f->initialize(_maxlrg, Thread::current()->resource_area()); } else { // Pull from free list _free_IndexSet = f->next(); - //f->_cnt = 0; // Reset to empty -// f->set_arena(Thread::current()->resource_area()); f->initialize(_maxlrg, Thread::current()->resource_area()); } return f; } // Free an IndexSet from a block. -void PhaseLive::freeset( Block *p ) { +void PhaseLive::freeset(Block *p) { IndexSet *f = _deltas[p->_pre_order-1]; - if ( _keep_deltas ) { + if (_keep_deltas) { add_livein(p, f); } f->set_next(_free_IndexSet); @@ -230,40 +228,45 @@ // Add a live-out value to a given blocks live-out set. If it is new, then // also add it to the delta set and stick the block on the worklist. -void PhaseLive::add_liveout( Block *p, uint r, VectorSet &first_pass ) { +void PhaseLive::add_liveout(Block *p, uint r, VectorSet &first_pass) { IndexSet *live = &_live[p->_pre_order-1]; - if( live->insert(r) ) { // If actually inserted... + if (live->insert(r)) { // If actually inserted... // We extended the live-out set. See if the value is generated locally. // If it is not, then we must extend the live-in set. - if( !_defs[p->_pre_order-1].member( r ) ) { - if( !_deltas[p->_pre_order-1] && // Not on worklist? - first_pass.test(p->_pre_order) ) + if (!_defs[p->_pre_order-1].member(r)) { + if (!_deltas[p->_pre_order-1] && // Not on worklist? + first_pass.test(p->_pre_order)) { _worklist->push(p); // Actually go on worklist if already 1st pass + } getset(p)->insert(r); } } } // Add a vector of live-out values to a given blocks live-out set. -void PhaseLive::add_liveout( Block *p, IndexSet *lo, VectorSet &first_pass ) { +void PhaseLive::add_liveout(Block *p, IndexSet *lo, VectorSet &first_pass) { IndexSet *live = &_live[p->_pre_order-1]; IndexSet *defs = &_defs[p->_pre_order-1]; IndexSet *on_worklist = _deltas[p->_pre_order-1]; IndexSet *delta = on_worklist ? on_worklist : getfreeset(); - IndexSetIterator elements(lo); - uint r; - while ((r = elements.next()) != 0) { - if( live->insert(r) && // If actually inserted... - !defs->member( r ) ) // and not defined locally - delta->insert(r); // Then add to live-in set + if (!lo->is_empty()) { + IndexSetIterator elements(lo); + uint r; + while ((r = elements.next()) != 0) { + if (live->insert(r) && // If actually inserted... + !defs->member(r)) { // and not defined locally + delta->insert(r); // Then add to live-in set + } + } } - if( delta->count() ) { // If actually added things + if (delta->count()) { // If actually added things _deltas[p->_pre_order-1] = delta; // Flag as on worklist now - if( !on_worklist && // Not on worklist? - first_pass.test(p->_pre_order) ) + if (!on_worklist && // Not on worklist? + first_pass.test(p->_pre_order)) { _worklist->push(p); // Actually go on worklist if already 1st pass + } } else { // Nothing there; just free it delta->set_next(_free_IndexSet); _free_IndexSet = delta; // Drop onto free list @@ -273,23 +276,25 @@ // Add a vector of live-in values to a given blocks live-in set. void PhaseLive::add_livein(Block *p, IndexSet *lo) { IndexSet *livein = &_livein[p->_pre_order-1]; - IndexSetIterator elements(lo); - uint r; - while ((r = elements.next()) != 0) { - livein->insert(r); // Then add to live-in set + if (!livein->is_empty()) { + IndexSetIterator elements(lo); + uint r; + while ((r = elements.next()) != 0) { + livein->insert(r); // Then add to live-in set + } } } #ifndef PRODUCT // Dump the live-out set for a block -void PhaseLive::dump( const Block *b ) const { +void PhaseLive::dump(const Block *b) const { tty->print("Block %d: ",b->_pre_order); - if ( _keep_deltas ) { + if (_keep_deltas) { tty->print("LiveIn: "); _livein[b->_pre_order-1].dump(); } tty->print("LiveOut: "); _live[b->_pre_order-1].dump(); uint cnt = b->number_of_nodes(); - for( uint i=0; iprint("L%d/", _names.at(b->get_node(i)->_idx)); b->get_node(i)->dump(); } @@ -297,7 +302,7 @@ } // Verify that base pointers and derived pointers are still sane. -void PhaseChaitin::verify_base_ptrs( ResourceArea *a ) const { +void PhaseChaitin::verify_base_ptrs(ResourceArea *a) const { #ifdef ASSERT Unique_Node_List worklist(a); for (uint i = 0; i < _cfg.number_of_blocks(); i++) { @@ -322,17 +327,17 @@ worklist.clear(); worklist.push(check); uint k = 0; - while( k < worklist.size() ) { + while (k < worklist.size()) { check = worklist.at(k); assert(check,"Bad base or derived pointer"); // See PhaseChaitin::find_base_for_derived() for all cases. int isc = check->is_Copy(); - if( isc ) { + if (isc) { worklist.push(check->in(isc)); - } else if( check->is_Phi() ) { + } else if (check->is_Phi()) { for (uint m = 1; m < check->req(); m++) worklist.push(check->in(m)); - } else if( check->is_Con() ) { + } else if (check->is_Con()) { if (is_derived) { // Derived is NULL+offset assert(!is_derived || check->bottom_type()->is_ptr()->ptr() == TypePtr::Null,"Bad derived pointer"); @@ -346,8 +351,8 @@ check->bottom_type()->is_ptr()->ptr() == TypePtr::Null,"Bad base pointer"); } } - } else if( check->bottom_type()->is_ptr()->_offset == 0 ) { - if(check->is_Proj() || (check->is_Mach() && + } else if (check->bottom_type()->is_ptr()->_offset == 0) { + if (check->is_Proj() || (check->is_Mach() && (check->as_Mach()->ideal_Opcode() == Op_CreateEx || check->as_Mach()->ideal_Opcode() == Op_ThreadLocal || check->as_Mach()->ideal_Opcode() == Op_CMoveP || @@ -381,7 +386,7 @@ } // Verify that graphs and base pointers are still sane. -void PhaseChaitin::verify( ResourceArea *a, bool verify_ifg ) const { +void PhaseChaitin::verify(ResourceArea *a, bool verify_ifg) const { #ifdef ASSERT if (VerifyRegisterAllocator) { _cfg.verify(); diff -r 355f4f42dda5 -r 95a99e617f28 src/hotspot/share/opto/reg_split.cpp --- a/src/hotspot/share/opto/reg_split.cpp Thu Nov 14 10:55:46 2019 +0100 +++ b/src/hotspot/share/opto/reg_split.cpp Thu Nov 14 15:24:35 2019 +0100 @@ -1271,10 +1271,12 @@ // it contains no members which compress to defidx. Finding such an // instance may be a case to add liveout adjustment in compress_uf_map(). // See 5063219. - uint member; - IndexSetIterator isi(liveout); - while ((member = isi.next()) != 0) { - assert(defidx != _lrg_map.find_const(member), "Live out member has not been compressed"); + if (!liveout->is_empty()) { + uint member; + IndexSetIterator isi(liveout); + while ((member = isi.next()) != 0) { + assert(defidx != _lrg_map.find_const(member), "Live out member has not been compressed"); + } } #endif Reachblock[slidx] = NULL;