1485 // Generate prefetch instructions for next allocations. |
1485 // Generate prefetch instructions for next allocations. |
1486 Node* PhaseMacroExpand::prefetch_allocation(Node* i_o, Node*& needgc_false, |
1486 Node* PhaseMacroExpand::prefetch_allocation(Node* i_o, Node*& needgc_false, |
1487 Node*& contended_phi_rawmem, |
1487 Node*& contended_phi_rawmem, |
1488 Node* old_eden_top, Node* new_eden_top, |
1488 Node* old_eden_top, Node* new_eden_top, |
1489 Node* length) { |
1489 Node* length) { |
|
1490 enum { fall_in_path = 1, pf_path = 2 }; |
1490 if( UseTLAB && AllocatePrefetchStyle == 2 ) { |
1491 if( UseTLAB && AllocatePrefetchStyle == 2 ) { |
1491 // Generate prefetch allocation with watermark check. |
1492 // Generate prefetch allocation with watermark check. |
1492 // As an allocation hits the watermark, we will prefetch starting |
1493 // As an allocation hits the watermark, we will prefetch starting |
1493 // at a "distance" away from watermark. |
1494 // at a "distance" away from watermark. |
1494 enum { fall_in_path = 1, pf_path = 2 }; |
|
1495 |
1495 |
1496 Node *pf_region = new (C, 3) RegionNode(3); |
1496 Node *pf_region = new (C, 3) RegionNode(3); |
1497 Node *pf_phi_rawmem = new (C, 3) PhiNode( pf_region, Type::MEMORY, |
1497 Node *pf_phi_rawmem = new (C, 3) PhiNode( pf_region, Type::MEMORY, |
1498 TypeRawPtr::BOTTOM ); |
1498 TypeRawPtr::BOTTOM ); |
1499 // I/O is used for Prefetch |
1499 // I/O is used for Prefetch |
1568 transform_later(pf_phi_abio); |
1568 transform_later(pf_phi_abio); |
1569 |
1569 |
1570 needgc_false = pf_region; |
1570 needgc_false = pf_region; |
1571 contended_phi_rawmem = pf_phi_rawmem; |
1571 contended_phi_rawmem = pf_phi_rawmem; |
1572 i_o = pf_phi_abio; |
1572 i_o = pf_phi_abio; |
|
1573 } else if( UseTLAB && AllocatePrefetchStyle == 3 ) { |
|
1574 // Insert a prefetch for each allocation only on the fast-path |
|
1575 Node *pf_region = new (C, 3) RegionNode(3); |
|
1576 Node *pf_phi_rawmem = new (C, 3) PhiNode( pf_region, Type::MEMORY, |
|
1577 TypeRawPtr::BOTTOM ); |
|
1578 |
|
1579 // Generate several prefetch instructions only for arrays. |
|
1580 uint lines = (length != NULL) ? AllocatePrefetchLines : 1; |
|
1581 uint step_size = AllocatePrefetchStepSize; |
|
1582 uint distance = AllocatePrefetchDistance; |
|
1583 |
|
1584 // Next cache address. |
|
1585 Node *cache_adr = new (C, 4) AddPNode(old_eden_top, old_eden_top, |
|
1586 _igvn.MakeConX(distance)); |
|
1587 transform_later(cache_adr); |
|
1588 cache_adr = new (C, 2) CastP2XNode(needgc_false, cache_adr); |
|
1589 transform_later(cache_adr); |
|
1590 Node* mask = _igvn.MakeConX(~(intptr_t)(step_size-1)); |
|
1591 cache_adr = new (C, 3) AndXNode(cache_adr, mask); |
|
1592 transform_later(cache_adr); |
|
1593 cache_adr = new (C, 2) CastX2PNode(cache_adr); |
|
1594 transform_later(cache_adr); |
|
1595 |
|
1596 // Prefetch |
|
1597 Node *prefetch = new (C, 3) PrefetchWriteNode( contended_phi_rawmem, cache_adr ); |
|
1598 prefetch->set_req(0, needgc_false); |
|
1599 transform_later(prefetch); |
|
1600 contended_phi_rawmem = prefetch; |
|
1601 Node *prefetch_adr; |
|
1602 distance = step_size; |
|
1603 for ( uint i = 1; i < lines; i++ ) { |
|
1604 prefetch_adr = new (C, 4) AddPNode( cache_adr, cache_adr, |
|
1605 _igvn.MakeConX(distance) ); |
|
1606 transform_later(prefetch_adr); |
|
1607 prefetch = new (C, 3) PrefetchWriteNode( contended_phi_rawmem, prefetch_adr ); |
|
1608 transform_later(prefetch); |
|
1609 distance += step_size; |
|
1610 contended_phi_rawmem = prefetch; |
|
1611 } |
1573 } else if( AllocatePrefetchStyle > 0 ) { |
1612 } else if( AllocatePrefetchStyle > 0 ) { |
1574 // Insert a prefetch for each allocation only on the fast-path |
1613 // Insert a prefetch for each allocation only on the fast-path |
1575 Node *prefetch_adr; |
1614 Node *prefetch_adr; |
1576 Node *prefetch; |
1615 Node *prefetch; |
1577 // Generate several prefetch instructions only for arrays. |
1616 // Generate several prefetch instructions only for arrays. |