1039   // graph traversal helpers

  1040 

  1041   MemBarNode *parent_membar(const Node *n);

  1042   MemBarNode *child_membar(const MemBarNode *n);

  1043   bool leading_membar(const MemBarNode *barrier);

  1044 

  1045   bool is_card_mark_membar(const MemBarNode *barrier);

  1047 

  1048   MemBarNode *leading_to_normal(MemBarNode *leading);

  1049   MemBarNode *normal_to_leading(const MemBarNode *barrier);

  1050   MemBarNode *card_mark_to_trailing(const MemBarNode *barrier);

  1051   MemBarNode *trailing_to_card_mark(const MemBarNode *trailing);

  1052   MemBarNode *trailing_to_leading(const MemBarNode *trailing);

  1275   // graph traversal helpers used for volatile put/get and CAS

  1276   // optimization

  1277 

  1278   // 1) general purpose helpers

  1279 

  1280   // if node n is linked to a parent MemBarNode by an intervening

  1281   // Control and Memory ProjNode return the MemBarNode otherwise return

  1282   // NULL.

  1283   //

  1284   // n may only be a Load or a MemBar.

  1285 

  1286   MemBarNode *parent_membar(const Node *n)

  1287   {

  1288     Node *ctl = NULL;

  1289     Node *mem = NULL;

  1290     Node *membar = NULL;

  1291 

  1292     if (n->is_Load()) {

  1293       ctl = n->lookup(LoadNode::Control);

  1294       mem = n->lookup(LoadNode::Memory);

  1295     } else if (n->is_MemBar()) {

  1296       ctl = n->lookup(TypeFunc::Control);

  1297       mem = n->lookup(TypeFunc::Memory);

  1298     } else {

  1299 	return NULL;

  1300     }

  1301 

  1302     if (!ctl || !mem || !ctl->is_Proj() || !mem->is_Proj()) {

  1303       return NULL;

  1304     }

  1305 

  1306     membar = ctl->lookup(0);

  1307 

  1308     if (!membar || !membar->is_MemBar()) {

  1309       return NULL;

  1310     }

  1311 

  1312     if (mem->lookup(0) != membar) {

  1313       return NULL;

  1314     }

  1315 

  1316     return membar->as_MemBar();

  1317   }

  1318 

  1319   // if n is linked to a child MemBarNode by intervening Control and

  1320   // Memory ProjNodes return the MemBarNode otherwise return NULL.

  1321 

  1322   MemBarNode *child_membar(const MemBarNode *n)

  1323   {

  1324     ProjNode *ctl = n->proj_out_or_null(TypeFunc::Control);

  1325     ProjNode *mem = n->proj_out_or_null(TypeFunc::Memory);

  1326 

  1327     // MemBar needs to have both a Ctl and Mem projection

  1328     if (! ctl || ! mem)

  1329       return NULL;

  1330 

  1331     MemBarNode *child = NULL;

  1332     Node *x;

  1333 

  1334     for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) {

  1335       x = ctl->fast_out(i);

  1336       // if we see a membar we keep hold of it. we may also see a new

  1337       // arena copy of the original but it will appear later

  1338       if (x->is_MemBar()) {

  1339 	  child = x->as_MemBar();

  1340 	  break;

  1341       }

  1342     }

  1343 

  1344     if (child == NULL) {

  1345       return NULL;

  1346     }

  1347 

  1348     for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {

  1349       x = mem->fast_out(i);

  1350       // if we see a membar we keep hold of it. we may also see a new

  1351       // arena copy of the original but it will appear later

  1352       if (x == child) {

  1353 	return child;

  1354       }

  1355     }

  1356     return NULL;

  1357   }

  1358 

  1359   // helper predicate use to filter candidates for a leading memory

  1360   // barrier

  1361   //

  1362   // returns true if barrier is a MemBarRelease or a MemBarCPUOrder

  1363   // whose Ctl and Mem feeds come from a MemBarRelease otherwise false

  1364 

  1365   bool leading_membar(const MemBarNode *barrier)

  1366   {

  1367     int opcode = barrier->Opcode();

  1368     // if this is a release membar we are ok

  1369     if (opcode == Op_MemBarRelease) {

  1370       return true;

  1371     }

  1372     // if its a cpuorder membar . . .

  1373     if (opcode != Op_MemBarCPUOrder) {

  1374       return false;

  1375     }

  1376     // then the parent has to be a release membar

  1377     MemBarNode *parent = parent_membar(barrier);

  1378     if (!parent) {

  1379       return false;

  1380     }

  1381     opcode = parent->Opcode();

  1382     return opcode == Op_MemBarRelease;

  1383   }

  1384 

  1385   // 2) card mark detection helper

  1386 

  1387   // helper predicate which can be used to detect a volatile membar

  1388   // introduced as part of a conditional card mark sequence either by

  1389   // G1 or by CMS when UseCondCardMark is true.

  1390   //

  1391   // membar can be definitively determined to be part of a card mark

  1392   // sequence if and only if all the following hold

  1393   //

  1394   // i) it is a MemBarVolatile

  1395   //

  1396   // ii) either UseG1GC or (UseConcMarkSweepGC && UseCondCardMark) is

  1397   // true

  1398   //

  1399   // iii) the node's Mem projection feeds a StoreCM node.

  1400 

  1401   bool is_card_mark_membar(const MemBarNode *barrier)

  1402   {

  1403     if (!UseG1GC && !(UseConcMarkSweepGC && UseCondCardMark)) {

  1404       return false;

  1405     }

  1406 

  1407     if (barrier->Opcode() != Op_MemBarVolatile) {

  1408       return false;

  1409     }

  1410 

  1411     ProjNode *mem = barrier->proj_out(TypeFunc::Memory);

  1412 

  1413     for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax ; i++) {

  1414       Node *y = mem->fast_out(i);

  1415       if (y->Opcode() == Op_StoreCM) {

  1416 	return true;

  1417       }

  1418     }

  1419 

  1420     return false;

  1421   }

  1422 

  1423 

  1424   // 3) helper predicates to traverse volatile put or CAS graphs which

  1425   // may contain GC barrier subgraphs

  1426 

  1427   // Preamble

  1428   // --------

  1429   //

  1430   // for volatile writes we can omit generating barriers and employ a

  1431   // releasing store when we see a node sequence sequence with a

  1432   // leading MemBarRelease and a trailing MemBarVolatile as follows

  1433   //

  1434   //   MemBarRelease

  1435   //  {      ||      } -- optional

  1436   //  {MemBarCPUOrder}

  1437   //         ||     \\

  1438   //         ||     StoreX[mo_release]

  1439   //         | \     /

  1440   //         | MergeMem

  1441   //         | /

  1442   //  {MemBarCPUOrder} -- optional

  1443   //  {      ||      }

  1444   //   MemBarVolatile

  1445   //

  1446   // where

  1447   //  || and \\ represent Ctl and Mem feeds via Proj nodes

  1448   //  | \ and / indicate further routing of the Ctl and Mem feeds

  1449   //

  1450   // this is the graph we see for non-object stores. however, for a

  1451   // volatile Object store (StoreN/P) we may see other nodes below the

  1452   // leading membar because of the need for a GC pre- or post-write

  1453   // barrier.

  1454   //

  1455   // with most GC configurations we with see this simple variant which

  1456   // includes a post-write barrier card mark.

  1457   //

  1458   //   MemBarRelease______________________________

  1459   //         ||    \\               Ctl \        \\

  1460   //         ||    StoreN/P[mo_release] CastP2X  StoreB/CM

  1461   //         | \     /                       . . .  /

  1462   //         | MergeMem

  1463   //         | /

  1464   //         ||      /

  1465   //  {MemBarCPUOrder} -- optional

  1466   //  {      ||      }

  1467   //   MemBarVolatile

  1468   //

  1469   // i.e. the leading membar feeds Ctl to a CastP2X (which converts

  1470   // the object address to an int used to compute the card offset) and

  1471   // Ctl+Mem to a StoreB node (which does the actual card mark).

  1472   //

  1473   // n.b. a StoreCM node will only appear in this configuration when

  1474   // using CMS or G1. StoreCM differs from a normal card mark write (StoreB)

  1475   // because it implies a requirement to order visibility of the card

  1476   // mark (StoreCM) relative to the object put (StoreP/N) using a

  1477   // StoreStore memory barrier (arguably this ought to be represented

  1478   // explicitly in the ideal graph but that is not how it works). This

  1479   // ordering is required for both non-volatile and volatile

  1480   // puts. Normally that means we need to translate a StoreCM using

  1481   // the sequence

  1482   //

  1483   //   dmb ishst

  1484   //   strb

  1485   //

  1486   // However, when using G1 or CMS with conditional card marking (as

  1487   // we shall see) we don't need to insert the dmb when translating

  1488   // StoreCM because there is already an intervening StoreLoad barrier

  1489   // between it and the StoreP/N.

  1490   //

  1491   // It is also possible to perform the card mark conditionally on it

  1492   // currently being unmarked in which case the volatile put graph

  1493   // will look slightly different

  1494   //

  1495   //   MemBarRelease____________________________________________

  1496   //         ||    \\               Ctl \     Ctl \     \\  Mem \

  1497   //         ||    StoreN/P[mo_release] CastP2X   If   LoadB     |

  1498   //         | \     /                              \            |

  1499   //         | MergeMem                            . . .      StoreB

  1500   //         | /                                                /

  1501   //         ||     /

  1502   //   MemBarVolatile

  1503   //

  1504   // It is worth noting at this stage that both the above

  1505   // configurations can be uniquely identified by checking that the

  1506   // memory flow includes the following subgraph:

  1507   //

  1508   //   MemBarRelease

  1509   //  {MemBarCPUOrder}

  1510   //          |  \      . . .

  1511   //          |  StoreX[mo_release]  . . .

  1512   //          |   /

  1513   //         MergeMem

  1514   //          |

  1515   //  {MemBarCPUOrder}

  1516   //   MemBarVolatile

  1517   //

  1518   // This is referred to as a *normal* subgraph. It can easily be

  1519   // detected starting from any candidate MemBarRelease,

  1520   // StoreX[mo_release] or MemBarVolatile.

  1521   //

  1522   // A simple variation on this normal case occurs for an unsafe CAS

  1523   // operation. The basic graph for a non-object CAS is

  1524   //

  1525   //   MemBarRelease

  1526   //         ||

  1527   //   MemBarCPUOrder

  1528   //         ||     \\   . . .

  1529   //         ||     CompareAndSwapX

  1530   //         ||       |

  1531   //         ||     SCMemProj

  1532   //         | \     /

  1533   //         | MergeMem

  1534   //         | /

  1535   //   MemBarCPUOrder

  1536   //         ||

  1537   //   MemBarAcquire

  1538   //

  1539   // The same basic variations on this arrangement (mutatis mutandis)

  1540   // occur when a card mark is introduced. i.e. we se the same basic

  1541   // shape but the StoreP/N is replaced with CompareAndSawpP/N and the

  1542   // tail of the graph is a pair comprising a MemBarCPUOrder +

  1543   // MemBarAcquire.

  1544   //

  1545   // So, in the case of a CAS the normal graph has the variant form

  1546   //

  1547   //   MemBarRelease

  1548   //   MemBarCPUOrder

  1549   //          |   \      . . .

  1550   //          |  CompareAndSwapX  . . .

  1551   //          |    |

  1552   //          |   SCMemProj

  1553   //          |   /  . . .

  1554   //         MergeMem

  1555   //          |

  1556   //   MemBarCPUOrder

  1557   //   MemBarAcquire

  1558   //

  1559   // This graph can also easily be detected starting from any

  1560   // candidate MemBarRelease, CompareAndSwapX or MemBarAcquire.

  1561   //

  1562   // the code below uses two helper predicates, leading_to_normal and

  1563   // normal_to_leading to identify these normal graphs, one validating

  1564   // the layout starting from the top membar and searching down and

  1565   // the other validating the layout starting from the lower membar

  1566   // and searching up.

  1567   //

  1568   // There are two special case GC configurations when a normal graph

  1569   // may not be generated: when using G1 (which always employs a

  1570   // conditional card mark); and when using CMS with conditional card

  1571   // marking configured. These GCs are both concurrent rather than

  1572   // stop-the world GCs. So they introduce extra Ctl+Mem flow into the

  1573   // graph between the leading and trailing membar nodes, in

  1574   // particular enforcing stronger memory serialisation beween the

  1575   // object put and the corresponding conditional card mark. CMS

  1576   // employs a post-write GC barrier while G1 employs both a pre- and

  1577   // post-write GC barrier. Of course the extra nodes may be absent --

  1578   // they are only inserted for object puts/swaps. This significantly

  1579   // complicates the task of identifying whether a MemBarRelease,

  1580   // StoreX[mo_release] or MemBarVolatile forms part of a volatile put

  1581   // when using these GC configurations (see below). It adds similar

  1582   // complexity to the task of identifying whether a MemBarRelease,

  1583   // CompareAndSwapX or MemBarAcquire forms part of a CAS.

  1584   //

  1585   // In both cases the post-write subtree includes an auxiliary

  1586   // MemBarVolatile (StoreLoad barrier) separating the object put/swap

  1587   // and the read of the corresponding card. This poses two additional

  1588   // problems.

  1589   //

  1590   // Firstly, a card mark MemBarVolatile needs to be distinguished

  1591   // from a normal trailing MemBarVolatile. Resolving this first

  1592   // problem is straightforward: a card mark MemBarVolatile always

  1593   // projects a Mem feed to a StoreCM node and that is a unique marker

  1594   //

  1595   //      MemBarVolatile (card mark)

  1596   //       C |    \     . . .

  1597   //         |   StoreCM   . . .

  1598   //       . . .

  1599   //

  1600   // The second problem is how the code generator is to translate the

  1601   // card mark barrier? It always needs to be translated to a "dmb

  1602   // ish" instruction whether or not it occurs as part of a volatile

  1603   // put. A StoreLoad barrier is needed after the object put to ensure

  1604   // i) visibility to GC threads of the object put and ii) visibility

  1605   // to the mutator thread of any card clearing write by a GC

  1606   // thread. Clearly a normal store (str) will not guarantee this

  1607   // ordering but neither will a releasing store (stlr). The latter

  1608   // guarantees that the object put is visible but does not guarantee

  1609   // that writes by other threads have also been observed.

  1610   //

  1611   // So, returning to the task of translating the object put and the

  1612   // leading/trailing membar nodes: what do the non-normal node graph

  1613   // look like for these 2 special cases? and how can we determine the

  1614   // status of a MemBarRelease, StoreX[mo_release] or MemBarVolatile

  1615   // in both normal and non-normal cases?

  1616   //

  1617   // A CMS GC post-barrier wraps its card write (StoreCM) inside an If

  1618   // which selects conditonal execution based on the value loaded

  1619   // (LoadB) from the card. Ctl and Mem are fed to the If via an

  1620   // intervening StoreLoad barrier (MemBarVolatile).

  1621   //

  1622   // So, with CMS we may see a node graph for a volatile object store

  1623   // which looks like this

  1624   //

  1625   //   MemBarRelease

  1626   //  {MemBarCPUOrder}_(leading)_________________

  1627   //     C |    M \       \\                   C \

  1628   //       |       \    StoreN/P[mo_release]  CastP2X

  1629   //       |    Bot \    /

  1630   //       |       MergeMem

  1631   //       |         /

  1632   //      MemBarVolatile (card mark)

  1633   //     C |  ||    M |

  1634   //       | LoadB    |

  1635   //       |   |      |

  1636   //       | Cmp      |\

  1637   //       | /        | \

  1638   //       If         |  \

  1639   //       | \        |   \

  1640   // IfFalse  IfTrue  |    \

  1641   //       \     / \  |     \

  1642   //        \   / StoreCM    |

  1643   //         \ /      |      |

  1644   //        Region   . . .   |

  1645   //          | \           /

  1646   //          |  . . .  \  / Bot

  1647   //          |       MergeMem

  1648   //          |          |

  1649   //       {MemBarCPUOrder}

  1650   //        MemBarVolatile (trailing)

  1651   //

  1652   // The first MergeMem merges the AliasIdxBot Mem slice from the

  1653   // leading membar and the oopptr Mem slice from the Store into the

  1654   // card mark membar. The trailing MergeMem merges the AliasIdxBot

  1655   // Mem slice from the card mark membar and the AliasIdxRaw slice

  1656   // from the StoreCM into the trailing membar (n.b. the latter

  1657   // proceeds via a Phi associated with the If region).

  1658   //

  1659   // The graph for a CAS varies slightly, the difference being

  1660   // that the StoreN/P node is replaced by a CompareAndSwapP/N node

  1661   // and the trailing MemBarVolatile by a MemBarCPUOrder +

  1662   // MemBarAcquire pair (also the MemBarCPUOrder nodes are not optional).

  1663   //

  1664   //   MemBarRelease

  1665   //   MemBarCPUOrder_(leading)_______________

  1666   //     C |    M \       \\                C \

  1667   //       |       \    CompareAndSwapN/P  CastP2X

  1668   //       |        \      |

  1669   //       |         \   SCMemProj

  1670   //       |      Bot \   /

  1671   //       |        MergeMem

  1672   //       |         /

  1673   //      MemBarVolatile (card mark)

  1674   //     C |  ||    M |

  1675   //       | LoadB    |

  1676   //       |   |      |

  1677   //       | Cmp      |\

  1678   //       | /        | \

  1679   //       If         |  \

  1680   //       | \        |   \

  1681   // IfFalse  IfTrue  |    \

  1682   //       \     / \  |     \

  1683   //        \   / StoreCM    |

  1684   //         \ /      |      |

  1685   //        Region   . . .   |

  1686   //          | \           /

  1687   //          |  . . .  \  / Bot

  1688   //          |       MergeMem

  1689   //          |          |

  1690   //        MemBarCPUOrder

  1691   //        MemBarVolatile (trailing)

  1692   //

  1693   //

  1694   // G1 is quite a lot more complicated. The nodes inserted on behalf

  1695   // of G1 may comprise: a pre-write graph which adds the old value to

  1696   // the SATB queue; the releasing store itself; and, finally, a

  1697   // post-write graph which performs a card mark.

  1698   //

  1699   // The pre-write graph may be omitted, but only when the put is

  1700   // writing to a newly allocated (young gen) object and then only if

  1701   // there is a direct memory chain to the Initialize node for the

  1702   // object allocation. This will not happen for a volatile put since

  1703   // any memory chain passes through the leading membar.

  1704   //

  1705   // The pre-write graph includes a series of 3 If tests. The outermost

  1706   // If tests whether SATB is enabled (no else case). The next If tests

  1707   // whether the old value is non-NULL (no else case). The third tests

  1708   // whether the SATB queue index is > 0, if so updating the queue. The

  1709   // else case for this third If calls out to the runtime to allocate a

  1710   // new queue buffer.

  1711   //

  1712   // So with G1 the pre-write and releasing store subgraph looks like

  1713   // this (the nested Ifs are omitted).

  1714   //

  1715   //  MemBarRelease

  1716   // {MemBarCPUOrder}_(leading)___________

  1717   //     C |  ||  M \   M \    M \  M \ . . .

  1718   //       | LoadB   \  LoadL  LoadN   \

  1719   //       | /        \                 \

  1720   //       If         |\                 \

  1721   //       | \        | \                 \

  1722   //  IfFalse  IfTrue |  \                 \

  1723   //       |     |    |   \                 |

  1724   //       |     If   |   /\                |

  1725   //       |     |          \               |

  1726   //       |                 \              |

  1727   //       |    . . .         \             |

  1728   //       | /       | /       |            |

  1729   //      Region  Phi[M]       |            |

  1730   //       | \       |         |            |

  1731   //       |  \_____ | ___     |            |

  1732   //     C | C \     |   C \ M |            |

  1733   //       | CastP2X | StoreN/P[mo_release] |

  1734   //       |         |         |            |

  1735   //     C |       M |       M |          M |

  1736   //        \        |         |           /

  1737   //                  . . .

  1738   //          (post write subtree elided)

  1739   //                    . . .

  1740   //             C \         M /

  1741   //                \         /

  1742   //             {MemBarCPUOrder}

  1743   //              MemBarVolatile (trailing)

  1744   //

  1745   // n.b. the LoadB in this subgraph is not the card read -- it's a

  1746   // read of the SATB queue active flag.

  1747   //

  1748   // The G1 post-write subtree is also optional, this time when the

  1749   // new value being written is either null or can be identified as a

  1750   // newly allocated (young gen) object with no intervening control

  1751   // flow. The latter cannot happen but the former may, in which case

  1752   // the card mark membar is omitted and the memory feeds form the

  1753   // leading membar and the SToreN/P are merged direct into the

  1754   // trailing membar as per the normal subgraph. So, the only special

  1755   // case which arises is when the post-write subgraph is generated.

  1756   //

  1757   // The kernel of the post-write G1 subgraph is the card mark itself

  1758   // which includes a card mark memory barrier (MemBarVolatile), a

  1759   // card test (LoadB), and a conditional update (If feeding a

  1760   // StoreCM). These nodes are surrounded by a series of nested Ifs

  1761   // which try to avoid doing the card mark. The top level If skips if

  1762   // the object reference does not cross regions (i.e. it tests if

  1763   // (adr ^ val) >> log2(regsize) != 0) -- intra-region references

  1764   // need not be recorded. The next If, which skips on a NULL value,

  1765   // may be absent (it is not generated if the type of value is >=

  1766   // OopPtr::NotNull). The 3rd If skips writes to young regions (by

  1767   // checking if card_val != young).  n.b. although this test requires

  1768   // a pre-read of the card it can safely be done before the StoreLoad

  1769   // barrier. However that does not bypass the need to reread the card

  1770   // after the barrier. A final, 4th If tests if the card is already

  1771   // marked.

  1772   //

  1773   //                (pre-write subtree elided)

  1774   //        . . .                  . . .    . . .  . . .

  1775   //        C |                    M |     M |    M |

  1776   //       Region                  Phi[M] StoreN    |

  1777   //          |                     / \      |      |

  1778   //         / \_______            /   \     |      |

  1779   //      C / C \      . . .            \    |      |

  1780   //       If   CastP2X . . .            |   |      |

  1781   //       / \                           |   |      |

  1782   //      /   \                          |   |      |

  1783   // IfFalse IfTrue                      |   |      |

  1784   //   |       |                         |   |     /|

  1785   //   |       If                        |   |    / |

  1786   //   |      / \                        |   |   /  |

  1787   //   |     /   \                        \  |  /   |

  1788   //   | IfFalse IfTrue                   MergeMem  |

  1789   //   |  . . .    / \                       /      |

  1790   //   |          /   \                     /       |

  1791   //   |     IfFalse IfTrue                /        |

  1792   //   |      . . .    |                  /         |

  1793   //   |               If                /          |

  1794   //   |               / \              /           |

  1795   //   |              /   \            /            |

  1796   //   |         IfFalse IfTrue       /             |

  1797   //   |           . . .   |         /              |

  1798   //   |                    \       /               |

  1799   //   |                     \     /                |

  1800   //   |             MemBarVolatile__(card mark)    |

  1801   //   |                ||   C |  M \  M \          |

  1802   //   |               LoadB   If    |    |         |

  1803   //   |                      / \    |    |         |

  1804   //   |                     . . .   |    |         |

  1805   //   |                          \  |    |        /

  1806   //   |                        StoreCM   |       /

  1807   //   |                          . . .   |      /

  1808   //   |                        _________/      /

  1809   //   |                       /  _____________/

  1810   //   |   . . .       . . .  |  /            /

  1811   //   |    |                 | /   _________/

  1812   //   |    |               Phi[M] /        /

  1813   //   |    |                 |   /        /

  1814   //   |    |                 |  /        /

  1815   //   |  Region  . . .     Phi[M]  _____/

  1816   //   |    /                 |    /

  1817   //   |                      |   /

  1818   //   | . . .   . . .        |  /

  1819   //   | /                    | /

  1820   // Region           |  |  Phi[M]

  1821   //   |              |  |  / Bot

  1822   //    \            MergeMem

  1823   //     \            /

  1824   //    {MemBarCPUOrder}

  1825   //     MemBarVolatile

  1826   //

  1827   // As with CMS the initial MergeMem merges the AliasIdxBot Mem slice

  1828   // from the leading membar and the oopptr Mem slice from the Store

  1829   // into the card mark membar i.e. the memory flow to the card mark

  1830   // membar still looks like a normal graph.

  1831   //

  1832   // The trailing MergeMem merges an AliasIdxBot Mem slice with other

  1833   // Mem slices (from the StoreCM and other card mark queue stores).

  1834   // However in this case the AliasIdxBot Mem slice does not come

  1835   // direct from the card mark membar. It is merged through a series

  1836   // of Phi nodes. These are needed to merge the AliasIdxBot Mem flow

  1837   // from the leading membar with the Mem feed from the card mark

  1838   // membar. Each Phi corresponds to one of the Ifs which may skip

  1839   // around the card mark membar. So when the If implementing the NULL

  1840   // value check has been elided the total number of Phis is 2

  1841   // otherwise it is 3.

  1842   //

  1843   // The CAS graph when using G1GC also includes a pre-write subgraph

  1844   // and an optional post-write subgraph. The same variations are

  1845   // introduced as for CMS with conditional card marking i.e. the

  1846   // StoreP/N is swapped for a CompareAndSwapP/N with a following

  1847   // SCMemProj, the trailing MemBarVolatile for a MemBarCPUOrder +

  1848   // MemBarAcquire pair. There may be an extra If test introduced in

  1849   // the CAS case, when the boolean result of the CAS is tested by the

  1850   // caller. In that case an extra Region and AliasIdxBot Phi may be

  1851   // introduced before the MergeMem

  1852   //

  1853   // So, the upshot is that in all cases the subgraph will include a

  1854   // *normal* memory subgraph betwen the leading membar and its child

  1855   // membar: either a normal volatile put graph including a releasing

  1856   // StoreX and terminating with a trailing volatile membar or card

  1857   // mark volatile membar; or a normal CAS graph including a

  1858   // CompareAndSwapX + SCMemProj pair and terminating with a card mark

  1859   // volatile membar or a trailing cpu order and acquire membar

  1860   // pair. If the child membar is not a (volatile) card mark membar

  1861   // then it marks the end of the volatile put or CAS subgraph. If the

  1862   // child is a card mark membar then the normal subgraph will form

  1863   // part of a larger volatile put or CAS subgraph if and only if the

  1864   // child feeds an AliasIdxBot Mem feed to a trailing barrier via a

  1865   // MergeMem. That feed is either direct (for CMS) or via 2, 3 or 4

  1866   // Phi nodes merging the leading barrier memory flow (for G1).

  1867   //

  1868   // The predicates controlling generation of instructions for store

  1869   // and barrier nodes employ a few simple helper functions (described

  1870   // below) which identify the presence or absence of all these

  1871   // subgraph configurations and provide a means of traversing from

  1872   // one node in the subgraph to another.

  1873