author | neliasso |
Thu, 14 Feb 2019 14:54:05 +0100 | |
changeset 55307 | ed12027517c0 |
parent 55123 | 7b70f269fe0d |
child 55321 | ddda023e6f66 |
permissions | -rw-r--r-- |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
1 |
/* |
53223
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
2 |
* Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
4 |
* |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
5 |
* This code is free software; you can redistribute it and/or modify it |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
6 |
* under the terms of the GNU General Public License version 2 only, as |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
7 |
* published by the Free Software Foundation. |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
8 |
* |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
13 |
* accompanied this code). |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
14 |
* |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
15 |
* You should have received a copy of the GNU General Public License version |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation, |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
18 |
* |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
20 |
* or visit www.oracle.com if you need additional information or have any |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
21 |
* questions. |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
22 |
*/ |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
23 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
24 |
#include "precompiled.hpp" |
55307 | 25 |
#include "opto/castnode.hpp" |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
26 |
#include "opto/compile.hpp" |
52568 | 27 |
#include "opto/escape.hpp" |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
28 |
#include "opto/graphKit.hpp" |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
29 |
#include "opto/loopnode.hpp" |
55307 | 30 |
#include "opto/machnode.hpp" |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
31 |
#include "opto/macro.hpp" |
55307 | 32 |
#include "opto/memnode.hpp" |
33 |
#include "opto/movenode.hpp" |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
34 |
#include "opto/node.hpp" |
55307 | 35 |
#include "opto/phase.hpp" |
36 |
#include "opto/phaseX.hpp" |
|
37 |
#include "opto/rootnode.hpp" |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
38 |
#include "opto/type.hpp" |
55307 | 39 |
#include "utilities/copy.hpp" |
40 |
#include "utilities/growableArray.hpp" |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
41 |
#include "utilities/macros.hpp" |
53407 | 42 |
#include "gc/z/zBarrierSet.hpp" |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
43 |
#include "gc/z/c2/zBarrierSetC2.hpp" |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
44 |
#include "gc/z/zThreadLocalData.hpp" |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
45 |
#include "gc/z/zBarrierSetRuntime.hpp" |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
46 |
|
53439 | 47 |
ZBarrierSetC2State::ZBarrierSetC2State(Arena* comp_arena) : |
48 |
_load_barrier_nodes(new (comp_arena) GrowableArray<LoadBarrierNode*>(comp_arena, 8, 0, NULL)) {} |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
49 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
50 |
int ZBarrierSetC2State::load_barrier_count() const { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
51 |
return _load_barrier_nodes->length(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
52 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
53 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
54 |
void ZBarrierSetC2State::add_load_barrier_node(LoadBarrierNode * n) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
55 |
assert(!_load_barrier_nodes->contains(n), " duplicate entry in expand list"); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
56 |
_load_barrier_nodes->append(n); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
57 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
58 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
59 |
void ZBarrierSetC2State::remove_load_barrier_node(LoadBarrierNode * n) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
60 |
// this function may be called twice for a node so check |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
61 |
// that the node is in the array before attempting to remove it |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
62 |
if (_load_barrier_nodes->contains(n)) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
63 |
_load_barrier_nodes->remove(n); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
64 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
65 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
66 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
67 |
LoadBarrierNode* ZBarrierSetC2State::load_barrier_node(int idx) const { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
68 |
return _load_barrier_nodes->at(idx); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
69 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
70 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
71 |
void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
72 |
return new(comp_arena) ZBarrierSetC2State(comp_arena); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
73 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
74 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
75 |
ZBarrierSetC2State* ZBarrierSetC2::state() const { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
76 |
return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state()); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
77 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
78 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
79 |
bool ZBarrierSetC2::is_gc_barrier_node(Node* node) const { |
51485
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
80 |
// 1. This step follows potential oop projections of a load barrier before expansion |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
81 |
if (node->is_Proj()) { |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
82 |
node = node->in(0); |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
83 |
} |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
84 |
|
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
85 |
// 2. This step checks for unexpanded load barriers |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
86 |
if (node->is_LoadBarrier()) { |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
87 |
return true; |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
88 |
} |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
89 |
|
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
90 |
// 3. This step checks for the phi corresponding to an optimized load barrier expansion |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
91 |
if (node->is_Phi()) { |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
92 |
PhiNode* phi = node->as_Phi(); |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
93 |
Node* n = phi->in(1); |
55307 | 94 |
if (n != NULL && n->is_LoadBarrierSlowReg()) { |
51485
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
95 |
return true; |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
96 |
} |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
97 |
} |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
98 |
|
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
99 |
return false; |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
100 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
101 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
102 |
void ZBarrierSetC2::register_potential_barrier_node(Node* node) const { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
103 |
if (node->is_LoadBarrier()) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
104 |
state()->add_load_barrier_node(node->as_LoadBarrier()); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
105 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
106 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
107 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
108 |
void ZBarrierSetC2::unregister_potential_barrier_node(Node* node) const { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
109 |
if (node->is_LoadBarrier()) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
110 |
state()->remove_load_barrier_node(node->as_LoadBarrier()); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
111 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
112 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
113 |
|
52224
4f2215a00ed1
8212611: Small collection of simple changes from shenandoah
roland
parents:
52036
diff
changeset
|
114 |
void ZBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const { |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
115 |
// Remove useless LoadBarrier nodes |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
116 |
ZBarrierSetC2State* s = state(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
117 |
for (int i = s->load_barrier_count()-1; i >= 0; i--) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
118 |
LoadBarrierNode* n = s->load_barrier_node(i); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
119 |
if (!useful.member(n)) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
120 |
unregister_potential_barrier_node(n); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
121 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
122 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
123 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
124 |
|
52224
4f2215a00ed1
8212611: Small collection of simple changes from shenandoah
roland
parents:
52036
diff
changeset
|
125 |
void ZBarrierSetC2::enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const { |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
126 |
if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) { |
52224
4f2215a00ed1
8212611: Small collection of simple changes from shenandoah
roland
parents:
52036
diff
changeset
|
127 |
igvn->_worklist.push(node); |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
128 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
129 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
130 |
|
55307 | 131 |
static bool load_require_barrier(LoadNode* load) { return ((load->barrier_data() & RequireBarrier) != 0); } |
132 |
static bool load_has_weak_barrier(LoadNode* load) { return ((load->barrier_data() & WeakBarrier) != 0); } |
|
133 |
static bool load_has_expanded_barrier(LoadNode* load) { return ((load->barrier_data() & ExpandedBarrier) != 0); } |
|
134 |
static void load_set_expanded_barrier(LoadNode* load) { return load->set_barrier_data(ExpandedBarrier); } |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
135 |
|
55307 | 136 |
static void load_set_barrier(LoadNode* load, bool weak) { |
137 |
if (weak) { |
|
138 |
load->set_barrier_data(WeakBarrier); |
|
139 |
} else { |
|
140 |
load->set_barrier_data(RequireBarrier); |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
141 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
142 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
143 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
144 |
// == LoadBarrierNode == |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
145 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
146 |
LoadBarrierNode::LoadBarrierNode(Compile* C, |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
147 |
Node* c, |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
148 |
Node* mem, |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
149 |
Node* val, |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
150 |
Node* adr, |
55307 | 151 |
bool weak) : |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
152 |
MultiNode(Number_of_Inputs), |
55307 | 153 |
_weak(weak) { |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
154 |
init_req(Control, c); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
155 |
init_req(Memory, mem); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
156 |
init_req(Oop, val); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
157 |
init_req(Address, adr); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
158 |
init_req(Similar, C->top()); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
159 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
160 |
init_class_id(Class_LoadBarrier); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
161 |
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
162 |
bs->register_potential_barrier_node(this); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
163 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
164 |
|
53223
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
165 |
uint LoadBarrierNode::size_of() const { |
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
166 |
return sizeof(*this); |
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
167 |
} |
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
168 |
|
54327 | 169 |
bool LoadBarrierNode::cmp(const Node& n) const { |
53223
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
170 |
ShouldNotReachHere(); |
54327 | 171 |
return false; |
53223
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
172 |
} |
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
173 |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
174 |
const Type *LoadBarrierNode::bottom_type() const { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
175 |
const Type** floadbarrier = (const Type **)(Compile::current()->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
176 |
Node* in_oop = in(Oop); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
177 |
floadbarrier[Control] = Type::CONTROL; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
178 |
floadbarrier[Memory] = Type::MEMORY; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
179 |
floadbarrier[Oop] = in_oop == NULL ? Type::TOP : in_oop->bottom_type(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
180 |
return TypeTuple::make(Number_of_Outputs, floadbarrier); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
181 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
182 |
|
53223
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
183 |
const TypePtr* LoadBarrierNode::adr_type() const { |
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
184 |
ShouldNotReachHere(); |
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
185 |
return NULL; |
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
186 |
} |
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
187 |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
188 |
const Type *LoadBarrierNode::Value(PhaseGVN *phase) const { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
189 |
const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
190 |
const Type* val_t = phase->type(in(Oop)); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
191 |
floadbarrier[Control] = Type::CONTROL; |
55307 | 192 |
floadbarrier[Memory] = Type::MEMORY; |
193 |
floadbarrier[Oop] = val_t; |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
194 |
return TypeTuple::make(Number_of_Outputs, floadbarrier); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
195 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
196 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
197 |
bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
198 |
if (phase != NULL) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
199 |
return phase->is_dominator(d, n); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
200 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
201 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
202 |
for (int i = 0; i < 10 && n != NULL; i++) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
203 |
n = IfNode::up_one_dom(n, linear_only); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
204 |
if (n == d) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
205 |
return true; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
206 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
207 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
208 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
209 |
return false; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
210 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
211 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
212 |
LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) { |
55307 | 213 |
if (is_weak()) { |
214 |
// Weak barriers can't be eliminated |
|
215 |
return NULL; |
|
216 |
} |
|
217 |
||
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
218 |
Node* val = in(LoadBarrierNode::Oop); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
219 |
if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
220 |
LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
221 |
assert(lb->in(Address) == in(Address), ""); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
222 |
// Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier. |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
223 |
if (lb->in(Oop) == in(Oop)) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
224 |
return lb; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
225 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
226 |
// Follow chain of load barrier through Similar edges |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
227 |
while (!lb->in(Similar)->is_top()) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
228 |
lb = lb->in(Similar)->in(0)->as_LoadBarrier(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
229 |
assert(lb->in(Address) == in(Address), ""); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
230 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
231 |
if (lb != in(Similar)->in(0)) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
232 |
return lb; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
233 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
234 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
235 |
for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
236 |
Node* u = val->fast_out(i); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
237 |
if (u != this && u->is_LoadBarrier() && u->in(Oop) == val && u->as_LoadBarrier()->has_true_uses()) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
238 |
Node* this_ctrl = in(LoadBarrierNode::Control); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
239 |
Node* other_ctrl = u->in(LoadBarrierNode::Control); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
240 |
if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
241 |
return u->as_LoadBarrier(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
242 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
243 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
244 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
245 |
|
55307 | 246 |
if (can_be_eliminated()) { |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
247 |
return NULL; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
248 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
249 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
250 |
if (!look_for_similar) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
251 |
return NULL; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
252 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
253 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
254 |
Node* addr = in(LoadBarrierNode::Address); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
255 |
for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
256 |
Node* u = addr->fast_out(i); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
257 |
if (u != this && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
258 |
Node* this_ctrl = in(LoadBarrierNode::Control); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
259 |
Node* other_ctrl = u->in(LoadBarrierNode::Control); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
260 |
if (is_dominator(phase, linear_only, other_ctrl, this_ctrl)) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
261 |
ResourceMark rm; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
262 |
Unique_Node_List wq; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
263 |
wq.push(in(LoadBarrierNode::Control)); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
264 |
bool ok = true; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
265 |
bool dom_found = false; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
266 |
for (uint next = 0; next < wq.size(); ++next) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
267 |
Node *n = wq.at(next); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
268 |
if (n->is_top()) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
269 |
return NULL; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
270 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
271 |
assert(n->is_CFG(), ""); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
272 |
if (n->is_SafePoint()) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
273 |
ok = false; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
274 |
break; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
275 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
276 |
if (n == u) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
277 |
dom_found = true; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
278 |
continue; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
279 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
280 |
if (n->is_Region()) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
281 |
for (uint i = 1; i < n->req(); i++) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
282 |
Node* m = n->in(i); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
283 |
if (m != NULL) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
284 |
wq.push(m); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
285 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
286 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
287 |
} else { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
288 |
Node* m = n->in(0); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
289 |
if (m != NULL) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
290 |
wq.push(m); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
291 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
292 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
293 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
294 |
if (ok) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
295 |
assert(dom_found, ""); |
55307 | 296 |
return u->as_LoadBarrier(); |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
297 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
298 |
break; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
299 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
300 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
301 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
302 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
303 |
return NULL; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
304 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
305 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
306 |
void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const { |
50875
2217b2fc29ea
8205993: ZGC: Fix typos and incorrect indentations
pliden
parents:
50803
diff
changeset
|
307 |
// Change to that barrier may affect a dominated barrier so re-push those |
55307 | 308 |
assert(!is_weak(), "sanity"); |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
309 |
Node* val = in(LoadBarrierNode::Oop); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
310 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
311 |
for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
312 |
Node* u = val->fast_out(i); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
313 |
if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
314 |
Node* this_ctrl = in(Control); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
315 |
Node* other_ctrl = u->in(Control); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
316 |
if (is_dominator(NULL, false, this_ctrl, other_ctrl)) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
317 |
igvn->_worklist.push(u); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
318 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
319 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
320 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
321 |
Node* addr = in(LoadBarrierNode::Address); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
322 |
for (DUIterator_Fast imax, i = addr->fast_outs(imax); i < imax; i++) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
323 |
Node* u = addr->fast_out(i); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
324 |
if (u != this && u->is_LoadBarrier() && u->in(Similar)->is_top()) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
325 |
Node* this_ctrl = in(Control); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
326 |
Node* other_ctrl = u->in(Control); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
327 |
if (is_dominator(NULL, false, this_ctrl, other_ctrl)) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
328 |
igvn->_worklist.push(u); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
329 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
330 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
331 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
332 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
333 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
334 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
335 |
Node *LoadBarrierNode::Identity(PhaseGVN *phase) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
336 |
LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
337 |
if (dominating_barrier != NULL) { |
55307 | 338 |
assert(!is_weak(), "Weak barriers cant be eliminated"); |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
339 |
assert(dominating_barrier->in(Oop) == in(Oop), ""); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
340 |
return dominating_barrier; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
341 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
342 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
343 |
return this; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
344 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
345 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
346 |
Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
347 |
if (remove_dead_region(phase, can_reshape)) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
348 |
return this; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
349 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
350 |
|
55307 | 351 |
Node *val = in(Oop); |
352 |
Node *mem = in(Memory); |
|
353 |
Node *ctrl = in(Control); |
|
354 |
||
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
355 |
assert(val->Opcode() != Op_LoadN, ""); |
55307 | 356 |
assert(val->Opcode() != Op_DecodeN, ""); |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
357 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
358 |
if (mem->is_MergeMem()) { |
55307 | 359 |
Node *new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
360 |
set_req(Memory, new_mem); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
361 |
if (mem->outcnt() == 0 && can_reshape) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
362 |
phase->is_IterGVN()->_worklist.push(mem); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
363 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
364 |
return this; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
365 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
366 |
|
55307 | 367 |
LoadBarrierNode *dominating_barrier = NULL; |
368 |
if (!is_weak()) { |
|
369 |
dominating_barrier = has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()); |
|
370 |
if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) { |
|
371 |
assert(in(Address) == dominating_barrier->in(Address), ""); |
|
372 |
set_req(Similar, dominating_barrier->proj_out(Oop)); |
|
373 |
return this; |
|
374 |
} |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
375 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
376 |
|
55307 | 377 |
bool eliminate = can_reshape && (dominating_barrier != NULL || !has_true_uses()); |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
378 |
if (eliminate) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
379 |
if (can_reshape) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
380 |
PhaseIterGVN* igvn = phase->is_IterGVN(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
381 |
Node* out_ctrl = proj_out_or_null(Control); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
382 |
Node* out_res = proj_out_or_null(Oop); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
383 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
384 |
if (out_ctrl != NULL) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
385 |
igvn->replace_node(out_ctrl, ctrl); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
386 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
387 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
388 |
// That transformation may cause the Similar edge on the load barrier to be invalid |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
389 |
fix_similar_in_uses(igvn); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
390 |
if (out_res != NULL) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
391 |
if (dominating_barrier != NULL) { |
55307 | 392 |
assert(!is_weak(), "Sanity"); |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
393 |
igvn->replace_node(out_res, dominating_barrier->proj_out(Oop)); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
394 |
} else { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
395 |
igvn->replace_node(out_res, val); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
396 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
397 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
398 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
399 |
return new ConINode(TypeInt::ZERO); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
400 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
401 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
402 |
// If the Similar edge is no longer a load barrier, clear it |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
403 |
Node* similar = in(Similar); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
404 |
if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
405 |
set_req(Similar, phase->C->top()); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
406 |
return this; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
407 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
408 |
|
55307 | 409 |
if (can_reshape && !is_weak()) { |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
410 |
// If this barrier is linked through the Similar edge by a |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
411 |
// dominated barrier and both barriers have the same Oop field, |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
412 |
// the dominated barrier can go away, so push it for reprocessing. |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
413 |
// We also want to avoid a barrier to depend on another dominating |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
414 |
// barrier through its Similar edge that itself depend on another |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
415 |
// barrier through its Similar edge and rather have the first |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
416 |
// depend on the third. |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
417 |
PhaseIterGVN* igvn = phase->is_IterGVN(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
418 |
Node* out_res = proj_out(Oop); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
419 |
for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
420 |
Node* u = out_res->fast_out(i); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
421 |
if (u->is_LoadBarrier() && u->in(Similar) == out_res && |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
422 |
(u->in(Oop) == val || !u->in(Similar)->is_top())) { |
55307 | 423 |
assert(!u->as_LoadBarrier()->is_weak(), "Sanity"); |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
424 |
igvn->_worklist.push(u); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
425 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
426 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
427 |
push_dominated_barriers(igvn); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
428 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
429 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
430 |
return NULL; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
431 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
432 |
|
53223
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
433 |
uint LoadBarrierNode::match_edge(uint idx) const { |
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
434 |
ShouldNotReachHere(); |
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
435 |
return 0; |
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
436 |
} |
df6cbf676c70
8215708: ZGC: Add missing LoadBarrierNode::size_of()
pliden
parents:
52829
diff
changeset
|
437 |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
438 |
void LoadBarrierNode::fix_similar_in_uses(PhaseIterGVN* igvn) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
439 |
Node* out_res = proj_out_or_null(Oop); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
440 |
if (out_res == NULL) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
441 |
return; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
442 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
443 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
444 |
for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
445 |
Node* u = out_res->fast_out(i); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
446 |
if (u->is_LoadBarrier() && u->in(Similar) == out_res) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
447 |
igvn->replace_input_of(u, Similar, igvn->C->top()); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
448 |
--i; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
449 |
--imax; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
450 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
451 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
452 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
453 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
454 |
bool LoadBarrierNode::has_true_uses() const { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
455 |
Node* out_res = proj_out_or_null(Oop); |
55307 | 456 |
if (out_res != NULL) { |
457 |
for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { |
|
458 |
Node *u = out_res->fast_out(i); |
|
459 |
if (!u->is_LoadBarrier() || u->in(Similar) != out_res) { |
|
460 |
return true; |
|
461 |
} |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
462 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
463 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
464 |
return false; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
465 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
466 |
|
52424 | 467 |
static bool barrier_needed(C2Access& access) { |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
468 |
return ZBarrierSet::barrier_needed(access.decorators(), access.type()); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
469 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
470 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
471 |
Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
472 |
Node* p = BarrierSetC2::load_at_resolved(access, val_type); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
473 |
if (!barrier_needed(access)) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
474 |
return p; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
475 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
476 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
477 |
bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0; |
55307 | 478 |
if (p->isa_Load()) { |
479 |
load_set_barrier(p->as_Load(), weak); |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
480 |
} |
55307 | 481 |
return p; |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
482 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
483 |
|
52424 | 484 |
Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
485 |
Node* new_val, const Type* val_type) const { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
486 |
Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); |
55307 | 487 |
LoadStoreNode* lsn = result->as_LoadStore(); |
488 |
if (barrier_needed(access)) { |
|
489 |
lsn->set_has_barrier(); |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
490 |
} |
55307 | 491 |
return lsn; |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
492 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
493 |
|
52424 | 494 |
Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
495 |
Node* new_val, const Type* value_type) const { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
496 |
Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); |
55307 | 497 |
LoadStoreNode* lsn = result->as_LoadStore(); |
498 |
if (barrier_needed(access)) { |
|
499 |
lsn->set_has_barrier(); |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
500 |
} |
55307 | 501 |
return lsn; |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
502 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
503 |
|
52424 | 504 |
Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const { |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
505 |
Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); |
55307 | 506 |
LoadStoreNode* lsn = result->as_LoadStore(); |
507 |
if (barrier_needed(access)) { |
|
508 |
lsn->set_has_barrier(); |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
509 |
} |
55307 | 510 |
return lsn; |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
511 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
512 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
513 |
// == Macro Expansion == |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
514 |
|
55307 | 515 |
// Optimized, low spill, loadbarrier variant using stub specialized on register used |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
516 |
void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
517 |
PhaseIterGVN &igvn = phase->igvn(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
518 |
float unlikely = PROB_UNLIKELY(0.999); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
519 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
520 |
Node* in_ctrl = barrier->in(LoadBarrierNode::Control); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
521 |
Node* in_mem = barrier->in(LoadBarrierNode::Memory); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
522 |
Node* in_val = barrier->in(LoadBarrierNode::Oop); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
523 |
Node* in_adr = barrier->in(LoadBarrierNode::Address); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
524 |
|
55307 | 525 |
Node* out_ctrl = barrier->proj_out_or_null(LoadBarrierNode::Control); |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
526 |
Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
527 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
528 |
assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null"); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
529 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
530 |
Node* jthread = igvn.transform(new ThreadLocalNode()); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
531 |
Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset())); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
532 |
Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
533 |
TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
534 |
MemNode::unordered)); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
535 |
Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val)); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
536 |
Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask)); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
537 |
Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type()))); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
538 |
Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
539 |
IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
540 |
Node* then = igvn.transform(new IfTrueNode(iff)); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
541 |
Node* elsen = igvn.transform(new IfFalseNode(iff)); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
542 |
|
55307 | 543 |
Node* new_loadp = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), |
544 |
(const TypePtr*) in_val->bottom_type(), MemNode::unordered, barrier->is_weak())); |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
545 |
|
50875
2217b2fc29ea
8205993: ZGC: Fix typos and incorrect indentations
pliden
parents:
50803
diff
changeset
|
546 |
// Create the final region/phi pair to converge cntl/data paths to downstream code |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
547 |
Node* result_region = igvn.transform(new RegionNode(3)); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
548 |
result_region->set_req(1, then); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
549 |
result_region->set_req(2, elsen); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
550 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
551 |
Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM)); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
552 |
result_phi->set_req(1, new_loadp); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
553 |
result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop)); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
554 |
|
55307 | 555 |
if (out_ctrl != NULL) { |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
556 |
igvn.replace_node(out_ctrl, result_region); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
557 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
558 |
igvn.replace_node(out_res, result_phi); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
559 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
560 |
assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!"); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
561 |
|
55307 | 562 |
igvn.remove_dead_node(barrier); |
563 |
igvn.remove_dead_node(out_ctrl); |
|
564 |
igvn.remove_dead_node(out_res); |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
565 |
|
51485
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
566 |
assert(is_gc_barrier_node(result_phi), "sanity"); |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
567 |
assert(step_over_gc_barrier(result_phi) == in_val, "sanity"); |
55307 | 568 |
|
569 |
phase->C->print_method(PHASE_BARRIER_EXPANSION, 4, barrier->_idx); |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
570 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
571 |
|
52711 | 572 |
bool ZBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const { |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
573 |
ZBarrierSetC2State* s = state(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
574 |
if (s->load_barrier_count() > 0) { |
52829
af52abc1f61e
8214541: ZGC: Refactoring from JDK-8214172 may leave PhaseIterGVN::_delay_transform set
roland
parents:
52759
diff
changeset
|
575 |
PhaseMacroExpand macro(igvn); |
55307 | 576 |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
577 |
int skipped = 0; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
578 |
while (s->load_barrier_count() > skipped) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
579 |
int load_barrier_count = s->load_barrier_count(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
580 |
LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
581 |
if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) { |
50875
2217b2fc29ea
8205993: ZGC: Fix typos and incorrect indentations
pliden
parents:
50803
diff
changeset
|
582 |
// Node is unreachable, so don't try to expand it |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
583 |
s->remove_load_barrier_node(n); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
584 |
continue; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
585 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
586 |
if (!n->can_be_eliminated()) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
587 |
skipped++; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
588 |
continue; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
589 |
} |
52711 | 590 |
expand_loadbarrier_node(¯o, n); |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
591 |
assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list"); |
53439 | 592 |
if (C->failing()) { |
593 |
return true; |
|
594 |
} |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
595 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
596 |
while (s->load_barrier_count() > 0) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
597 |
int load_barrier_count = s->load_barrier_count(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
598 |
LoadBarrierNode* n = s->load_barrier_node(load_barrier_count - 1); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
599 |
assert(!(igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())), "should have been processed already"); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
600 |
assert(!n->can_be_eliminated(), "should have been processed already"); |
52711 | 601 |
expand_loadbarrier_node(¯o, n); |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
602 |
assert(s->load_barrier_count() < load_barrier_count, "must have deleted a node from load barrier list"); |
53439 | 603 |
if (C->failing()) { |
604 |
return true; |
|
605 |
} |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
606 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
607 |
igvn.set_delay_transform(false); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
608 |
igvn.optimize(); |
53439 | 609 |
if (C->failing()) { |
610 |
return true; |
|
611 |
} |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
612 |
} |
53439 | 613 |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
614 |
return false; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
615 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
616 |
|
51485
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
617 |
Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const { |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
618 |
Node* node = c; |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
619 |
|
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
620 |
// 1. This step follows potential oop projections of a load barrier before expansion |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
621 |
if (node->is_Proj()) { |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
622 |
node = node->in(0); |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
623 |
} |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
624 |
|
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
625 |
// 2. This step checks for unexpanded load barriers |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
626 |
if (node->is_LoadBarrier()) { |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
627 |
return node->in(LoadBarrierNode::Oop); |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
628 |
} |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
629 |
|
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
630 |
// 3. This step checks for the phi corresponding to an optimized load barrier expansion |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
631 |
if (node->is_Phi()) { |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
632 |
PhiNode* phi = node->as_Phi(); |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
633 |
Node* n = phi->in(1); |
55307 | 634 |
if (n != NULL && n->is_LoadBarrierSlowReg()) { |
51485
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
635 |
assert(c == node, "projections from step 1 should only be seen before macro expansion"); |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
636 |
return phi->in(2); |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
637 |
} |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
638 |
} |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
639 |
|
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
640 |
return c; |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
641 |
} |
0c7040d1d1ca
8208601: Introduce native oop barriers in C2 for OopHandle
eosterlund
parents:
50875
diff
changeset
|
642 |
|
52036
5d6d636cefff
8211776: 8210887 broke arraycopy optimization when ZGC is enabled
roland
parents:
51489
diff
changeset
|
643 |
bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const { |
5d6d636cefff
8211776: 8210887 broke arraycopy optimization when ZGC is enabled
roland
parents:
51489
diff
changeset
|
644 |
return type == T_OBJECT || type == T_ARRAY; |
5d6d636cefff
8211776: 8210887 broke arraycopy optimization when ZGC is enabled
roland
parents:
51489
diff
changeset
|
645 |
} |
5d6d636cefff
8211776: 8210887 broke arraycopy optimization when ZGC is enabled
roland
parents:
51489
diff
changeset
|
646 |
|
52471
04d7e790aa2e
8213489: GC/C2 abstraction for Compile::final_graph_reshaping()
rkennke
parents:
52426
diff
changeset
|
647 |
bool ZBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const { |
55307 | 648 |
switch (opcode) { |
649 |
case Op_LoadBarrier: |
|
650 |
assert(0, "There should be no load barriers left"); |
|
651 |
case Op_ZGetAndSetP: |
|
652 |
case Op_ZCompareAndExchangeP: |
|
653 |
case Op_ZCompareAndSwapP: |
|
654 |
case Op_ZWeakCompareAndSwapP: |
|
655 |
case Op_LoadBarrierSlowReg: |
|
52471
04d7e790aa2e
8213489: GC/C2 abstraction for Compile::final_graph_reshaping()
rkennke
parents:
52426
diff
changeset
|
656 |
#ifdef ASSERT |
55307 | 657 |
if (VerifyOptoOopOffsets) { |
658 |
MemNode *mem = n->as_Mem(); |
|
659 |
// Check to see if address types have grounded out somehow. |
|
660 |
const TypeInstPtr *tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr(); |
|
661 |
ciInstanceKlass *k = tp->klass()->as_instance_klass(); |
|
662 |
bool oop_offset_is_sane = k->contains_field_offset(tp->offset()); |
|
663 |
assert(!tp || oop_offset_is_sane, ""); |
|
664 |
} |
|
665 |
#endif |
|
666 |
return true; |
|
667 |
default: |
|
668 |
return false; |
|
53439 | 669 |
} |
52471
04d7e790aa2e
8213489: GC/C2 abstraction for Compile::final_graph_reshaping()
rkennke
parents:
52426
diff
changeset
|
670 |
} |
04d7e790aa2e
8213489: GC/C2 abstraction for Compile::final_graph_reshaping()
rkennke
parents:
52426
diff
changeset
|
671 |
|
52627 | 672 |
bool ZBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const { |
55307 | 673 |
switch(opcode) { |
674 |
case Op_CallLeaf: |
|
675 |
if (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() || |
|
676 |
n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr()) { |
|
677 |
mem_op = true; |
|
678 |
mem_addr_idx = TypeFunc::Parms + 1; |
|
679 |
return true; |
|
680 |
} |
|
681 |
return false; |
|
682 |
default: |
|
683 |
return false; |
|
52627 | 684 |
} |
55307 | 685 |
} |
53439 | 686 |
|
55307 | 687 |
bool ZBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const { |
688 |
switch(opcode) { |
|
689 |
case Op_ZCompareAndExchangeP: |
|
690 |
case Op_ZCompareAndSwapP: |
|
691 |
case Op_ZWeakCompareAndSwapP: { |
|
692 |
Node *mem = n->in(MemNode::Address); |
|
693 |
Node *keepalive = n->in(5); |
|
694 |
Node *pair1 = new BinaryNode(mem, keepalive); |
|
695 |
||
696 |
Node *newval = n->in(MemNode::ValueIn); |
|
697 |
Node *oldval = n->in(LoadStoreConditionalNode::ExpectedIn); |
|
698 |
Node *pair2 = new BinaryNode(oldval, newval); |
|
699 |
||
700 |
n->set_req(MemNode::Address, pair1); |
|
701 |
n->set_req(MemNode::ValueIn, pair2); |
|
702 |
n->del_req(5); |
|
703 |
n->del_req(LoadStoreConditionalNode::ExpectedIn); |
|
704 |
return true; |
|
705 |
} |
|
706 |
case Op_ZGetAndSetP: { |
|
707 |
Node *keepalive = n->in(4); |
|
708 |
Node *newval = n->in(MemNode::ValueIn); |
|
709 |
Node *pair = new BinaryNode(newval, keepalive); |
|
710 |
n->set_req(MemNode::ValueIn, pair); |
|
711 |
n->del_req(4); |
|
712 |
return true; |
|
713 |
} |
|
714 |
||
715 |
default: |
|
716 |
return false; |
|
717 |
} |
|
52627 | 718 |
} |
719 |
||
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
720 |
// == Verification == |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
721 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
722 |
#ifdef ASSERT |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
723 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
724 |
static bool look_for_barrier(Node* n, bool post_parse, VectorSet& visited) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
725 |
if (visited.test_set(n->_idx)) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
726 |
return true; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
727 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
728 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
729 |
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
730 |
Node* u = n->fast_out(i); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
731 |
if (u->is_LoadBarrier()) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
732 |
} else if ((u->is_Phi() || u->is_CMove()) && !post_parse) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
733 |
if (!look_for_barrier(u, post_parse, visited)) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
734 |
return false; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
735 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
736 |
} else if (u->Opcode() == Op_EncodeP || u->Opcode() == Op_DecodeN) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
737 |
if (!look_for_barrier(u, post_parse, visited)) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
738 |
return false; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
739 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
740 |
} else if (u->Opcode() != Op_SCMemProj) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
741 |
tty->print("bad use"); u->dump(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
742 |
return false; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
743 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
744 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
745 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
746 |
return true; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
747 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
748 |
|
52426
38bf0c9c4e64
8213384: Move G1/C2 barrier verification into G1BarrierSetC2
rkennke
parents:
52424
diff
changeset
|
749 |
void ZBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const { |
55307 | 750 |
switch(phase) { |
751 |
case BarrierSetC2::BeforeOptimize: |
|
752 |
case BarrierSetC2::BeforeLateInsertion: |
|
753 |
assert(state()->load_barrier_count() == 0, "No barriers inserted yet"); |
|
754 |
break; |
|
755 |
case BarrierSetC2::BeforeMacroExpand: |
|
756 |
// Barrier placement should be set by now. |
|
757 |
verify_gc_barriers(false /*post_parse*/); |
|
758 |
break; |
|
759 |
case BarrierSetC2::BeforeCodeGen: |
|
760 |
// Barriers has been fully expanded. |
|
761 |
assert(state()->load_barrier_count() == 0, "No more macro barriers"); |
|
762 |
break; |
|
763 |
default: |
|
764 |
assert(0, "Phase without verification"); |
|
765 |
} |
|
52426
38bf0c9c4e64
8213384: Move G1/C2 barrier verification into G1BarrierSetC2
rkennke
parents:
52424
diff
changeset
|
766 |
} |
38bf0c9c4e64
8213384: Move G1/C2 barrier verification into G1BarrierSetC2
rkennke
parents:
52424
diff
changeset
|
767 |
|
55307 | 768 |
// post_parse implies that there might be load barriers without uses after parsing |
769 |
// That only applies when adding barriers at parse time. |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
770 |
void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
771 |
ZBarrierSetC2State* s = state(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
772 |
Compile* C = Compile::current(); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
773 |
ResourceMark rm; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
774 |
VectorSet visited(Thread::current()->resource_area()); |
55307 | 775 |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
776 |
for (int i = 0; i < s->load_barrier_count(); i++) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
777 |
LoadBarrierNode* n = s->load_barrier_node(i); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
778 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
779 |
// The dominating barrier on the same address if it exists and |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
780 |
// this barrier must not be applied on the value from the same |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
781 |
// load otherwise the value is not reloaded before it's used the |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
782 |
// second time. |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
783 |
assert(n->in(LoadBarrierNode::Similar)->is_top() || |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
784 |
(n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() && |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
785 |
n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) && |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
786 |
n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)), |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
787 |
"broken similar edge"); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
788 |
|
55307 | 789 |
assert(n->as_LoadBarrier()->has_true_uses(), |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
790 |
"found unneeded load barrier"); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
791 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
792 |
// Several load barrier nodes chained through their Similar edge |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
793 |
// break the code that remove the barriers in final graph reshape. |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
794 |
assert(n->in(LoadBarrierNode::Similar)->is_top() || |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
795 |
(n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() && |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
796 |
n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Similar)->is_top()), |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
797 |
"chain of Similar load barriers"); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
798 |
|
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
799 |
if (!n->in(LoadBarrierNode::Similar)->is_top()) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
800 |
ResourceMark rm; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
801 |
Unique_Node_List wq; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
802 |
Node* other = n->in(LoadBarrierNode::Similar)->in(0); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
803 |
wq.push(n); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
804 |
for (uint next = 0; next < wq.size(); ++next) { |
55307 | 805 |
Node *nn = wq.at(next); |
806 |
assert(nn->is_CFG(), ""); |
|
807 |
assert(!nn->is_SafePoint(), ""); |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
808 |
|
55307 | 809 |
if (nn == other) { |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
810 |
continue; |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
811 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
812 |
|
55307 | 813 |
if (nn->is_Region()) { |
814 |
for (uint i = 1; i < nn->req(); i++) { |
|
815 |
Node* m = nn->in(i); |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
816 |
if (m != NULL) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
817 |
wq.push(m); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
818 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
819 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
820 |
} else { |
55307 | 821 |
Node* m = nn->in(0); |
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
822 |
if (m != NULL) { |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
823 |
wq.push(m); |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
824 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
825 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
826 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
827 |
} |
55307 | 828 |
} |
829 |
} |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
830 |
|
55307 | 831 |
#endif // end verification code |
832 |
||
833 |
static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl); |
|
834 |
||
835 |
// This code is cloning all uses of a load that is between a call and the catch blocks, |
|
836 |
// to each use. |
|
837 |
||
838 |
static bool fixup_uses_in_catch(PhaseIdealLoop *phase, Node *start_ctrl, Node *node) { |
|
839 |
||
840 |
if (!phase->has_ctrl(node)) { |
|
841 |
// This node is floating - doesn't need to be cloned. |
|
842 |
assert(node != start_ctrl, "check"); |
|
843 |
return false; |
|
844 |
} |
|
845 |
||
846 |
Node* ctrl = phase->get_ctrl(node); |
|
847 |
if (ctrl != start_ctrl) { |
|
848 |
// We are in a successor block - the node is ok. |
|
849 |
return false; // Unwind |
|
850 |
} |
|
851 |
||
852 |
// Process successor nodes |
|
853 |
int outcnt = node->outcnt(); |
|
854 |
for (int i = 0; i < outcnt; i++) { |
|
855 |
Node* n = node->raw_out(0); |
|
856 |
assert(!n->is_LoadBarrier(), "Sanity"); |
|
857 |
// Calling recursively, visiting leafs first |
|
858 |
fixup_uses_in_catch(phase, start_ctrl, n); |
|
859 |
} |
|
860 |
||
861 |
// Now all successors are outside |
|
862 |
// - Clone this node to both successors |
|
863 |
int no_succs = node->outcnt(); |
|
864 |
assert(!node->is_Store(), "Stores not expected here"); |
|
865 |
||
866 |
// In some very rare cases a load that doesn't need a barrier will end up here |
|
867 |
// Treat it as a LoadP and the insertion of phis will be done correctly. |
|
868 |
if (node->is_Load()) { |
|
869 |
assert(node->as_Load()->barrier_data() == 0, "Sanity"); |
|
870 |
call_catch_cleanup_one(phase, node->as_Load(), phase->get_ctrl(node)); |
|
871 |
} else { |
|
872 |
for (DUIterator_Fast jmax, i = node->fast_outs(jmax); i < jmax; i++) { |
|
873 |
Node* use = node->fast_out(i); |
|
874 |
Node* clone = node->clone(); |
|
875 |
assert(clone->outcnt() == 0, ""); |
|
876 |
||
877 |
assert(use->find_edge(node) != -1, "check"); |
|
878 |
phase->igvn().rehash_node_delayed(use); |
|
879 |
use->replace_edge(node, clone); |
|
880 |
||
881 |
Node* new_ctrl; |
|
882 |
if (use->is_block_start()) { |
|
883 |
new_ctrl = use; |
|
884 |
} else if (use->is_CFG()) { |
|
885 |
new_ctrl = use->in(0); |
|
886 |
assert (new_ctrl != NULL, ""); |
|
887 |
} else { |
|
888 |
new_ctrl = phase->get_ctrl(use); |
|
889 |
} |
|
890 |
||
891 |
phase->set_ctrl(clone, new_ctrl); |
|
892 |
||
893 |
if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr(" Clone op %i as %i to control %i", node->_idx, clone->_idx, new_ctrl->_idx); |
|
894 |
phase->igvn().register_new_node_with_optimizer(clone); |
|
895 |
--i, --jmax; |
|
896 |
} |
|
897 |
assert(node->outcnt() == 0, "must be empty now"); |
|
898 |
||
899 |
// Node node is dead. |
|
900 |
phase->igvn().remove_dead_node(node); |
|
901 |
} |
|
902 |
return true; // unwind - return if a use was processed |
|
903 |
} |
|
904 |
||
905 |
// Clone a load to a specific catch_proj |
|
906 |
static Node* clone_load_to_catchproj(PhaseIdealLoop* phase, Node* load, Node* catch_proj) { |
|
907 |
Node* cloned_load = load->clone(); |
|
908 |
cloned_load->set_req(0, catch_proj); // set explicit control |
|
909 |
phase->set_ctrl(cloned_load, catch_proj); // update |
|
910 |
if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr(" Clone LOAD %i as %i to control %i", load->_idx, cloned_load->_idx, catch_proj->_idx); |
|
911 |
phase->igvn().register_new_node_with_optimizer(cloned_load); |
|
912 |
return cloned_load; |
|
913 |
} |
|
914 |
||
915 |
static Node* get_dominating_region(PhaseIdealLoop* phase, Node* node, Node* stop) { |
|
916 |
Node* region = node; |
|
917 |
while (!region->isa_Region()) { |
|
918 |
Node *up = phase->idom(region); |
|
919 |
assert(up != region, "Must not loop"); |
|
920 |
assert(up != stop, "Must not find original control"); |
|
921 |
region = up; |
|
922 |
} |
|
923 |
return region; |
|
924 |
} |
|
925 |
||
926 |
// Clone this load to each catch block |
|
927 |
static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) { |
|
928 |
bool trace = phase->C->directive()->ZTraceLoadBarriersOption; |
|
929 |
phase->igvn().set_delay_transform(true); |
|
930 |
||
931 |
// Verify pre conditions |
|
932 |
assert(ctrl->isa_Proj() && ctrl->in(0)->isa_Call(), "Must be a call proj"); |
|
933 |
assert(ctrl->raw_out(0)->isa_Catch(), "Must be a catch"); |
|
934 |
||
935 |
if (ctrl->raw_out(0)->isa_Catch()->outcnt() == 1) { |
|
936 |
if (trace) tty->print_cr("Cleaning up catch: Skipping load %i, call with single catch", load->_idx); |
|
937 |
return; |
|
938 |
} |
|
939 |
||
940 |
// Process the loads successor nodes - if any is between |
|
941 |
// the call and the catch blocks, they need to be cloned to. |
|
942 |
// This is done recursively |
|
943 |
int outcnt = load->outcnt(); |
|
944 |
uint index = 0; |
|
945 |
for (int i = 0; i < outcnt; i++) { |
|
946 |
if (index < load->outcnt()) { |
|
947 |
Node *n = load->raw_out(index); |
|
948 |
assert(!n->is_LoadBarrier(), "Sanity"); |
|
949 |
if (!fixup_uses_in_catch(phase, ctrl, n)) { |
|
950 |
// if no successor was cloned, progress to next out. |
|
951 |
index++; |
|
952 |
} |
|
953 |
} |
|
954 |
} |
|
955 |
||
956 |
// Now all the loads uses has been cloned down |
|
957 |
// Only thing left is to clone the loads, but they must end up |
|
958 |
// first in the catch blocks. |
|
959 |
||
960 |
// We clone the loads oo the catch blocks only when needed. |
|
961 |
// An array is used to map the catch blocks to each lazily cloned load. |
|
962 |
// In that way no extra unnecessary loads are cloned. |
|
963 |
||
964 |
// Any use dominated by original block must have an phi and a region added |
|
965 |
||
966 |
Node* catch_node = ctrl->raw_out(0); |
|
967 |
int number_of_catch_projs = catch_node->outcnt(); |
|
968 |
Node** proj_to_load_mapping = NEW_RESOURCE_ARRAY(Node*, number_of_catch_projs); |
|
969 |
Copy::zero_to_bytes(proj_to_load_mapping, sizeof(Node*) * number_of_catch_projs); |
|
970 |
||
971 |
// The phi_map is used to keep track of where phis have already been inserted |
|
972 |
int phi_map_len = phase->C->unique(); |
|
973 |
Node** phi_map = NEW_RESOURCE_ARRAY(Node*, phi_map_len); |
|
974 |
Copy::zero_to_bytes(phi_map, sizeof(Node*) * phi_map_len); |
|
975 |
||
976 |
for (unsigned int i = 0; i < load->outcnt(); i++) { |
|
977 |
Node* load_use_control = NULL; |
|
978 |
Node* load_use = load->raw_out(i); |
|
979 |
||
980 |
if (phase->has_ctrl(load_use)) { |
|
981 |
load_use_control = phase->get_ctrl(load_use); |
|
982 |
} else { |
|
983 |
load_use_control = load_use->in(0); |
|
984 |
} |
|
985 |
assert(load_use_control != NULL, "sanity"); |
|
986 |
if (trace) tty->print_cr(" Handling use: %i, with control: %i", load_use->_idx, load_use_control->_idx); |
|
987 |
||
988 |
// Some times the loads use is a phi. For them we need to determine from which catch block |
|
989 |
// the use is defined. |
|
990 |
bool load_use_is_phi = false; |
|
991 |
unsigned int load_use_phi_index = 0; |
|
992 |
Node* phi_ctrl = NULL; |
|
993 |
if (load_use->is_Phi()) { |
|
994 |
// Find phi input that matches load |
|
995 |
for (unsigned int u = 1; u < load_use->req(); u++) { |
|
996 |
if (load_use->in(u) == load) { |
|
997 |
load_use_is_phi = true; |
|
998 |
load_use_phi_index = u; |
|
999 |
assert(load_use->in(0)->is_Region(), "Region or broken"); |
|
1000 |
phi_ctrl = load_use->in(0)->in(u); |
|
1001 |
assert(phi_ctrl->is_CFG(), "check"); |
|
1002 |
assert(phi_ctrl != load, "check"); |
|
1003 |
break; |
|
1004 |
} |
|
1005 |
} |
|
1006 |
assert(load_use_is_phi, "must find"); |
|
1007 |
assert(load_use_phi_index > 0, "sanity"); |
|
1008 |
} |
|
1009 |
||
1010 |
// For each load use, see which catch projs dominates, create load clone lazily and reconnect |
|
1011 |
bool found_dominating_catchproj = false; |
|
1012 |
for (int c = 0; c < number_of_catch_projs; c++) { |
|
1013 |
Node* catchproj = catch_node->raw_out(c); |
|
1014 |
assert(catchproj != NULL && catchproj->isa_CatchProj(), "Sanity"); |
|
1015 |
||
1016 |
if (!phase->is_dominator(catchproj, load_use_control)) { |
|
1017 |
if (load_use_is_phi && phase->is_dominator(catchproj, phi_ctrl)) { |
|
1018 |
// The loads use is local to the catchproj. |
|
1019 |
// fall out and replace load with catch-local load clone. |
|
1020 |
} else { |
|
1021 |
continue; |
|
1022 |
} |
|
1023 |
} |
|
1024 |
assert(!found_dominating_catchproj, "Max one should match"); |
|
1025 |
||
1026 |
// Clone loads to catch projs |
|
1027 |
Node* load_clone = proj_to_load_mapping[c]; |
|
1028 |
if (load_clone == NULL) { |
|
1029 |
load_clone = clone_load_to_catchproj(phase, load, catchproj); |
|
1030 |
proj_to_load_mapping[c] = load_clone; |
|
1031 |
} |
|
1032 |
phase->igvn().rehash_node_delayed(load_use); |
|
1033 |
||
1034 |
if (load_use_is_phi) { |
|
1035 |
// phis are special - the load is defined from a specific control flow |
|
1036 |
load_use->set_req(load_use_phi_index, load_clone); |
|
1037 |
} else { |
|
1038 |
// Multipe edges can be replaced at once - on calls for example |
|
1039 |
load_use->replace_edge(load, load_clone); |
|
1040 |
} |
|
1041 |
--i; // more than one edge can have been removed, but the next is in later iterations |
|
1042 |
||
1043 |
// We could break the for-loop after finding a dominating match. |
|
1044 |
// But keep iterating to catch any bad idom early. |
|
1045 |
found_dominating_catchproj = true; |
|
1046 |
} |
|
1047 |
||
1048 |
// We found no single catchproj that dominated the use - The use is at a point after |
|
1049 |
// where control flow from multiple catch projs have merged. We will have to create |
|
1050 |
// phi nodes before the use and tie the output from the cloned loads together. It |
|
1051 |
// can be a single phi or a number of chained phis, depending on control flow |
|
1052 |
if (!found_dominating_catchproj) { |
|
1053 |
||
1054 |
// Use phi-control if use is a phi |
|
1055 |
if (load_use_is_phi) { |
|
1056 |
load_use_control = phi_ctrl; |
|
1057 |
} |
|
1058 |
assert(phase->is_dominator(ctrl, load_use_control), "Common use but no dominator"); |
|
1059 |
||
1060 |
// Clone a load on all paths |
|
1061 |
for (int c = 0; c < number_of_catch_projs; c++) { |
|
1062 |
Node* catchproj = catch_node->raw_out(c); |
|
1063 |
Node* load_clone = proj_to_load_mapping[c]; |
|
1064 |
if (load_clone == NULL) { |
|
1065 |
load_clone = clone_load_to_catchproj(phase, load, catchproj); |
|
1066 |
proj_to_load_mapping[c] = load_clone; |
|
1067 |
} |
|
1068 |
} |
|
1069 |
||
1070 |
// Move up dominator tree from use until dom front is reached |
|
1071 |
Node* next_region = get_dominating_region(phase, load_use_control, ctrl); |
|
1072 |
while (phase->idom(next_region) != catch_node) { |
|
1073 |
next_region = phase->idom(next_region); |
|
1074 |
if (trace) tty->print_cr("Moving up idom to region ctrl %i", next_region->_idx); |
|
1075 |
} |
|
1076 |
assert(phase->is_dominator(catch_node, next_region), "Sanity"); |
|
1077 |
||
1078 |
// Create or reuse phi node that collect all cloned loads and feed it to the use. |
|
1079 |
Node* test_phi = phi_map[next_region->_idx]; |
|
1080 |
if ((test_phi != NULL) && test_phi->is_Phi()) { |
|
1081 |
// Reuse an already created phi |
|
1082 |
if (trace) tty->print_cr(" Using cached Phi %i on load_use %i", test_phi->_idx, load_use->_idx); |
|
1083 |
phase->igvn().rehash_node_delayed(load_use); |
|
1084 |
load_use->replace_edge(load, test_phi); |
|
1085 |
// Now this use is done |
|
1086 |
} else { |
|
1087 |
// Otherwise we need to create one or more phis |
|
1088 |
PhiNode* next_phi = new PhiNode(next_region, load->type()); |
|
1089 |
phi_map[next_region->_idx] = next_phi; // cache new phi |
|
1090 |
phase->igvn().rehash_node_delayed(load_use); |
|
1091 |
load_use->replace_edge(load, next_phi); |
|
1092 |
||
1093 |
int dominators_of_region = 0; |
|
1094 |
do { |
|
1095 |
// New phi, connect to region and add all loads as in. |
|
1096 |
Node* region = next_region; |
|
1097 |
assert(region->isa_Region() && region->req() > 2, "Catch dead region nodes"); |
|
1098 |
PhiNode* new_phi = next_phi; |
|
1099 |
||
1100 |
if (trace) tty->print_cr("Created Phi %i on load %i with control %i", new_phi->_idx, load->_idx, region->_idx); |
|
1101 |
||
1102 |
// Need to add all cloned loads to the phi, taking care that the right path is matched |
|
1103 |
dominators_of_region = 0; // reset for new region |
|
1104 |
for (unsigned int reg_i = 1; reg_i < region->req(); reg_i++) { |
|
1105 |
Node* region_pred = region->in(reg_i); |
|
1106 |
assert(region_pred->is_CFG(), "check"); |
|
1107 |
bool pred_has_dominator = false; |
|
1108 |
for (int c = 0; c < number_of_catch_projs; c++) { |
|
1109 |
Node* catchproj = catch_node->raw_out(c); |
|
1110 |
if (phase->is_dominator(catchproj, region_pred)) { |
|
1111 |
new_phi->set_req(reg_i, proj_to_load_mapping[c]); |
|
1112 |
if (trace) tty->print_cr(" - Phi in(%i) set to load %i", reg_i, proj_to_load_mapping[c]->_idx); |
|
1113 |
pred_has_dominator = true; |
|
1114 |
dominators_of_region++; |
|
1115 |
break; |
|
1116 |
} |
|
1117 |
} |
|
1118 |
||
1119 |
// Sometimes we need to chain several phis. |
|
1120 |
if (!pred_has_dominator) { |
|
1121 |
assert(dominators_of_region <= 1, "More than one region can't require extra phi"); |
|
1122 |
if (trace) tty->print_cr(" - Region %i pred %i not dominated by catch proj", region->_idx, region_pred->_idx); |
|
1123 |
// Continue search on on this region_pred |
|
1124 |
// - walk up to next region |
|
1125 |
// - create a new phi and connect to first new_phi |
|
1126 |
next_region = get_dominating_region(phase, region_pred, ctrl); |
|
1127 |
||
1128 |
// Lookup if there already is a phi, create a new otherwise |
|
1129 |
Node* test_phi = phi_map[next_region->_idx]; |
|
1130 |
if ((test_phi != NULL) && test_phi->is_Phi()) { |
|
1131 |
next_phi = test_phi->isa_Phi(); |
|
1132 |
dominators_of_region++; // record that a match was found and that we are done |
|
1133 |
if (trace) tty->print_cr(" Using cached phi Phi %i on control %i", next_phi->_idx, next_region->_idx); |
|
1134 |
} else { |
|
1135 |
next_phi = new PhiNode(next_region, load->type()); |
|
1136 |
phi_map[next_region->_idx] = next_phi; |
|
1137 |
} |
|
1138 |
new_phi->set_req(reg_i, next_phi); |
|
1139 |
} |
|
1140 |
} |
|
1141 |
||
1142 |
new_phi->set_req(0, region); |
|
1143 |
phase->igvn().register_new_node_with_optimizer(new_phi); |
|
1144 |
phase->set_ctrl(new_phi, region); |
|
1145 |
||
1146 |
assert(dominators_of_region != 0, "Must have found one this iteration"); |
|
1147 |
} while (dominators_of_region == 1); |
|
1148 |
} |
|
1149 |
--i; |
|
1150 |
} |
|
1151 |
} // end of loop over uses |
|
1152 |
||
1153 |
assert(load->outcnt() == 0, "All uses should be handled"); |
|
1154 |
phase->igvn().remove_dead_node(load); |
|
1155 |
phase->C->print_method(PHASE_CALL_CATCH_CLEANUP, 4, load->_idx); |
|
1156 |
||
1157 |
// Now we should be home |
|
1158 |
phase->igvn().set_delay_transform(false); |
|
1159 |
} |
|
1160 |
||
1161 |
// Sort out the loads that are between a call ant its catch blocks |
|
1162 |
static void process_catch_cleanup_candidate(PhaseIdealLoop* phase, LoadNode* load) { |
|
1163 |
bool trace = phase->C->directive()->ZTraceLoadBarriersOption; |
|
1164 |
||
1165 |
Node* ctrl = phase->get_ctrl(load); |
|
1166 |
if (!ctrl->is_Proj() || (ctrl->in(0) == NULL) || !ctrl->in(0)->isa_Call()) { |
|
1167 |
return; |
|
1168 |
} |
|
1169 |
||
1170 |
Node* catch_node = ctrl->isa_Proj()->raw_out(0); |
|
1171 |
if (catch_node->is_Catch()) { |
|
1172 |
if (catch_node->outcnt() > 1) { |
|
1173 |
call_catch_cleanup_one(phase, load, ctrl); |
|
1174 |
} else { |
|
1175 |
if (trace) tty->print_cr("Call catch cleanup with only one catch: load %i ", load->_idx); |
|
1176 |
} |
|
1177 |
} |
|
1178 |
} |
|
1179 |
||
1180 |
void ZBarrierSetC2::barrier_insertion_phase(Compile* C, PhaseIterGVN& igvn) const { |
|
1181 |
PhaseIdealLoop::optimize(igvn, LoopOptsZBarrierInsertion); |
|
1182 |
if (C->failing()) return; |
|
1183 |
} |
|
1184 |
||
1185 |
bool ZBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { |
|
1186 |
||
1187 |
if (mode == LoopOptsZBarrierInsertion) { |
|
1188 |
// First make sure all loads between call and catch are moved to the catch block |
|
1189 |
clean_catch_blocks(phase); |
|
1190 |
||
1191 |
// Then expand barriers on all loads |
|
1192 |
insert_load_barriers(phase); |
|
1193 |
||
1194 |
// Handle all Unsafe that need barriers. |
|
1195 |
insert_barriers_on_unsafe(phase); |
|
1196 |
||
1197 |
phase->C->clear_major_progress(); |
|
1198 |
return true; |
|
1199 |
} else { |
|
1200 |
return false; |
|
1201 |
} |
|
1202 |
} |
|
1203 |
||
1204 |
static bool can_simplify_cas(LoadStoreNode* node) { |
|
1205 |
if (node->isa_LoadStoreConditional()) { |
|
1206 |
Node *expected_in = node->as_LoadStoreConditional()->in(LoadStoreConditionalNode::ExpectedIn); |
|
1207 |
return (expected_in->get_ptr_type() == TypePtr::NULL_PTR); |
|
1208 |
} else { |
|
1209 |
return false; |
|
1210 |
} |
|
1211 |
} |
|
1212 |
||
1213 |
static void insert_barrier_before_unsafe(PhaseIdealLoop* phase, LoadStoreNode* old_node) { |
|
1214 |
||
1215 |
Compile *C = phase->C; |
|
1216 |
PhaseIterGVN &igvn = phase->igvn(); |
|
1217 |
LoadStoreNode* zclone = NULL; |
|
1218 |
bool is_weak = false; |
|
1219 |
||
1220 |
Node *in_ctrl = old_node->in(MemNode::Control); |
|
1221 |
Node *in_mem = old_node->in(MemNode::Memory); |
|
1222 |
Node *in_adr = old_node->in(MemNode::Address); |
|
1223 |
Node *in_val = old_node->in(MemNode::ValueIn); |
|
1224 |
const TypePtr *adr_type = old_node->adr_type(); |
|
1225 |
const TypePtr* load_type = TypeOopPtr::BOTTOM; // The type for the load we are adding |
|
1226 |
||
1227 |
switch (old_node->Opcode()) { |
|
1228 |
case Op_CompareAndExchangeP: { |
|
1229 |
zclone = new ZCompareAndExchangePNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), |
|
1230 |
adr_type, old_node->get_ptr_type(), ((CompareAndExchangeNode*)old_node)->order()); |
|
1231 |
load_type = old_node->bottom_type()->is_ptr(); |
|
1232 |
break; |
|
1233 |
} |
|
1234 |
case Op_WeakCompareAndSwapP: { |
|
1235 |
if (can_simplify_cas(old_node)) { |
|
1236 |
break; |
|
1237 |
} |
|
1238 |
is_weak = true; |
|
1239 |
zclone = new ZWeakCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), |
|
1240 |
((CompareAndSwapNode*)old_node)->order()); |
|
1241 |
adr_type = TypePtr::BOTTOM; |
|
1242 |
break; |
|
1243 |
} |
|
1244 |
case Op_CompareAndSwapP: { |
|
1245 |
if (can_simplify_cas(old_node)) { |
|
1246 |
break; |
|
1247 |
} |
|
1248 |
zclone = new ZCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), |
|
1249 |
((CompareAndSwapNode*)old_node)->order()); |
|
1250 |
adr_type = TypePtr::BOTTOM; |
|
1251 |
break; |
|
1252 |
} |
|
1253 |
case Op_GetAndSetP: { |
|
1254 |
zclone = new ZGetAndSetPNode(in_ctrl, in_mem, in_adr, in_val, old_node->adr_type(), old_node->get_ptr_type()); |
|
1255 |
load_type = old_node->bottom_type()->is_ptr(); |
|
1256 |
break; |
|
1257 |
} |
|
1258 |
} |
|
1259 |
if (zclone != NULL) { |
|
1260 |
igvn.register_new_node_with_optimizer(zclone, old_node); |
|
1261 |
||
1262 |
// Make load |
|
1263 |
LoadPNode *load = new LoadPNode(NULL, in_mem, in_adr, adr_type, load_type, MemNode::unordered, |
|
1264 |
LoadNode::DependsOnlyOnTest); |
|
1265 |
load_set_expanded_barrier(load); |
|
1266 |
igvn.register_new_node_with_optimizer(load); |
|
1267 |
igvn.replace_node(old_node, zclone); |
|
1268 |
||
1269 |
Node *barrier = new LoadBarrierNode(C, NULL, in_mem, load, in_adr, is_weak); |
|
1270 |
Node *barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop); |
|
1271 |
Node *barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control); |
|
1272 |
||
1273 |
igvn.register_new_node_with_optimizer(barrier); |
|
1274 |
igvn.register_new_node_with_optimizer(barrier_val); |
|
1275 |
igvn.register_new_node_with_optimizer(barrier_ctrl); |
|
1276 |
||
1277 |
// loop over all of in_ctrl usages and move to barrier_ctrl |
|
1278 |
for (DUIterator_Last imin, i = in_ctrl->last_outs(imin); i >= imin; --i) { |
|
1279 |
Node *use = in_ctrl->last_out(i); |
|
1280 |
uint l; |
|
1281 |
for (l = 0; use->in(l) != in_ctrl; l++) {} |
|
1282 |
igvn.replace_input_of(use, l, barrier_ctrl); |
|
1283 |
} |
|
1284 |
||
1285 |
load->set_req(MemNode::Control, in_ctrl); |
|
1286 |
barrier->set_req(LoadBarrierNode::Control, in_ctrl); |
|
1287 |
zclone->add_req(barrier_val); // add req as keep alive. |
|
1288 |
||
1289 |
C->print_method(PHASE_ADD_UNSAFE_BARRIER, 4, zclone->_idx); |
|
1290 |
} |
|
1291 |
} |
|
1292 |
||
1293 |
void ZBarrierSetC2::insert_barriers_on_unsafe(PhaseIdealLoop* phase) const { |
|
1294 |
Compile *C = phase->C; |
|
1295 |
PhaseIterGVN &igvn = phase->igvn(); |
|
1296 |
uint new_ids = C->unique(); |
|
1297 |
VectorSet visited(Thread::current()->resource_area()); |
|
1298 |
GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL); |
|
1299 |
nodeStack.push(C->root()); |
|
1300 |
visited.test_set(C->root()->_idx); |
|
1301 |
||
1302 |
// Traverse all nodes, visit all unsafe ops that require a barrier |
|
1303 |
while (nodeStack.length() > 0) { |
|
1304 |
Node *n = nodeStack.pop(); |
|
1305 |
||
1306 |
bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup |
|
1307 |
if (is_old_node) { |
|
1308 |
if (n->is_LoadStore()) { |
|
1309 |
LoadStoreNode* lsn = n->as_LoadStore(); |
|
1310 |
if (lsn->has_barrier()) { |
|
1311 |
BasicType bt = lsn->in(MemNode::Address)->bottom_type()->basic_type(); |
|
1312 |
assert ((bt == T_OBJECT || bt == T_ARRAY), "Sanity test"); |
|
1313 |
insert_barrier_before_unsafe(phase, lsn); |
|
1314 |
} |
|
1315 |
} |
|
1316 |
} |
|
1317 |
for (uint i = 0; i < n->len(); i++) { |
|
1318 |
if (n->in(i)) { |
|
1319 |
if (!visited.test_set(n->in(i)->_idx)) { |
|
1320 |
nodeStack.push(n->in(i)); |
|
1321 |
} |
|
1322 |
} |
|
1323 |
} |
|
1324 |
} |
|
1325 |
||
1326 |
igvn.optimize(); |
|
1327 |
C->print_method(PHASE_ADD_UNSAFE_BARRIER, 2); |
|
1328 |
} |
|
1329 |
||
1330 |
// The purpose of ZBarrierSetC2::clean_catch_blocks is to prepare the IR for |
|
1331 |
// splicing in load barrier nodes. |
|
1332 |
// |
|
1333 |
// The problem is that we might have instructions between a call and its catch nodes. |
|
1334 |
// (This is usually handled in PhaseCFG:call_catch_cleanup, which clones mach nodes in |
|
1335 |
// already scheduled blocks.) We can't have loads that require barriers there, |
|
1336 |
// because we need to splice in new control flow, and that would violate the IR. |
|
1337 |
// |
|
1338 |
// clean_catch_blocks find all Loads that require a barrier and clone them and any |
|
1339 |
// dependent instructions to each use. The loads must be in the beginning of the catch block |
|
1340 |
// before any store. |
|
1341 |
// |
|
1342 |
// Sometimes the loads use will be at a place dominated by all catch blocks, then we need |
|
1343 |
// a load in each catch block, and a Phi at the dominated use. |
|
1344 |
||
1345 |
void ZBarrierSetC2::clean_catch_blocks(PhaseIdealLoop* phase) const { |
|
1346 |
||
1347 |
Compile *C = phase->C; |
|
1348 |
uint new_ids = C->unique(); |
|
1349 |
PhaseIterGVN &igvn = phase->igvn(); |
|
1350 |
VectorSet visited(Thread::current()->resource_area()); |
|
1351 |
GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL); |
|
1352 |
nodeStack.push(C->root()); |
|
1353 |
visited.test_set(C->root()->_idx); |
|
1354 |
||
1355 |
// Traverse all nodes, visit all loads that require a barrier |
|
1356 |
while(nodeStack.length() > 0) { |
|
1357 |
Node *n = nodeStack.pop(); |
|
1358 |
||
1359 |
bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup |
|
1360 |
if (n->is_Load() && is_old_node) { |
|
1361 |
LoadNode* load = n->isa_Load(); |
|
1362 |
// only care about loads that will have a barrier |
|
1363 |
if (load_require_barrier(load)) { |
|
1364 |
process_catch_cleanup_candidate(phase, load); |
|
1365 |
} |
|
1366 |
} |
|
1367 |
||
1368 |
for (uint i = 0; i < n->len(); i++) { |
|
1369 |
if (n->in(i)) { |
|
1370 |
if (!visited.test_set(n->in(i)->_idx)) { |
|
1371 |
nodeStack.push(n->in(i)); |
|
1372 |
} |
|
1373 |
} |
|
1374 |
} |
|
1375 |
} |
|
1376 |
||
1377 |
C->print_method(PHASE_CALL_CATCH_CLEANUP, 2); |
|
1378 |
} |
|
1379 |
||
1380 |
class DomDepthCompareClosure : public CompareClosure<LoadNode*> { |
|
1381 |
PhaseIdealLoop* _phase; |
|
1382 |
||
1383 |
public: |
|
1384 |
DomDepthCompareClosure(PhaseIdealLoop* phase) : _phase(phase) { } |
|
1385 |
||
1386 |
int do_compare(LoadNode* const &n1, LoadNode* const &n2) { |
|
1387 |
int d1 = _phase->dom_depth(_phase->get_ctrl(n1)); |
|
1388 |
int d2 = _phase->dom_depth(_phase->get_ctrl(n2)); |
|
1389 |
if (d1 == d2) { |
|
1390 |
// Compare index if the depth is the same, ensures all entries are unique. |
|
1391 |
return n1->_idx - n2->_idx; |
|
1392 |
} else { |
|
1393 |
return d2 - d1; |
|
1394 |
} |
|
1395 |
} |
|
1396 |
}; |
|
1397 |
||
1398 |
// Traverse graph and add all loadPs to list, sorted by dom depth |
|
1399 |
void gather_loadnodes_sorted(PhaseIdealLoop* phase, GrowableArray<LoadNode*>* loadList) { |
|
1400 |
||
1401 |
VectorSet visited(Thread::current()->resource_area()); |
|
1402 |
GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL); |
|
1403 |
DomDepthCompareClosure ddcc(phase); |
|
1404 |
||
1405 |
nodeStack.push(phase->C->root()); |
|
1406 |
while(nodeStack.length() > 0) { |
|
1407 |
Node *n = nodeStack.pop(); |
|
1408 |
if (visited.test(n->_idx)) { |
|
1409 |
continue; |
|
1410 |
} |
|
1411 |
||
1412 |
if (n->isa_Load()) { |
|
1413 |
LoadNode *load = n->as_Load(); |
|
1414 |
if (load_require_barrier(load)) { |
|
1415 |
assert(phase->get_ctrl(load) != NULL, "sanity"); |
|
1416 |
assert(phase->dom_depth(phase->get_ctrl(load)) != 0, "sanity"); |
|
1417 |
loadList->insert_sorted(&ddcc, load); |
|
1418 |
} |
|
1419 |
} |
|
1420 |
||
1421 |
visited.set(n->_idx); |
|
1422 |
for (uint i = 0; i < n->req(); i++) { |
|
1423 |
if (n->in(i)) { |
|
1424 |
if (!visited.test(n->in(i)->_idx)) { |
|
1425 |
nodeStack.push(n->in(i)); |
|
50525
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
1426 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
1427 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
1428 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
1429 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
1430 |
} |
767cdb97f103
8204210: Implementation: JEP 333: ZGC: A Scalable Low-Latency Garbage Collector (Experimental)
pliden
parents:
diff
changeset
|
1431 |
|
55307 | 1432 |
// Add LoadBarriers to all LoadPs |
1433 |
void ZBarrierSetC2::insert_load_barriers(PhaseIdealLoop* phase) const { |
|
1434 |
||
1435 |
bool trace = phase->C->directive()->ZTraceLoadBarriersOption; |
|
1436 |
GrowableArray<LoadNode *> loadList(Thread::current()->resource_area(), 0, 0, NULL); |
|
1437 |
gather_loadnodes_sorted(phase, &loadList); |
|
1438 |
||
1439 |
PhaseIterGVN &igvn = phase->igvn(); |
|
1440 |
int count = 0; |
|
1441 |
||
1442 |
for (GrowableArrayIterator<LoadNode *> loadIter = loadList.begin(); loadIter != loadList.end(); ++loadIter) { |
|
1443 |
LoadNode *load = *loadIter; |
|
52568 | 1444 |
|
55307 | 1445 |
if (load_has_expanded_barrier(load)) { |
1446 |
continue; |
|
1447 |
} |
|
53439 | 1448 |
|
55307 | 1449 |
do { |
1450 |
// Insert a barrier on a loadP |
|
1451 |
// if another load is found that needs to be expanded first, retry on that one |
|
1452 |
LoadNode* result = insert_one_loadbarrier(phase, load, phase->get_ctrl(load)); |
|
1453 |
while (result != NULL) { |
|
1454 |
result = insert_one_loadbarrier(phase, result, phase->get_ctrl(result)); |
|
52568 | 1455 |
} |
55307 | 1456 |
} while (!load_has_expanded_barrier(load)); |
52568 | 1457 |
} |
53439 | 1458 |
|
55307 | 1459 |
phase->C->print_method(PHASE_INSERT_BARRIER, 2); |
1460 |
} |
|
1461 |
||
1462 |
void push_antidependent_stores(PhaseIdealLoop* phase, Node_Stack& nodestack, LoadNode* start_load) { |
|
1463 |
// push all stores on the same mem, that can_alias |
|
1464 |
// Any load found must be handled first |
|
1465 |
PhaseIterGVN &igvn = phase->igvn(); |
|
1466 |
int load_alias_idx = igvn.C->get_alias_index(start_load->adr_type()); |
|
1467 |
||
1468 |
Node *mem = start_load->in(1); |
|
1469 |
for (DUIterator_Fast imax, u = mem->fast_outs(imax); u < imax; u++) { |
|
1470 |
Node *mem_use = mem->fast_out(u); |
|
1471 |
||
1472 |
if (mem_use == start_load) continue; |
|
1473 |
if (!mem_use->is_Store()) continue; |
|
1474 |
if (!phase->has_ctrl(mem_use)) continue; |
|
1475 |
if (phase->get_ctrl(mem_use) != phase->get_ctrl(start_load)) continue; |
|
1476 |
||
1477 |
// add any aliasing store in this block |
|
1478 |
StoreNode *store = mem_use->isa_Store(); |
|
1479 |
const TypePtr *adr_type = store->adr_type(); |
|
1480 |
if (igvn.C->can_alias(adr_type, load_alias_idx)) { |
|
1481 |
nodestack.push(store, 0); |
|
1482 |
} |
|
1483 |
} |
|
1484 |
} |
|
1485 |
||
1486 |
LoadNode* ZBarrierSetC2::insert_one_loadbarrier(PhaseIdealLoop* phase, LoadNode* start_load, Node* ctrl) const { |
|
1487 |
bool trace = phase->C->directive()->ZTraceLoadBarriersOption; |
|
1488 |
PhaseIterGVN &igvn = phase->igvn(); |
|
1489 |
||
1490 |
// Check for other loadPs at the same loop depth that is reachable by a DFS |
|
1491 |
// - if found - return it. It needs to be inserted first |
|
1492 |
// - otherwise proceed and insert barrier |
|
1493 |
||
1494 |
VectorSet visited(Thread::current()->resource_area()); |
|
1495 |
Node_Stack nodestack(100); |
|
1496 |
||
1497 |
nodestack.push(start_load, 0); |
|
1498 |
push_antidependent_stores(phase, nodestack, start_load); |
|
1499 |
||
1500 |
while(!nodestack.is_empty()) { |
|
1501 |
Node* n = nodestack.node(); // peek |
|
1502 |
nodestack.pop(); |
|
1503 |
if (visited.test(n->_idx)) { |
|
1504 |
continue; |
|
1505 |
} |
|
1506 |
||
1507 |
if (n->is_Load() && n != start_load && load_require_barrier(n->as_Load()) && !load_has_expanded_barrier(n->as_Load())) { |
|
1508 |
// Found another load that needs a barrier in the same block. Must expand later loads first. |
|
1509 |
if (trace) tty->print_cr(" * Found LoadP %i on DFS", n->_idx); |
|
1510 |
return n->as_Load(); // return node that should be expanded first |
|
1511 |
} |
|
1512 |
||
1513 |
if (!phase->has_ctrl(n)) continue; |
|
1514 |
if (phase->get_ctrl(n) != phase->get_ctrl(start_load)) continue; |
|
1515 |
if (n->is_Phi()) continue; |
|
1516 |
||
1517 |
visited.set(n->_idx); |
|
1518 |
// push all children |
|
1519 |
for (DUIterator_Fast imax, ii = n->fast_outs(imax); ii < imax; ii++) { |
|
1520 |
Node* c = n->fast_out(ii); |
|
1521 |
if (c != NULL) { |
|
1522 |
nodestack.push(c, 0); |
|
1523 |
} |
|
1524 |
} |
|
1525 |
} |
|
1526 |
||
1527 |
insert_one_loadbarrier_inner(phase, start_load, ctrl, visited); |
|
1528 |
return NULL; |
|
52568 | 1529 |
} |
1530 |
||
55307 | 1531 |
void ZBarrierSetC2::insert_one_loadbarrier_inner(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl, VectorSet visited2) const { |
1532 |
PhaseIterGVN &igvn = phase->igvn(); |
|
1533 |
Compile* C = igvn.C; |
|
1534 |
bool trace = C->directive()->ZTraceLoadBarriersOption; |
|
1535 |
||
1536 |
// create barrier |
|
1537 |
Node* barrier = new LoadBarrierNode(C, NULL, load->in(LoadNode::Memory), NULL, load->in(LoadNode::Address), load_has_weak_barrier(load)); |
|
1538 |
Node* barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop); |
|
1539 |
Node* barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control); |
|
1540 |
||
1541 |
if (trace) tty->print_cr("Insert load %i with barrier: %i and ctrl : %i", load->_idx, barrier->_idx, ctrl->_idx); |
|
1542 |
||
1543 |
// Splice control |
|
1544 |
// - insert barrier control diamond between loads ctrl and ctrl successor on path to block end. |
|
1545 |
// - If control successor is a catch, step over to next. |
|
1546 |
Node* ctrl_succ = NULL; |
|
1547 |
for (DUIterator_Fast imax, j = ctrl->fast_outs(imax); j < imax; j++) { |
|
1548 |
Node* tmp = ctrl->fast_out(j); |
|
53439 | 1549 |
|
55307 | 1550 |
// - CFG nodes is the ones we are going to splice (1 only!) |
1551 |
// - Phi nodes will continue to hang from the region node! |
|
1552 |
// - self loops should be skipped |
|
1553 |
if (tmp->is_Phi() || tmp == ctrl) { |
|
1554 |
continue; |
|
1555 |
} |
|
1556 |
||
1557 |
if (tmp->is_CFG()) { |
|
1558 |
assert(ctrl_succ == NULL, "There can be only one"); |
|
1559 |
ctrl_succ = tmp; |
|
1560 |
continue; |
|
1561 |
} |
|
1562 |
} |
|
1563 |
||
1564 |
// Now splice control |
|
1565 |
assert(ctrl_succ != load, "sanity"); |
|
1566 |
assert(ctrl_succ != NULL, "Broken IR"); |
|
1567 |
bool found = false; |
|
1568 |
for(uint k = 0; k < ctrl_succ->req(); k++) { |
|
1569 |
if (ctrl_succ->in(k) == ctrl) { |
|
1570 |
assert(!found, "sanity"); |
|
1571 |
if (trace) tty->print_cr(" Move CFG ctrl_succ %i to barrier_ctrl", ctrl_succ->_idx); |
|
1572 |
igvn.replace_input_of(ctrl_succ, k, barrier_ctrl); |
|
1573 |
found = true; |
|
1574 |
k--; |
|
1575 |
} |
|
52568 | 1576 |
} |
53439 | 1577 |
|
55307 | 1578 |
// For all successors of ctrl - move all visited to become successors of barrier_ctrl instead |
1579 |
for (DUIterator_Fast imax, r = ctrl->fast_outs(imax); r < imax; r++) { |
|
1580 |
Node* tmp = ctrl->fast_out(r); |
|
1581 |
if (visited2.test(tmp->_idx) && (tmp != load)) { |
|
1582 |
if (trace) tty->print_cr(" Move ctrl_succ %i to barrier_ctrl", tmp->_idx); |
|
1583 |
igvn.replace_input_of(tmp, 0, barrier_ctrl); |
|
1584 |
--r; --imax; |
|
1585 |
} |
|
1586 |
} |
|
1587 |
||
1588 |
// Move the loads user to the barrier |
|
1589 |
for (DUIterator_Fast imax, i = load->fast_outs(imax); i < imax; i++) { |
|
1590 |
Node* u = load->fast_out(i); |
|
1591 |
if (u->isa_LoadBarrier()) { |
|
1592 |
continue; |
|
1593 |
} |
|
1594 |
||
1595 |
// find correct input - replace with iterator? |
|
1596 |
for(uint j = 0; j < u->req(); j++) { |
|
1597 |
if (u->in(j) == load) { |
|
1598 |
igvn.replace_input_of(u, j, barrier_val); |
|
1599 |
--i; --imax; // Adjust the iterator of the *outer* loop |
|
1600 |
break; // some nodes (calls) might have several uses from the same node |
|
1601 |
} |
|
1602 |
} |
|
1603 |
} |
|
1604 |
||
1605 |
// Connect barrier to load and control |
|
1606 |
barrier->set_req(LoadBarrierNode::Oop, load); |
|
1607 |
barrier->set_req(LoadBarrierNode::Control, ctrl); |
|
1608 |
||
1609 |
igvn.rehash_node_delayed(load); |
|
1610 |
igvn.register_new_node_with_optimizer(barrier); |
|
1611 |
igvn.register_new_node_with_optimizer(barrier_val); |
|
1612 |
igvn.register_new_node_with_optimizer(barrier_ctrl); |
|
1613 |
load_set_expanded_barrier(load); |
|
1614 |
||
1615 |
C->print_method(PHASE_INSERT_BARRIER, 3, load->_idx); |
|
52711 | 1616 |
} |
55307 | 1617 |
|
1618 |
// The bad_mask in the ThreadLocalData shouldn't have an anti-dep-check. |
|
1619 |
// The bad_mask address if of type TypeRawPtr, but that will alias |
|
1620 |
// InitializeNodes until the type system is expanded. |
|
1621 |
bool ZBarrierSetC2::needs_anti_dependence_check(const Node* node) const { |
|
1622 |
MachNode* mnode = node->as_Mach(); |
|
1623 |
if (mnode != NULL) { |
|
1624 |
intptr_t offset = 0; |
|
1625 |
const TypePtr *adr_type2 = NULL; |
|
1626 |
const Node* base = mnode->get_base_and_disp(offset, adr_type2); |
|
1627 |
if ((base != NULL) && |
|
1628 |
(base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_ThreadLocal) && |
|
1629 |
(offset == in_bytes(ZThreadLocalData::address_bad_mask_offset()))) { |
|
1630 |
return false; |
|
1631 |
} |
|
1632 |
} |
|
1633 |
return true; |
|
1634 |
} |